diff --git a/lib/clouddrop-upload.js b/lib/clouddrop-upload.js index 25a08de..5223b88 100644 --- a/lib/clouddrop-upload.js +++ b/lib/clouddrop-upload.js @@ -105,7 +105,7 @@ class ClouddropUploader { let bytesRead = 0; async function* generate() { yield preambleBuf; - const fileStream = fs.createReadStream(filePath, { highWaterMark: 256 * 1024 }); + const fileStream = fs.createReadStream(filePath, { highWaterMark: 1024 * 1024 }); for await (const chunk of fileStream) { if (signal && signal.aborted) throw new Error('Aborted'); if (throttle) await throttle.consume(chunk.length, signal); diff --git a/lib/config-store.js b/lib/config-store.js index c455f35..9b81c8b 100644 --- a/lib/config-store.js +++ b/lib/config-store.js @@ -187,6 +187,7 @@ class ConfigStore { this._writeQueue = Promise.resolve(); // Serializes all writes to prevent race conditions this._cache = null; this._cacheKey = ''; + this._perfLog = null; // Migrate config from old location if current doesn't exist if (!fs.existsSync(this.filePath) && app && app.isPackaged) { @@ -228,7 +229,23 @@ class ConfigStore { catch { return JSON.parse(JSON.stringify(obj)); } } + setPerfLog(fn) { this._perfLog = typeof fn === 'function' ? fn : null; } + load() { + if (!this._perfLog) return this._loadImpl(); + const hadCache = !!this._cache; + const t0 = performance.now(); + const r = this._loadImpl(); + const dt = performance.now() - t0; + if (dt >= 20) { + const q = ((r && r.globalSettings && r.globalSettings.pendingQueue) || []).length; + const h = (r && r.history || []).length; + this._perfLog(`config-load wall=${dt.toFixed(0)}ms cache=${hadCache ? 'hit' : 'miss'} hist=${h} queue=${q}`); + } + return r; + } + + _loadImpl() { try { // In-memory cache keyed on the file's mtime+size. The processed config // (merged + credential-decrypted) is reparsed/re-decrypted from disk ONLY @@ -356,7 +373,16 @@ class ConfigStore { } _commit(config) { - return this._atomicWrite(this._serializeForDisk(config)); + if (!this._perfLog) return this._atomicWrite(this._serializeForDisk(config)); + const t0 = performance.now(); + const data = this._serializeForDisk(config); + const dt = performance.now() - t0; + if (dt >= 20) { + const q = ((config.globalSettings && config.globalSettings.pendingQueue) || []).length; + const h = (config.history || []).length; + this._perfLog(`config-serialize wall=${dt.toFixed(0)}ms bytes=${data.length} hist=${h} queue=${q}`); + } + return this._atomicWrite(data); } _enqueueWrite(fn) { diff --git a/lib/doodstream-upload.js b/lib/doodstream-upload.js index 02fcaaa..1c3821b 100644 --- a/lib/doodstream-upload.js +++ b/lib/doodstream-upload.js @@ -339,7 +339,7 @@ class DoodstreamUploader { const epilogueBuf = Buffer.from(epilogue, 'utf-8'); const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; - const CHUNK_SIZE = 256 * 1024; + const CHUNK_SIZE = 1024 * 1024; let bytesRead = 0; async function* generate() { diff --git a/lib/hosters.js b/lib/hosters.js index 1069243..5bd7a5d 100644 --- a/lib/hosters.js +++ b/lib/hosters.js @@ -288,7 +288,7 @@ function createUploadBody(filePath, formFields, onProgress, throttle, signal) { const { boundary, preambleBuf, epilogueBuf, totalSize, fileSize } = buildMultipart(filePath, formFields); let bytesRead = 0; - const CHUNK_SIZE = 256 * 1024; + const CHUNK_SIZE = 1024 * 1024; async function* generate() { yield preambleBuf; diff --git a/lib/vidmoly-upload.js b/lib/vidmoly-upload.js index 608fcaf..cd9b145 100644 --- a/lib/vidmoly-upload.js +++ b/lib/vidmoly-upload.js @@ -187,7 +187,7 @@ class VidmolyUploader { const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; let bytesRead = 0; - const CHUNK_SIZE = 256 * 1024; + const CHUNK_SIZE = 1024 * 1024; async function* generate() { yield preambleBuf; diff --git a/lib/voe-upload.js b/lib/voe-upload.js index 3f523d2..4c105c3 100644 --- a/lib/voe-upload.js +++ b/lib/voe-upload.js @@ -242,7 +242,7 @@ class VoeUploader { const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; let bytesRead = 0; - const CHUNK_SIZE = 256 * 1024; + const CHUNK_SIZE = 1024 * 1024; async function* generate() { yield preambleBuf; diff --git a/main.js b/main.js index 4796004..b844a85 100644 --- a/main.js +++ b/main.js @@ -51,6 +51,7 @@ let _lastImportPath = null; let dropTargetWindow = null; let tray = null; const configStore = new ConfigStore(app); +configStore.setPerfLog((m) => { try { logInfo(m); } catch {} }); let uploadManager = null; let diagnosticAgent = null; let _diagHandler = null; diff --git a/package.json b/package.json index c092181..efe1356 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "multi-hoster-uploader", - "version": "3.3.97", + "version": "3.3.98", "description": "Upload files to doodstream, voe, vidmoly, byse simultaneously", "main": "main.js", "scripts": { diff --git a/tasks/lessons.md b/tasks/lessons.md index bdb77ed..ac82809 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -1,5 +1,33 @@ # Lessons +## 2026-06-21 — Ein-Variablen-Disziplin: nicht zwei Fixes bündeln, wenn einer den anderen maskiert +**Symptom:** Nach dem tp=8-Win wollte ich in EINEM Build A (1MB highWaterMark, Read-Burst) + B (Renderer +chunked rAF Batch-Drain, der 243ms-Longtask) + C-Instrument shippen. +**Root cause / Korrektur (Advisor):** Der Renderer war 14/15 Fenstern gesund; der EINE 243ms-Longtask (W14) +ist laut beiden Agenten DOWNSTREAM des Main-Thread-Read-Bursts (geflutetes IPC). Fix A reduziert diese +Stalls → der Renderer-Longtask verschwindet wahrscheinlich OHNE B. B mitzuliefern (a) verwässert die nächste +Messung (war die Besserung A oder B?) und (b) fasst den Progress-Hot-Path an, der hier schon gebissen hat +(formatDateTime-Burst, ghost-fix). +**Regel:** Wenn Fix A einen vermuteten Symptom-Treiber X reduziert und Fix B genau X behandeln würde — +NUR A shippen, messen, B nur nachziehen wenn X überlebt. Sonst kann das nächste Log nicht sauber attribuieren. +Bei gekoppelten Symptomen ist die Reihenfolge (Upstream-Fix zuerst, dann messen) wichtiger als „alles auf +einmal". +**Wie anwenden:** Vor dem Bündeln fragen: „Maskiert Fix A die Wirkung, die Fix B beheben soll?" Wenn ja → +entkoppeln, A zuerst, eine Variable pro Build. + +## 2026-06-21 — Nicht aus EINEM konfundierten Sample eine Ursache behaupten +**Symptom:** Ich wollte dem User sagen „1-Sekunden-Persist-Freeze gefunden" auf Basis von W13 (max=1021ms, +heap→142MB). +**Root cause / Korrektur (Advisor):** W13 ist EIN Sample und konfundiert (hat gleichzeitig FSReqCallback=66) +und das EINZIGE Heap-Spike-Fenster. Die anderen isolierten Maxes (W4 415ms/heap41, W10 852ms/heap18) haben +NIEDRIGEN Heap → sind KEIN 140MB-structuredClone+stringify → eine andere Ursache (account-failed sync load() +nahe Connection-Churn). Eine Behauptung aus einem konfundierten Punkt hätte den falschen Fix priorisiert. +**Regel:** Bei isolierten Spitzen erst die Co-Signale (heap, FSReq, gc, Nachbarfenster) gegenchecken, ob sie +EINE Familie sind. Wenn die Magnitude-Signatur (hier: Heap-Spike) nicht bei allen passt → es sind mehrere +Ursachen. „Instrumentieren + bestätigen", nicht „gefunden", solange nur ein konfundierter Punkt existiert. +**Wie anwenden:** Vor „Ursache X gefunden": gibt es ≥2 unkonfundierte Samples mit derselben Signatur? Wenn +nein → als Hypothese formulieren und messen, nicht als Befund verkaufen. + ## 2026-06-21 — Histogram-Korrelation beweist KEINE Kausalrichtung; rss-Mathe als Sanity-Check **Symptom:** ELD-Spikes korrelierten exakt mit hohem `FSReqCallback` (File-Reads in flight) → ich wollte sofort ein Read-Concurrency-Semaphore über 5 Dateien bauen. diff --git a/tasks/todo.md b/tasks/todo.md index a492c75..de67d9d 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -1,3 +1,46 @@ +# v3.3.98 — read-burst absorption (1MB hwm) + persist/load instrument; B (renderer) DEFERRED + +v3.3.97 (threadpool 64→8) was a DECISIVE win: mean ELD 200ms→~11ms at 70 active (18×), renderer healthy +14/15 windows. User: "ganz flüssig isses noch nicht". A 5-agent ultracode workflow + adversarial verify +localized the RESIDUAL to TWO distinct, measured spike sources (full data: subagents output wjskjo1xk): + +1. READ-BURSTS (tail W13/14/15, 15:18:53-19:04): FSReqCallback 66/70/46 vs threadpool=8 (~8.75× queue + depth), SimpleWriteWrap collapses to 7/4/24, mean climbs 12.9→30.3→41.9ms. GC EXCLUDED (gcMax ≤27ms + always). The FSReq↔SimpleWrite inversion at stable active=70/pending=1287 proves reads are CAUSAL, not + a symptom of a block elsewhere. +2. SYNC CONFIG PERSIST (suspected): save()→load() reparses the WHOLE electron-config.json (1287-job + pendingQueue nested in globalSettings + full history) on every persist because _atomicWrite nulls the + cache; _serializeForDisk JSON.stringify(...,null,2) of all of it. W13's single 1021ms max with heap→142MB + fits a big synchronous structuredClone+stringify. CAVEAT (advisor): W13 is ONE confounded sample (also + FSReq=66) and the ONLY heap-spike window; W4(415ms,heap41) & W10(852ms,heap18) are LOW-heap → NOT persist + clones → likely the SECONDARY suspect: account-failed's synchronous configStore.load() per failure near + connection churn (W6 teardown had doodstream connect-timeouts). So: INSTRUMENT, don't claim "found a 1s + freeze". + +SHIPPED v3.3.98 (one-variable discipline — advisor cut B to keep the next measurement clean): +- A: highWaterMark 256KB→1MB in all 5 streaming read loops (hosters.js:291, doodstream:342, voe:245, + vidmoly:190 CHUNK_SIZE consts; clouddrop:108 inline — NOT clouddrop:12's 16MB server chunk). Keep tp=8. + Deepens per-stream read-ahead 0.43s→~1.7s (absorbs threadpool-queue latency so writes don't starve), + 4× fewer read completions + allocs. Zero multipart byte-risk (Content-Length=preamble+fileSize+epilogue, + independent of chunk size). REVERSIBLE PROBE; read-semaphore held in reserve (trigger: FSReq still ~70 + + writes starved + mean elevated after 1MB). +- C-instrument (BROADENED per advisor): config-store.js times load() (full reparse, incl. account-failed + path) AND _commit serialize; logs `config-load wall=Xms cache=hit/miss hist=N queue=M` and + `config-serialize wall=Xms bytes=Y hist=N queue=M` when ≥20ms (perfLog hook set in main.js via + configStore.setPerfLog→logInfo). load() split into wrapper + _loadImpl. 397 tests pass. +- B (renderer chunked rAF batch drain, app.js:188-193 — the 243ms longtask at W14) DEFERRED: renderer was + healthy 14/15 windows and the one longtask is DOWNSTREAM of the main-thread read-burst flooding IPC. + Fix A should make it self-heal. Bundling B would confound attribution + touches the progress hot path + that bit before (formatDateTime burst, ghost-fix). Add B next round ONLY if renderer still janks after A. + +NEXT LOG answers 3 things cleanly: (1) did A kill the read-bursts (FSReq per-window + tail mean drop)? +(2) is the persist/load actually heavy (new config-load/config-serialize lines + their wall/queue/hist)? +(3) did the renderer self-heal from A alone (longtasks back to 0)? Then decide: persist refactor for v3.3.99 +(queue-out-of-config OR cache-repopulation — latter lower-risk but renderer's incoming globalSettings isn't +default-merged like load() produces, so confirm merge-equivalence first), and/or B, and/or read-semaphore. + +--- + # v3.3.97 — DECISIVE ELD finding: file-read phase-flip + threadpool 64→8 + GC instrument The v3.3.96 `eventloop-delay` logs gave the decisive signal. At CONSTANT active-count, the system flips