From 38ecc6a4cbf572c64398b5e97df53c6b9748fe96 Mon Sep 17 00:00:00 2001 From: Administrator Date: Tue, 28 Apr 2026 03:37:13 +0200 Subject: [PATCH] perf(queue): coalesce removeFromQueueOnDone removals into one filter pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleProgress on a 'done' event with removeFromQueueOnDone=true was calling queueJobs.filter() once per event. With 500 parallel jobs all finishing at roughly the same time, that's 500 × O(N) = O(N²) work synchronously on the IPC handler thread — visible as a brief UI freeze when a big batch completes. Coalesce into one microtask: removeJobFromIndex + selection cleanup stay synchronous (so subsequent lookups see the right state), but the array rewrite is deferred to a single filter against a Set of all ids that came in this tick. JS microtask runs after the sync IPC batch, so within one batch-of-events we get one filter pass instead of N. beforeunload drains the pending set synchronously before persisting so removeFromQueueOnDone=true users don't see jobs reappear after restart that they expected to be gone. --- renderer/app.js | 31 +++++++++++++++++++++++++++++-- tasks/todo.md | 43 ++++++++++++++++++------------------------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/renderer/app.js b/renderer/app.js index 25f0ebe..028d7d2 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -47,6 +47,10 @@ const _sessionTrackedJobs = new Set(); // Job IDs already counted for totalBytes const _sessionDoneJobs = new Set(); // Job IDs already counted for uploadedBytes const _completedUploadKeys = new Set(); // 'filepath|hoster' keys for done uploads (survives removeFromQueueOnDone) const _deletedJobIds = new Set(); // IDs of jobs explicitly deleted by user (prevents re-creation from stale progress callbacks) +// Coalesce removeFromQueueOnDone removals into one filter pass per microtask +// to avoid O(N²) behaviour when a burst of jobs finish at once. +let _pendingDoneRemovalIds = new Set(); +let _doneRemovalScheduled = false; const queueSortState = { key: 'filename', direction: 'asc' }; // History state @@ -1925,11 +1929,26 @@ function handleProgress(data) { _completedUploadKeys.add(`${job.file}|${job.hoster}`); } - // Remove finished jobs from queue immediately if setting is enabled + // Remove finished jobs from queue if setting is enabled. Coalesce the + // actual array filter into one microtask: a burst of 500 done events + // would otherwise fire 500 individual O(N) filters = O(N²) work, visible + // as a brief UI freeze when a big batch finishes. Index/selection are + // updated synchronously so subsequent lookups see the right state — only + // the array rewrite is deferred. if (job.status === 'done' && config.globalSettings && config.globalSettings.removeFromQueueOnDone) { removeJobFromIndex(job); selectedJobIds.delete(job.id); - queueJobs = queueJobs.filter(j => j !== job); + _pendingDoneRemovalIds.add(job.id); + if (!_doneRemovalScheduled) { + _doneRemovalScheduled = true; + queueMicrotask(() => { + _doneRemovalScheduled = false; + if (_pendingDoneRemovalIds.size === 0) return; + const drop = _pendingDoneRemovalIds; + _pendingDoneRemovalIds = new Set(); + queueJobs = queueJobs.filter(j => !drop.has(j.id)); + }); + } } // Status changes (done/error/etc) get one coalesced update per frame so a @@ -3683,6 +3702,14 @@ window.addEventListener('beforeunload', () => { } clearTimeout(queuePersistTimer); queuePersistTimer = null; + // Drain pending done-removals synchronously before persisting so jobs the + // user expected to disappear (removeFromQueueOnDone=true) don't reappear + // on next launch. Microtask wouldn't run before the sync IPC below. + if (_pendingDoneRemovalIds.size > 0) { + const drop = _pendingDoneRemovalIds; + _pendingDoneRemovalIds = new Set(); + queueJobs = queueJobs.filter(j => !drop.has(j.id)); + } const globalSettings = { ...(config.globalSettings || {}), pendingQueue: buildPersistedQueueState() diff --git a/tasks/todo.md b/tasks/todo.md index 09c6b03..c76a10c 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -1,32 +1,25 @@ -# 3.3.0 — Performance + Log-Recovery +# Verbesserungs-Loop — open items -## Probleme (User-Bericht) -1. Lag beim Tab-Wechsel + Scrollen nach langer Session. -2. File-Uploader-Log: bei einem von zwei Hostern gar keine Log-Einträge — möglicherweise nach Datei/Verzeichnis-Löschung kein Recovery. +## Released +- ✅ 3.3.0 — Performance-Fixes (queue-cap, sort-throttle, history-delegation, recent-cap) + Log-Recovery -## Root Causes (aus Audit) +## Open items (priorisiert) + +### Stabilität +- [ ] **removeFromQueueOnDone O(N²)** in `handleProgress` (renderer/app.js) — bei 500 gleichzeitig fertig werdenden Jobs 500× ein O(N)-`queueJobs.filter()` aufgerufen. Coalesce über microtask + Set. +- [ ] **fileuploader.log wächst unbounded** — automatische Rotation bei >50MB (rename → .1, neue Datei). +- [ ] **`_jobLogCollector`** (main.js) — wird nur bei start-upload geleert, nicht bei batch-done. Bei vielen Batches ohne neuen start-upload wächst es. Cleanup bei batch-done für jobs die nicht mehr in queueJobs sind. ### Performance -- **queueJobs wächst unbounded** (Default `removeFromQueueOnDone=false`). Sortierung + Stats + In-place-Render skalieren O(N) und O(N log N) je render. Nach ~5000 Jobs spürbar. -- **`sortQueueJobs` cached nur für static keys** (filename, host). Bei status/speed/progress: voller Sort jedes Mal. -- **`renderHistoryTable`** bindet per-row click listener + voller `innerHTML`-Rebuild bei jedem Tab-Switch. -- **`renderRecentUploadsPanel`** baut komplettes innerHTML bei sort-change neu, `sessionFilesData` Array ungecapt. +- [ ] **`applyQueueSelectionClasses`** (renderer/app.js:891) — `tbody.querySelectorAll` bei jedem Klick. Bei 5000-Jobs-Queue O(N) per click. Cache last rendered range. -### Log-Bug -- `_flushUploadLog` clear buffer **vor** appendFile. Bei `ENOENT` (Datei oder Dir wurde gelöscht) → silent log + buffer verloren. -- `_resolveUploadLogTarget` cached den Path. Bei mid-session Verzeichnis-Löschung wird der cached target wiederverwendet → permanent ENOENT bis Neustart. +### Code-Qualität +- [ ] **Test-Coverage für 3.3.0** — keine Tests für die queue-cap-prune-Logik in handleBatchDone, sortQueueJobs dynamic-throttle, log-error-recovery. -## Plan -- [ ] Log: bei error in `appendFile` → cache invalidieren + buffer prepend + retry -- [ ] Log: vor jedem flush mkdirSync (idempotent, recreated deleted dir) -- [ ] queueJobs: auto-prune nach `handleBatchDone` für älteste 'done' jobs jenseits Cap (z.B. 500) -- [ ] sortQueueJobs: für dynamic keys → coalesce auf 1× pro UI_UPDATE_INTERVAL (200ms) -- [ ] renderHistoryTable: delegated click-listener auf tbody (nicht per-row), short-circuit wenn `_historyDirty=false` -- [ ] renderRecentUploadsPanel: `sessionFilesData` auf letzten 2000 Einträge cappen, `_sessionFileKeys` parallel prunen -- [ ] removeFromQueueOnDone: O(N²) → splice via `_jobIndexById` (oder defer auf batch-done) -- [ ] Tests: Log-recovery, queue-prune, sort-throttle -- [ ] Release als 3.3.0 +### UX-Politur +- [ ] **CSS `.queue-row` transition** auf `:hover` scopen (aktuell auf jedem row → unnötiger Repaint bei Status-Flip). +- [ ] Module-level Sets `_sessionTrackedJobs`/`_sessionDoneJobs`/`_completedUploadKeys` werden nie geleert — minor memory growth. -## Skip (Low Impact / kosmetisch) -- CSS transition fix -- Module-level Sets clearing (genug Memory bisher) +## Loop-Notes +- Cron-Job `01e33ae1` läuft alle 30min (:07/:37), Session-only. +- Pro Iteration: GENAU EIN Issue. Auto-Release bei grünen Tests. Boundary: keine Features, keine Major-Refactors.