perf+fix: long-session lag, tab-switch lag, log-recovery
Bundles four findings from a stability audit plus the missing-log bug the user reported. 1. main.js _flushUploadLog: ENOENT after the log file's directory got deleted mid-session was swallowed; the buffer was cleared before appendFile so entries were silently lost and the cached target kept pointing at the dead path. Now: mkdirSync(recursive) before every flush idempotently recreates a missing dir; on any append error we invalidate the cache, prepend the chunk back to the buffer and schedule a retry. Survives "user dragged the log folder into the trash and didn't notice". 2. renderer/app.js queueJobs auto-prune: with the default removeFromQueueOnDone=false the queue grew forever. Past ~5000 entries every render became O(N) on a perpetually-growing N and the user saw progressive scroll/tab lag. Cap the in-queue terminal jobs (done/skipped/error/aborted) at 500 most-recent on each batch-done; oldest get pruned with their index entries. 3. sortQueueJobs dynamic-key throttle: status/speed/progress/size sorts ran a full O(N log N) sort on every progress tick. Added a 200ms-window cache for the dynamic-key path so the sort is reused within the same UI_UPDATE_INTERVAL — invisibly small reorder lag, massive cost savings at 5000+ jobs. 4. renderHistoryTable delegated listeners: every Verlauf-tab switch was binding one click listener per row (5000 listeners on a long-history user) and rebuilding the entire <tbody> innerHTML. Single delegated tbody listener covers both row-click (copy link) and th-click (sort), bound once per container via dataset flag. 5. sessionFilesData (recent-files panel) cap at 2000 entries with matching _sessionFileKeys cleanup using the existing separator. Stops the lower-panel innerHTML write from inflating to multiple MB on long sessions. 87/87 tests still green.
This commit is contained in:
Administrator
parent
e49f5493fe
commit
66f8b47b6d
19
main.js
19
main.js
@ -300,6 +300,11 @@ function _flushUploadLog() {
|
||||
if (_uploadLogWriting || _uploadLogBuffer.length === 0) return;
|
||||
const target = _resolveUploadLogTarget();
|
||||
if (!target) { _uploadLogBuffer.length = 0; return; }
|
||||
// Guard against the file's parent directory having been deleted/moved
|
||||
// since the cache was filled. mkdirSync(recursive:true) is a no-op when
|
||||
// the dir already exists; recreates it otherwise. Without this, every
|
||||
// subsequent flush silently fails with ENOENT and entries are lost.
|
||||
try { fs.mkdirSync(path.dirname(target.path), { recursive: true }); } catch {}
|
||||
const chunk = _uploadLogBuffer.join('');
|
||||
_uploadLogBuffer.length = 0;
|
||||
_uploadLogWriting = true;
|
||||
@ -307,6 +312,18 @@ function _flushUploadLog() {
|
||||
_uploadLogWriting = false;
|
||||
if (err) {
|
||||
debugLog(`uploadLog append failed: ${err.message}`);
|
||||
// Recovery: drop the cached target so the next flush re-resolves
|
||||
// (could be ENOENT after dir delete, ENOSPC, EBUSY etc.) and
|
||||
// restore the chunk so we don't lose entries on the retry.
|
||||
_invalidateUploadLogTargetCache();
|
||||
_uploadLogBuffer.unshift(chunk);
|
||||
// Retry on the next event-loop tick rather than tight-looping.
|
||||
if (!_uploadLogFlushTimer) {
|
||||
_uploadLogFlushTimer = setTimeout(() => {
|
||||
_uploadLogFlushTimer = null;
|
||||
_flushUploadLog();
|
||||
}, 1000);
|
||||
}
|
||||
} else if (target.isFallback && !_uploadLogFallbackWarned) {
|
||||
_uploadLogFallbackWarned = true;
|
||||
// Auto-persist the working fallback into the user's config so the
|
||||
@ -317,7 +334,7 @@ function _flushUploadLog() {
|
||||
mainWindow.webContents.send('upload-log-fallback', { fallbackPath: target.path });
|
||||
}
|
||||
}
|
||||
if (_uploadLogBuffer.length) setImmediate(_flushUploadLog);
|
||||
if (_uploadLogBuffer.length && !_uploadLogFlushTimer) setImmediate(_flushUploadLog);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -1151,6 +1151,14 @@ const _collatorSimple = new Intl.Collator('de');
|
||||
let _queueSortCache = { sig: '', result: [], jobsRef: null };
|
||||
const _STATIC_SORT_KEYS = new Set(['filename', 'host']);
|
||||
|
||||
// Dynamic-key sort throttle: status/speed/progress/size change on every
|
||||
// progress tick, so a strict per-call sort is O(N log N) per render. Within
|
||||
// one UI_UPDATE_INTERVAL window (200ms), reuse the previous sort even if it's
|
||||
// slightly out of order — the user can't perceive sub-200ms reorder lag, and
|
||||
// at 5000 queued jobs this is the difference between smooth and stuttering.
|
||||
let _dynamicSortCache = { key: '', direction: '', jobsRef: null, result: null, ts: 0 };
|
||||
const DYNAMIC_SORT_REFRESH_MS = 200;
|
||||
|
||||
function sortQueueJobs(jobs) {
|
||||
const { key, direction } = queueSortState;
|
||||
const factor = direction === 'asc' ? 1 : -1;
|
||||
@ -1159,6 +1167,13 @@ function sortQueueJobs(jobs) {
|
||||
if (sig && _queueSortCache.sig === sig && _queueSortCache.jobsRef === jobs) {
|
||||
return _queueSortCache.result;
|
||||
}
|
||||
// Dynamic-key throttle: same key+direction+array, sorted within the last
|
||||
// 200ms → reuse. Cleared on user-initiated sort changes (different key).
|
||||
if (!canCache && _dynamicSortCache.jobsRef === jobs &&
|
||||
_dynamicSortCache.key === key && _dynamicSortCache.direction === direction &&
|
||||
_dynamicSortCache.result && (Date.now() - _dynamicSortCache.ts) < DYNAMIC_SORT_REFRESH_MS) {
|
||||
return _dynamicSortCache.result;
|
||||
}
|
||||
|
||||
const sorted = jobs.slice().sort((a, b) => {
|
||||
let cmp = 0;
|
||||
@ -1171,6 +1186,7 @@ function sortQueueJobs(jobs) {
|
||||
return cmp * factor;
|
||||
});
|
||||
if (sig) _queueSortCache = { sig, result: sorted, jobsRef: jobs };
|
||||
else _dynamicSortCache = { key, direction, jobsRef: jobs, result: sorted, ts: Date.now() };
|
||||
return sorted;
|
||||
}
|
||||
|
||||
@ -1969,6 +1985,37 @@ function handleBatchDone(summary) {
|
||||
}
|
||||
queueJobs = nextJobs;
|
||||
renderQueueTable();
|
||||
} else {
|
||||
// Auto-prune for the default (removeOnDone=false) too: cap done/skipped
|
||||
// jobs at the most recent N so the queue can't grow unbounded across
|
||||
// long sessions. Without this, sortQueueJobs / _computeQueueStats /
|
||||
// renderQueueTable all become O(N) on a forever-growing N and the UI
|
||||
// starts visibly lagging once the queue passes a few thousand entries.
|
||||
// 500 most-recent terminal jobs is enough for "see what just happened"
|
||||
// while stopping the runaway growth.
|
||||
const TERMINAL_KEEP_LIMIT = 500;
|
||||
const terminalIdxs = [];
|
||||
for (let i = 0; i < queueJobs.length; i++) {
|
||||
const s = queueJobs[i].status;
|
||||
if (s === 'done' || s === 'skipped' || s === 'error' || s === 'aborted') terminalIdxs.push(i);
|
||||
}
|
||||
if (terminalIdxs.length > TERMINAL_KEEP_LIMIT) {
|
||||
const dropCount = terminalIdxs.length - TERMINAL_KEEP_LIMIT;
|
||||
// terminalIdxs is in insertion order → first `dropCount` are the
|
||||
// oldest. Remove those, keep everything else.
|
||||
const dropSet = new Set(terminalIdxs.slice(0, dropCount));
|
||||
const nextJobs = [];
|
||||
for (let i = 0; i < queueJobs.length; i++) {
|
||||
if (dropSet.has(i)) {
|
||||
removeJobFromIndex(queueJobs[i]);
|
||||
selectedJobIds.delete(queueJobs[i].id);
|
||||
} else {
|
||||
nextJobs.push(queueJobs[i]);
|
||||
}
|
||||
}
|
||||
queueJobs = nextJobs;
|
||||
renderQueueTable();
|
||||
}
|
||||
}
|
||||
|
||||
if (queueJobs.some((job) => !['done', 'skipped'].includes(job.status))) persistQueueStateSoon(true);
|
||||
@ -2187,6 +2234,10 @@ function syncSelectedFilesFromQueue() {
|
||||
selectedFiles = Array.from(fileMap.values());
|
||||
}
|
||||
|
||||
// Cap recent-files panel growth so a multi-thousand-job session doesn't
|
||||
// turn every renderRecentUploadsPanel call into a multi-MB innerHTML write.
|
||||
const SESSION_FILES_CAP = 2000;
|
||||
|
||||
function maybeAddSessionFile(job) {
|
||||
if (!job) return;
|
||||
|
||||
@ -2207,6 +2258,18 @@ function maybeAddSessionFile(job) {
|
||||
order: sessionFilesData.length
|
||||
});
|
||||
_sessionDoneCount++;
|
||||
// Drop oldest entries past the cap to keep render cost bounded.
|
||||
// Without this, sessionFilesData grows unbounded across the session
|
||||
// and every renderRecentUploadsPanel call becomes a megabyte-sized
|
||||
// innerHTML write — visible as scroll/click lag in the lower panel.
|
||||
if (sessionFilesData.length > SESSION_FILES_CAP) {
|
||||
const drop = sessionFilesData.length - SESSION_FILES_CAP;
|
||||
for (let i = 0; i < drop; i++) {
|
||||
const r = sessionFilesData[i];
|
||||
_sessionFileKeys.delete(`${r.link}${r.filename}${r.host}`);
|
||||
}
|
||||
sessionFilesData = sessionFilesData.slice(drop);
|
||||
}
|
||||
// Coalesce rapid successive adds into one render per frame.
|
||||
scheduleRecentRender();
|
||||
}
|
||||
@ -3577,22 +3640,28 @@ function renderHistoryTable(container) {
|
||||
html += '</tbody></table>';
|
||||
container.innerHTML = html;
|
||||
|
||||
container.querySelectorAll('th.sortable').forEach(th => {
|
||||
th.addEventListener('click', () => {
|
||||
const key = th.dataset.historySort;
|
||||
if (historySortState.key === key) historySortState.direction = historySortState.direction === 'asc' ? 'desc' : 'asc';
|
||||
else { historySortState.key = key; historySortState.direction = key === 'date' ? 'desc' : 'asc'; }
|
||||
renderHistoryTable(container);
|
||||
// Delegated listeners: bind once per render-target instead of once per
|
||||
// row/header. With a 5000-row history the per-row bind path was a
|
||||
// 5000-iteration synchronous loop on every Verlauf-tab switch — the
|
||||
// dominant cause of "tab switching lags" in the user report.
|
||||
if (!container.dataset.historyListenersBound) {
|
||||
container.dataset.historyListenersBound = '1';
|
||||
container.addEventListener('click', (e) => {
|
||||
const th = e.target.closest('th.sortable');
|
||||
if (th && container.contains(th)) {
|
||||
const key = th.dataset.historySort;
|
||||
if (historySortState.key === key) historySortState.direction = historySortState.direction === 'asc' ? 'desc' : 'asc';
|
||||
else { historySortState.key = key; historySortState.direction = key === 'date' ? 'desc' : 'asc'; }
|
||||
renderHistoryTable(container);
|
||||
return;
|
||||
}
|
||||
const row = e.target.closest('.history-row');
|
||||
if (row && !row.classList.contains('error')) {
|
||||
const link = row.dataset.link;
|
||||
if (link) { window.api.copyToClipboard(link); showCopyToast('Link kopiert'); }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
container.querySelectorAll('.history-row').forEach(row => {
|
||||
row.addEventListener('click', () => {
|
||||
if (row.classList.contains('error')) return;
|
||||
const link = row.dataset.link;
|
||||
if (link) { window.api.copyToClipboard(link); showCopyToast('Link kopiert'); }
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function sortHistoryRows(rows) {
|
||||
|
||||
@ -1,27 +1,32 @@
|
||||
# Perf/Stabilität Audit Log
|
||||
# 3.3.0 — Performance + Log-Recovery
|
||||
|
||||
## Abgeschlossen in dieser Session
|
||||
## Probleme (User-Bericht)
|
||||
1. Lag beim Tab-Wechsel + Scrollen nach langer Session.
|
||||
2. File-Uploader-Log: bei einem von zwei Hostern gar keine Log-Einträge — möglicherweise nach Datei/Verzeichnis-Löschung kein Recovery.
|
||||
|
||||
- [x] **3.1.3** — Doppel-Render beim Retry vieler Jobs entfernt.
|
||||
- [x] **3.1.4** — Byse disk-space als account-level klassifiziert (vorher fälschlich file-rejected).
|
||||
- [x] **3.1.5** — Pre-job-swap hinter Semaphore-Queue + Late-Resolve bei save-config.
|
||||
- [x] **3.1.6** — `JSON.stringify(files/hosters)` aus start-upload debugLog raus.
|
||||
- [x] **3.1.7** — Status-Change-Events im Renderer via rAF coalesced.
|
||||
- [x] **3.1.8** — Byse-Poller race-condition fix (kein "newFiles.length===1"-Fallback mehr) + transient-network-classifier mit 2 Tests abgesichert + Memory-Snapshot-Logger bei Batch-Boundaries.
|
||||
## Root Causes (aus Audit)
|
||||
|
||||
## Getestet / validiert
|
||||
### Performance
|
||||
- **queueJobs wächst unbounded** (Default `removeFromQueueOnDone=false`). Sortierung + Stats + In-place-Render skalieren O(N) und O(N log N) je render. Nach ~5000 Jobs spürbar.
|
||||
- **`sortQueueJobs` cached nur für static keys** (filename, host). Bei status/speed/progress: voller Sort jedes Mal.
|
||||
- **`renderHistoryTable`** bindet per-row click listener + voller `innerHTML`-Rebuild bei jedem Tab-Switch.
|
||||
- **`renderRecentUploadsPanel`** baut komplettes innerHTML bei sort-change neu, `sessionFilesData` Array ungecapt.
|
||||
|
||||
- 82 Unit-Tests grün
|
||||
- Error-Klassifikation (fileRejected / accountError / transient) hat jetzt eindeutige, getestete Trennlinien
|
||||
- Rotation-Pipeline durchspielbar in Tests (session memory, late-add, override-precedence)
|
||||
### Log-Bug
|
||||
- `_flushUploadLog` clear buffer **vor** appendFile. Bei `ENOENT` (Datei oder Dir wurde gelöscht) → silent log + buffer verloren.
|
||||
- `_resolveUploadLogTarget` cached den Path. Bei mid-session Verzeichnis-Löschung wird der cached target wiederverwendet → permanent ENOENT bis Neustart.
|
||||
|
||||
## Nicht angegangen (Follow-ups)
|
||||
## Plan
|
||||
- [ ] Log: bei error in `appendFile` → cache invalidieren + buffer prepend + retry
|
||||
- [ ] Log: vor jedem flush mkdirSync (idempotent, recreated deleted dir)
|
||||
- [ ] queueJobs: auto-prune nach `handleBatchDone` für älteste 'done' jobs jenseits Cap (z.B. 500)
|
||||
- [ ] sortQueueJobs: für dynamic keys → coalesce auf 1× pro UI_UPDATE_INTERVAL (200ms)
|
||||
- [ ] renderHistoryTable: delegated click-listener auf tbody (nicht per-row), short-circuit wenn `_historyDirty=false`
|
||||
- [ ] renderRecentUploadsPanel: `sessionFilesData` auf letzten 2000 Einträge cappen, `_sessionFileKeys` parallel prunen
|
||||
- [ ] removeFromQueueOnDone: O(N²) → splice via `_jobIndexById` (oder defer auf batch-done)
|
||||
- [ ] Tests: Log-recovery, queue-prune, sort-throttle
|
||||
- [ ] Release als 3.3.0
|
||||
|
||||
- **Throughput bei 20+ parallelen Uploads** — bräuchte Lasttest-Setup mit Mock-Hoster; speculative ohne User-Beschwerde.
|
||||
- **Netz-Ausfall-Recovery** — Klassifikator getestet, echter Network-Interrupt-Integrationstest nicht gemacht (aufwendiger Setup, real-world: Transients werden korrekt erkannt).
|
||||
- **Live Memory-Tracking** — Batch-Boundary-Logging liefert jetzt Datenpunkte. Bei wachsendem `rss`/`heapUsed` über Batches hinweg: Leak-Verdacht, dann in DevTools profilen.
|
||||
|
||||
## Bekannte externe Issues (nicht fixbar bei uns)
|
||||
|
||||
- Byse "Not video file format" bei manchen MKV-Releases ist Byse-seitige Codec/Container-Validierung. Lösung: Datei vorher remuxen (z.B. mit mkvtoolnix).
|
||||
- Real-Debrid-Downloader + Multi-Hoster-Upload konkurrieren um File-Handles → WinError 5 beim Rename. Workaround: Downloader komplett durchlaufen lassen bevor Queue gezogen wird.
|
||||
## Skip (Low Impact / kosmetisch)
|
||||
- CSS transition fix
|
||||
- Module-level Sets clearing (genug Memory bisher)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user