diff --git a/lib/queue-dedup.js b/lib/queue-dedup.js new file mode 100644 index 0000000..384f304 --- /dev/null +++ b/lib/queue-dedup.js @@ -0,0 +1,65 @@ +// Startup queue auto-dedup logic. Extracted from renderer/app.js +// _autoDeduplicateFromLog so the decision can be unit-tested without a DOM or +// the renderer's module-level state. +// +// Loaded both as a CommonJS module (Node tests) and as a browser global +// (renderer/app.js via index.html script tag) so a single implementation backs +// runtime and tests — no drift. +// +// Behaviour: on launch the restored queue is compared against the lifetime +// upload log. ONLY genuinely-completed ('done') jobs that also appear in the +// log are dropped — that's pure decluttering of work that already finished. +// +// Pending jobs (preview / queued) and failed ones (error / aborted) are NEVER +// dropped here, even if a same-name+hoster line exists in the log. Those are +// work the user intentionally has queued (often a deliberate re-upload of a +// file that was uploaded before). The old code filtered on log-presence alone, +// regardless of status, so the ENTIRE restored queue vanished on the next +// restart/update whenever the files had been uploaded previously — surfacing as +// an empty "Dateien hierhin ziehen oder klicken" queue. Manual log import +// (importUploadLog) stays separate and explicit for users who do want bulk +// dedup of pending jobs. + +(function (root) { + 'use strict'; + + function _key(fileName, hoster) { + return `${String(fileName).toLowerCase()}|${String(hoster).toLowerCase()}`; + } + + /** + * Partition restored queue jobs into kept vs removed, given lifetime log + * entries. Removes only 'done' jobs whose fileName|hoster is in the log. + * @param {Array<{status:string,fileName:string,hoster:string}>} jobs + * @param {Array<{fileName:string,hoster:string}>} logEntries + * @returns {{ kept: Array, removed: Array }} + */ + function partitionRestoredJobsByLog(jobs, logEntries) { + const kept = []; + const removed = []; + if (!Array.isArray(jobs) || jobs.length === 0) return { kept, removed }; + + const logKeys = new Set(); + for (const e of (Array.isArray(logEntries) ? logEntries : [])) { + if (e && e.fileName && e.hoster) logKeys.add(_key(e.fileName, e.hoster)); + } + + for (const job of jobs) { + const isDone = job && job.status === 'done' && job.fileName && job.hoster; + if (isDone && logKeys.has(_key(job.fileName, job.hoster))) { + removed.push(job); + } else { + kept.push(job); + } + } + return { kept, removed }; + } + + const api = { partitionRestoredJobsByLog }; + + if (typeof module !== 'undefined' && module.exports) { + module.exports = api; + } else if (root) { + root.QueueDedup = api; + } +})(typeof window !== 'undefined' ? window : this); diff --git a/renderer/app.js b/renderer/app.js index 2c7b9a2..472aa0d 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -3997,24 +3997,20 @@ async function _autoDeduplicateFromLog() { try { const entries = await window.api.readOwnUploadLog(); if (!entries || entries.length === 0) return; - const logKeys = new Set(); - for (const entry of entries) { - logKeys.add(`${entry.fileName.toLowerCase()}|${entry.hoster.toLowerCase()}`); - } - let removed = 0; - queueJobs = queueJobs.filter(job => { - const key = `${job.fileName.toLowerCase()}|${job.hoster.toLowerCase()}`; - if (logKeys.has(key)) { + // Only 'done' jobs are dropped here (declutter completed uploads). Pending + // and failed jobs survive even if their name+hoster is in the log — they're + // intentional queued work. Decision lives in lib/queue-dedup.js (Node-tested, + // see tests/queue-dedup.test.js) so it can't silently regress to nuking the + // whole restored queue on restart/update. + const { kept, removed } = window.QueueDedup.partitionRestoredJobsByLog(queueJobs, entries); + if (removed.length > 0) { + queueJobs = kept; + for (const job of removed) { if (job.file && job.hoster) _completedUploadKeys.add(`${job.file}|${job.hoster}`); - removed++; - return false; } - return true; - }); - if (removed > 0) { rebuildJobIndex(); syncSelectedFilesFromQueue(); - window.api.debugLog(`auto-dedup: removed ${removed} already-uploaded jobs from restored queue (${entries.length} log entries)`); + window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`); } } catch {} } diff --git a/renderer/index.html b/renderer/index.html index d2217e8..8b0df96 100644 --- a/renderer/index.html +++ b/renderer/index.html @@ -331,6 +331,7 @@ + diff --git a/tasks/lessons.md b/tasks/lessons.md index 7032d2f..3c75938 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -40,3 +40,10 @@ **Symptom:** "upload_result Seite hat keinen filecode ()" — nichtssagend; User dachte doodstream-Format geändert. **Root cause:** XFileSharing liefert den echten Grund im `st`-Feld (Error: duplicate / file too big / …). Code ignorierte `st` komplett und warf nur den leeren Body. **Regel:** Bei Hoster-Parsefehlern immer die Server-Statusfelder (st/msg/code) + Kontext (welcher CDN-Node, war filecode da) in die Fehlermeldung packen. Format-Struktur unverändert + leerer Inhalt = Backend-Ablehnung, kein Parsing-Bug. + +## 2026-05-25 — Queue leer nach Update: Auto-Dedup zu aggressiv (nicht Save/Restore) +**Symptom:** Queue gestoppt, App-Update -> nach Neustart Queue leer ("Dateien hierhin ziehen"). User dachte Save/Restore kaputt. +**Root cause:** Queue WIRD korrekt gespeichert (pendingQueue) + restored. ABER `_autoDeduplicateFromLog` (läuft bei init nach restore) entfernte Jobs per `fileName|hoster`-Match gegen das GESAMTE Lifetime-fileuploader.log — UNABHÄNGIG vom Status. Pending 'preview'-Jobs, deren Datei früher mal hochgeladen wurde, flogen alle raus -> komplette Queue weg. "Update-spezifisch" nur weil der Server-App nur beim Update neustartet (normaler Restart hätte dasselbe getan). +**Verifiziert:** Reale electron-config.json: 4 preview-Jobs, alle 4 Keys im Log -> alte Logik entfernt 4/4. Neue Logik (nur status==='done' droppen) entfernt 0/4. +**Regel:** Auto-Cleanup/Dedup darf NIE pending/actionable User-Arbeit löschen. Nur genuin abgeschlossene ('done') Jobs decluttern. Lifetime-Logs sind Historie, nicht Session-Fortschritt — nicht als "schon erledigt"-Quelle für pending Jobs missbrauchen. +**Wie anwenden:** Bei jeder Filter/Remove-Logik auf User-State: nach Status gaten, nicht nur nach Identitäts-Match gegen historische Daten. diff --git a/tests/queue-dedup.test.js b/tests/queue-dedup.test.js new file mode 100644 index 0000000..0830064 --- /dev/null +++ b/tests/queue-dedup.test.js @@ -0,0 +1,72 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { partitionRestoredJobsByLog } = require('../lib/queue-dedup'); + +function job(status, fileName, hoster) { + return { status, fileName, hoster, file: `C:/dl/${fileName}` }; +} + +test('regression: pending preview jobs are NEVER dropped, even when all match the log', () => { + // Exact shape of the reproduced bug: 4 preview jobs for one file across 4 + // hosters, every fileName|hoster present in the lifetime upload log. + const jobs = [ + job('preview', 'Einfach mal die Fresse halten!!!.mp4', 'doodstream.com'), + job('preview', 'Einfach mal die Fresse halten!!!.mp4', 'voe.sx'), + job('preview', 'Einfach mal die Fresse halten!!!.mp4', 'vidmoly.me'), + job('preview', 'Einfach mal die Fresse halten!!!.mp4', 'byse.sx') + ]; + const log = [ + { fileName: 'Einfach mal die Fresse halten!!!.mp4', hoster: 'doodstream.com' }, + { fileName: 'Einfach mal die Fresse halten!!!.mp4', hoster: 'voe.sx' }, + { fileName: 'Einfach mal die Fresse halten!!!.mp4', hoster: 'vidmoly.me' }, + { fileName: 'Einfach mal die Fresse halten!!!.mp4', hoster: 'byse.sx' } + ]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, log); + assert.equal(removed.length, 0, 'no pending job may be removed'); + assert.equal(kept.length, 4, 'all 4 pending jobs survive restart/update'); +}); + +test('done jobs in the log are dropped (declutter); pending/error/aborted kept', () => { + const jobs = [ + job('done', 'a.mkv', 'doodstream.com'), + job('preview', 'a.mkv', 'voe.sx'), + job('error', 'b.mkv', 'doodstream.com'), + job('aborted', 'c.mkv', 'doodstream.com') + ]; + const log = [ + { fileName: 'a.mkv', hoster: 'doodstream.com' }, + { fileName: 'a.mkv', hoster: 'voe.sx' }, + { fileName: 'b.mkv', hoster: 'doodstream.com' }, + { fileName: 'c.mkv', hoster: 'doodstream.com' } + ]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, log); + assert.equal(removed.length, 1); + assert.equal(removed[0].status, 'done'); + assert.equal(removed[0].hoster, 'doodstream.com'); + // The preview a.mkv|voe.sx, error b.mkv, aborted c.mkv all survive. + assert.equal(kept.length, 3); + assert.ok(kept.some(j => j.status === 'preview' && j.hoster === 'voe.sx')); + assert.ok(kept.some(j => j.status === 'error')); + assert.ok(kept.some(j => j.status === 'aborted')); +}); + +test('done job NOT in the log is kept (e.g. hoster had logToFile disabled)', () => { + const jobs = [job('done', 'd.mkv', 'doodstream.com')]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, []); + assert.equal(removed.length, 0); + assert.equal(kept.length, 1); +}); + +test('case-insensitive match on fileName and hoster', () => { + const jobs = [job('done', 'Movie.MKV', 'DoodStream.com')]; + const log = [{ fileName: 'movie.mkv', hoster: 'doodstream.com' }]; + const { removed } = partitionRestoredJobsByLog(jobs, log); + assert.equal(removed.length, 1); +}); + +test('empty/missing inputs do not throw', () => { + assert.deepEqual(partitionRestoredJobsByLog([], []), { kept: [], removed: [] }); + assert.deepEqual(partitionRestoredJobsByLog(null, null), { kept: [], removed: [] }); + const jobs = [job('done', 'x.mkv', 'voe.sx')]; + assert.equal(partitionRestoredJobsByLog(jobs, undefined).kept.length, 1); +});