From 6b0c515a9be72b96d4ed9af9cdf2ad888c83aaf9 Mon Sep 17 00:00:00 2001 From: Administrator Date: Fri, 19 Jun 2026 06:51:48 +0200 Subject: [PATCH] fix(queue): close two lost-work / sticky-ghost edge cases found by intensive testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up hardening on the v3.3.80 queue-persistence fix after a deeper adversarial sweep + reviewer pass over the WHOLE subsystem (not just the diff). Two real defects: 1. Basename-collision lost work (regression introduced by v3.3.80 FIX A). restoreQueueStateFromConfig collapses jobs on the FULL path while the ts-gate keys on basename|hoster. Two genuinely different files with the same basename queued to the same hoster from different folders therefore share a gate key: if one was logged after savedAt, the ts-rule dropped BOTH — silently losing the still-pending one. Pre-FIX-A only 'done' jobs were dropped, so a pending file was never at risk. Fix: an ambiguity guard in partitionRestoredJobsByLog — the ts-rule is suppressed when a basename|hoster key maps to more than one distinct file path (the log records only basenames, so it can't say which physical file completed). The done-in-log rule is unchanged. Fails safe: worst case a visible ghost survives, never silent data loss. 2. selectedFiles re-materialization with removeFromQueueOnDone=ON (second mechanism, independent of the stale snapshot). When that setting is on, a completed job is stripped from queueJobs but its path stays in selectedFiles (syncSelectedFilesFromQueue only runs at batch-done, never on a mid-upload close). On restart the ts-gate operates on queueJobs and never sees it, then the startup updateUploadView -> buildQueuePreview re-creates it as a preview ghost AFTER the gate ran, and it re-persists with a fresh savedAt — sticky. Fix: completedSelectionKeys() seeds _completedUploadKeys (the set buildQueuePreview already consults) from the log at startup, keyed on full path, with the same ambiguity guard. Log-based so it survives a hard kill, consistent with FIX A. Also extracts the orphan-tmp sweep decision into lib/orphan-tmp.js (was untested inline code in main.js; behavior-preserving) and adds executable coverage for the paths that were previously only argued from logic: orphan-tmp sweep, config-store pendingQueue+savedAt round-trip, an end-to-end scenario in the exact user-reported shape (300 queued / ~200 finished mid-session), and a 3000+500-iteration property fuzz of the gate invariant including the lost-work guarantee. 359/359 green, ESLint clean, smoke-boot unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/orphan-tmp.js | 29 ++++++ lib/queue-dedup.js | 33 ++++++- main.js | 21 ++--- renderer/app.js | 7 +- tasks/todo.md | 29 ++++++ tests/config-store.test.js | 36 +++++++ tests/orphan-tmp.test.js | 56 +++++++++++ tests/queue-dedup-property.test.js | 114 +++++++++++++++++++++++ tests/queue-dedup.test.js | 94 ++++++++++++++++++- tests/queue-persistence-scenario.test.js | 83 +++++++++++++++++ 10 files changed, 487 insertions(+), 15 deletions(-) create mode 100644 lib/orphan-tmp.js create mode 100644 tests/orphan-tmp.test.js create mode 100644 tests/queue-dedup-property.test.js create mode 100644 tests/queue-persistence-scenario.test.js diff --git a/lib/orphan-tmp.js b/lib/orphan-tmp.js new file mode 100644 index 0000000..eb66d51 --- /dev/null +++ b/lib/orphan-tmp.js @@ -0,0 +1,29 @@ +(function (root) { + 'use strict'; + + function selectOrphanTmps(fileNames, opts) { + const o = opts || {}; + const baseName = String(o.baseName || ''); + const currentPid = o.currentPid; + const isAlive = typeof o.isAlive === 'function' ? o.isAlive : () => false; + const out = []; + if (!baseName || !Array.isArray(fileNames)) return out; + const prefix = baseName + '.'; + const suffix = '.tmp'; + for (const file of fileNames) { + if (typeof file !== 'string') continue; + if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue; + const mid = file.slice(prefix.length, file.length - suffix.length); + if (!/^\d+$/.test(mid)) continue; + const pid = Number(mid); + if (pid === currentPid) continue; + if (isAlive(pid)) continue; + out.push(file); + } + return out; + } + + const api = { selectOrphanTmps }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.OrphanTmp = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/queue-dedup.js b/lib/queue-dedup.js index 3614c5d..4b97799 100644 --- a/lib/queue-dedup.js +++ b/lib/queue-dedup.js @@ -59,11 +59,22 @@ ? Math.floor(savedAt / 1000) * 1000 : null; + const filesPerKey = new Map(); + for (const job of jobs) { + if (job && job.fileName && job.hoster) { + const jk = _key(job.fileName, job.hoster); + let set = filesPerKey.get(jk); + if (!set) { set = new Set(); filesPerKey.set(jk, set); } + set.add(job.file || ''); + } + } + for (const job of jobs) { const hasIds = job && job.fileName && job.hoster; const k = hasIds ? _key(job.fileName, job.hoster) : null; const doneInLog = job && job.status === 'done' && hasIds && logKeys.has(k); - const uploadedAfterSnapshot = savedAtFloor !== null && k !== null + const keyUnambiguous = k !== null && filesPerKey.get(k).size <= 1; + const uploadedAfterSnapshot = savedAtFloor !== null && k !== null && keyUnambiguous && logMaxTs.has(k) && logMaxTs.get(k) >= savedAtFloor; if (doneInLog || uploadedAfterSnapshot) { removed.push(job); @@ -74,7 +85,25 @@ return { kept, removed }; } - const api = { partitionRestoredJobsByLog }; + function completedSelectionKeys(selectedFiles, hosters, logEntries, savedAt) { + const out = []; + if (!Array.isArray(selectedFiles) || !Array.isArray(hosters)) return out; + if (!(typeof savedAt === 'number' && isFinite(savedAt))) return out; + const synthetic = []; + for (const f of selectedFiles) { + if (!f || !f.path) continue; + const name = f.name || String(f.path).split(/[\\/]/).pop(); + for (const h of hosters) { + if (h) synthetic.push({ fileName: name, hoster: h, file: f.path, status: 'preview' }); + } + } + if (synthetic.length === 0) return out; + const { removed } = partitionRestoredJobsByLog(synthetic, logEntries, savedAt); + for (const job of removed) out.push(`${job.file}|${job.hoster}`); + return out; + } + + const api = { partitionRestoredJobsByLog, completedSelectionKeys }; if (typeof module !== 'undefined' && module.exports) { module.exports = api; diff --git a/main.js b/main.js index 81a42ce..8f23886 100644 --- a/main.js +++ b/main.js @@ -18,6 +18,7 @@ const RemoteServer = require('./lib/remote-server'); const { maybeRotateLogFile } = require('./lib/log-rotation'); const { hosterLogToFileEnabled } = require('./lib/log-policy'); const { formatUploadLogLine, parseUploadLogLine } = require('./lib/upload-log'); +const { selectOrphanTmps } = require('./lib/orphan-tmp'); const { sanitizeConfig, buildSupportBundleText } = require('./lib/support-bundle'); const { buildWebhookRequest, isAllAborted } = require('./lib/webhook-notify'); @@ -2263,17 +2264,15 @@ function _sleepSyncMs(ms) { function _sweepOrphanConfigTmps() { try { const dir = path.dirname(configStore.filePath); - const prefix = path.basename(configStore.filePath) + '.'; - const suffix = '.tmp'; - for (const file of fs.readdirSync(dir)) { - if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue; - const mid = file.slice(prefix.length, file.length - suffix.length); - if (!/^\d+$/.test(mid)) continue; - const pid = Number(mid); - if (pid === process.pid) continue; - let alive = false; - try { process.kill(pid, 0); alive = true; } catch (e) { alive = !!(e && e.code === 'EPERM'); } - if (alive) continue; + const isAlive = (pid) => { + try { process.kill(pid, 0); return true; } catch (e) { return !!(e && e.code === 'EPERM'); } + }; + const orphans = selectOrphanTmps(fs.readdirSync(dir), { + baseName: path.basename(configStore.filePath), + currentPid: process.pid, + isAlive + }); + for (const file of orphans) { try { fs.unlinkSync(path.join(dir, file)); } catch {} } } catch {} diff --git a/renderer/app.js b/renderer/app.js index d126891..d0b5572 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -4935,7 +4935,7 @@ function handleShutdownCountdown(data) { // --- Auto-deduplicate restored queue against own upload log on startup --- async function _autoDeduplicateFromLog() { - if (queueJobs.length === 0) return; + if (queueJobs.length === 0 && selectedFiles.length === 0) return; try { const entries = await window.api.readOwnUploadLog(); if (!entries || entries.length === 0) return; @@ -4954,6 +4954,11 @@ async function _autoDeduplicateFromLog() { syncSelectedFilesFromQueue(); window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`); } + const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt); + if (seedKeys.length > 0) { + for (const k of seedKeys) _completedUploadKeys.add(k); + window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`); + } } catch {} } diff --git a/tasks/todo.md b/tasks/todo.md index fa5a923..c0a3e62 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -56,3 +56,32 @@ fertig hochgeladene Dateien wieder in der Liste. Windows-Profil verschlüsselt, nicht entschlüsselbar). Symptom tritt nur auf bei Close WÄHREND aktivem Upload oder Hard-Kill — ein sauberer Idle-Close war schon vorher korrekt. + +## Runde 2 — "noch intensiver" (v3.3.81) +- **Echte ausführbare Tests** für bisher nur logisch abgedeckte Pfade: `tests/orphan-tmp.test.js` + (Sweep-Entscheidung, extrahiert nach `lib/orphan-tmp.js`), config-store `pendingQueue`+`savedAt` + Roundtrip, `tests/queue-persistence-scenario.test.js` (exakte 300/100-Bug-Form + multi-hoster), + `tests/queue-dedup-property.test.js` (3000+500 Fuzz-Iterationen gegen die formale Invariante). +- **Breite adversariale Bug-Jagd** übers GANZE Subsystem: lief tooling-bedingt teils kaputt + (bug-analyzer ohne File-Tools, Socket-Fehler) → 3 unverifizierte Hypothesen SELBST am Code + geprüft: + - #2 (gedroppte done-Jobs reappear via buildQueuePreview) → REFUTED: `_completedUploadKeys` + (full path) == buildQueuePreview-Key; nach Gate räumt `syncSelectedFilesFromQueue` selectedFiles. + - #3 (done-File bleibt in selectedFiles → re-preview) → **REAL (Advisor-Catch) & gefixt.** Meine + erste Abweisung war zu schnell: `syncSelectedFilesFromQueue` läuft NICHT beim Mid-Upload-Close. + Bei `removeFromQueueOnDone=ON` werden fertige Jobs aus queueJobs entfernt, bleiben aber in + selectedFiles; `updateUploadView`→`buildQueuePreview` (Startup, Zeile 990) re-materialisiert sie + als Preview-Ghost NACH dem Gate → sticky. Fix: `completedSelectionKeys` (queue-dedup.js) seedet + beim Start `_completedUploadKeys` (full-path, log-basiert/hard-kill-durabel, gleiche Ambiguity- + Guard) → buildQueuePreview überspringt fertige (file|hoster)-Paare. Nur relevant bei + removeFromQueueOnDone=ON (Default OFF). + - #1 (basename-Kollision droppt PENDING Datei = Lost Work) → **REAL & gefixt.** FIX A's ts-Regel + keyt auf basename, Restore-Collapse auf full path → zwei gleichnamige Dateien aus verschiedenen + Ordnern an denselben Hoster: die geloggte droppte fälschlich auch die andere PENDING. **Ambiguity- + Guard** in queue-dedup.js: ts-Regel wird unterdrückt, wenn ein basename|hoster-Key auf mehrere + DISTINKTE Pfade zeigt (done-Regel unberührt). Fail-safe: schlimmstenfalls überlebt ein + sichtbarer Ghost, NIE stiller Datenverlust. +- **Un-gehuntete Bereiche selbst abgeklopft:** DST/Clock-Skew → Fehler nur in SICHERER Richtung + (Ghost bleibt, kein Lost Work), inhärente Grenze von Sekunden-Lokalzeit-Logs. Log-Discovery + readdir-Filter `startsWith(base)&&endsWith(ext)` fängt single/daily/session — keine verpassten + Log-Files. 353/353 grün, ESLint clean, 3× Suite ohne Flake. diff --git a/tests/config-store.test.js b/tests/config-store.test.js index 794cb14..edb7c55 100644 --- a/tests/config-store.test.js +++ b/tests/config-store.test.js @@ -61,6 +61,42 @@ describe('ConfigStore', () => { assert.equal(config.globalSettings.logMode, 'single'); }); + it('persists pendingQueue incl. savedAt (number) and ts-bearing jobs across save/load', async () => { + // The queue-persistence fix stamps pendingQueue.savedAt and restores it on launch. + // This proves the persistence layer round-trips the new fields untouched (the + // ts-gate is worthless if savedAt does not survive serialization). + const pendingQueue = { + savedAt: 1750000000123, + selectedUploadHosters: ['voe.sx', 'byse.sx'], + selectedFiles: [{ path: 'C:/dl/a.mkv', name: 'a.mkv', size: 4242 }], + queueJobs: [ + { id: 'j1', file: 'C:/dl/a.mkv', fileName: 'a.mkv', hoster: 'voe.sx', status: 'preview', bytesTotal: 4242, maxAttempts: 0 }, + { id: 'j2', file: 'C:/dl/a.mkv', fileName: 'a.mkv', hoster: 'byse.sx', status: 'error', error: 'boom', maxAttempts: 3 } + ] + }; + const current = store.load(); + await store.save({ globalSettings: { ...current.globalSettings, pendingQueue } }); + const loaded = store.load(); + const pq = loaded.globalSettings.pendingQueue; + assert.equal(pq.savedAt, 1750000000123, 'savedAt epoch survives JSON round-trip'); + assert.equal(typeof pq.savedAt, 'number'); + assert.equal(pq.queueJobs.length, 2); + assert.equal(pq.queueJobs[0].fileName, 'a.mkv'); + assert.equal(pq.queueJobs[0].hoster, 'voe.sx'); + assert.equal(pq.queueJobs[1].status, 'error'); + assert.deepEqual(pq.selectedUploadHosters, ['voe.sx', 'byse.sx']); + assert.equal(pq.selectedFiles[0].path, 'C:/dl/a.mkv'); + }); + + it('pendingQueue can be cleared back to null (clearPersistedQueueStateSoon path)', async () => { + const current = store.load(); + await store.save({ globalSettings: { ...current.globalSettings, pendingQueue: { savedAt: 1, queueJobs: [] } } }); + assert.ok(store.load().globalSettings.pendingQueue); + const c2 = store.load(); + await store.save({ globalSettings: { ...c2.globalSettings, pendingQueue: null } }); + assert.equal(store.load().globalSettings.pendingQueue, null); + }); + it('regression: legacy sessionLog:true on disk normalizes to logMode "daily" (NOT "session")', async () => { // Write a config with the legacy boolean only (what an existing user has). await store.save({ globalSettings: { sessionLog: true } }); diff --git a/tests/orphan-tmp.test.js b/tests/orphan-tmp.test.js new file mode 100644 index 0000000..f639565 --- /dev/null +++ b/tests/orphan-tmp.test.js @@ -0,0 +1,56 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { selectOrphanTmps } = require('../lib/orphan-tmp'); + +const BASE = 'electron-config.json'; +const aliveSet = new Set([100, 200]); +const isAlive = (pid) => aliveSet.has(pid); + +test('selects only dead-pid ..tmp orphans', () => { + const files = [ + 'electron-config.json', + 'electron-config.json.bak', + 'electron-config.json.tmp', + 'electron-config.json.100.tmp', + 'electron-config.json.200.tmp', + 'electron-config.json.999.tmp', + 'electron-config.json.4242.tmp', + 'something-else.500.tmp', + 'electron-config.json.abc.tmp' + ]; + const orphans = selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive }); + assert.deepEqual(orphans.sort(), ['electron-config.json.4242.tmp', 'electron-config.json.999.tmp']); +}); + +test('never selects the current process tmp', () => { + const files = ['electron-config.json.7.tmp']; + assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []); +}); + +test('never selects the FIXED .tmp (used by async _atomicWrite)', () => { + const files = ['electron-config.json.tmp']; + assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []); +}); + +test('never selects the live config or its .bak', () => { + const files = ['electron-config.json', 'electron-config.json.bak']; + assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []); +}); + +test('alive pid (incl. EPERM-as-alive) is skipped, preventing deletion of a concurrent instance tmp', () => { + const files = ['electron-config.json.100.tmp', 'electron-config.json.300.tmp']; + const orphans = selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: (p) => p === 100 }); + assert.deepEqual(orphans, ['electron-config.json.300.tmp']); +}); + +test('robust to junk / missing inputs', () => { + assert.deepEqual(selectOrphanTmps(null, { baseName: BASE, currentPid: 1, isAlive }), []); + assert.deepEqual(selectOrphanTmps(['x', 42, null, undefined], { baseName: BASE, currentPid: 1, isAlive }), []); + assert.deepEqual(selectOrphanTmps(['electron-config.json.5.tmp'], {}), []); + assert.deepEqual(selectOrphanTmps(['electron-config.json.5.tmp'], { baseName: '', currentPid: 1, isAlive }), []); +}); + +test('does not match a different base that shares a prefix', () => { + const files = ['electron-config.json.backup.5.tmp']; + assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []); +}); diff --git a/tests/queue-dedup-property.test.js b/tests/queue-dedup-property.test.js new file mode 100644 index 0000000..9d4c0a7 --- /dev/null +++ b/tests/queue-dedup-property.test.js @@ -0,0 +1,114 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { partitionRestoredJobsByLog } = require('../lib/queue-dedup'); + +function lcg(seed) { + let s = seed >>> 0; + return () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 4294967296; }; +} + +function key(f, h) { return `${String(f).toLowerCase()}|${String(h).toLowerCase()}`; } + +test('property: removed iff (done && key in log) OR (savedAt finite && key unambiguous && newest matching log ts >= floor(savedAt/1000)*1000)', () => { + const rnd = lcg(0x9e3779b1); + const statuses = ['preview', 'done', 'error', 'aborted', 'queued', 'skipped']; + const hosters = ['voe.sx', 'byse.sx', 'doodstream.com']; + const names = ['a.mkv', 'b.mp4', 'A.MKV', 'c.mov']; + const folders = ['C:/A/', 'C:/B/', 'D:/down/']; + const pick = (arr) => arr[Math.floor(rnd() * arr.length)]; + + for (let iter = 0; iter < 3000; iter++) { + const useSavedAt = rnd() < 0.7; + const savedAt = useSavedAt ? Math.floor(rnd() * 2_000_000_000_000) : undefined; + + const jobs = []; + const nJobs = 1 + Math.floor(rnd() * 6); + for (let i = 0; i < nJobs; i++) { + const name = pick(names); + // Mix shared and distinct paths so ambiguous keys (same name+hoster, + // different folder) actually occur and exercise the guard. + jobs.push({ id: `j${i}`, fileName: name, hoster: pick(hosters), status: pick(statuses), file: `${pick(folders)}${name}` }); + } + + const log = []; + const nLog = Math.floor(rnd() * 5); + for (let i = 0; i < nLog; i++) { + const hasTs = rnd() < 0.8; + log.push({ fileName: pick(names), hoster: pick(hosters), ts: hasTs ? Math.floor(rnd() * 2_000_000_000_000) : undefined }); + } + + const logKeys = new Set(); + const maxTs = new Map(); + for (const e of log) { + const k = key(e.fileName, e.hoster); + logKeys.add(k); + if (typeof e.ts === 'number' && isFinite(e.ts)) { + const prev = maxTs.get(k); + if (prev === undefined || e.ts > prev) maxTs.set(k, e.ts); + } + } + const filesPerKey = new Map(); + for (const job of jobs) { + const k = key(job.fileName, job.hoster); + if (!filesPerKey.has(k)) filesPerKey.set(k, new Set()); + filesPerKey.get(k).add(job.file || ''); + } + const floor = (typeof savedAt === 'number' && isFinite(savedAt)) ? Math.floor(savedAt / 1000) * 1000 : null; + + const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt); + + assert.equal(kept.length + removed.length, jobs.length, `iter ${iter}: partition must cover every job exactly once`); + const keptIds = new Set(kept.map(j => j.id)); + const removedIds = new Set(removed.map(j => j.id)); + assert.equal(keptIds.size + removedIds.size, jobs.length, `iter ${iter}: no job in both partitions`); + + for (const job of jobs) { + const k = key(job.fileName, job.hoster); + const doneInLog = job.status === 'done' && logKeys.has(k); + const unambiguous = filesPerKey.get(k).size <= 1; + const afterSnap = floor !== null && unambiguous && maxTs.has(k) && maxTs.get(k) >= floor; + const shouldRemove = doneInLog || afterSnap; + assert.equal(removedIds.has(job.id), shouldRemove, + `iter ${iter}: job ${job.id} (status=${job.status} key=${k} unambig=${unambiguous}) expected removed=${shouldRemove}`); + } + } +}); + +test('property: a genuinely-pending job is NEVER lost to a same-basename sibling completing after the snapshot', () => { + const rnd = lcg(0x1234abcd); + for (let iter = 0; iter < 500; iter++) { + const savedAt = 1_000_000_000_000 + Math.floor(rnd() * 1_000_000); + // X completed after the snapshot (logged); Y is a DIFFERENT file, same + // basename + hoster, genuinely pending. Y must survive. + const jobs = [ + { id: 'X', fileName: 'clip.mp4', hoster: 'voe.sx', status: 'preview', file: 'C:/A/clip.mp4' }, + { id: 'Y', fileName: 'clip.mp4', hoster: 'voe.sx', status: 'preview', file: 'C:/B/clip.mp4' } + ]; + const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: savedAt + 1000 + Math.floor(rnd() * 1000) }]; + const { kept } = partitionRestoredJobsByLog(jobs, log, savedAt); + assert.ok(kept.some(j => j.id === 'Y'), `iter ${iter}: pending Y must never be silently dropped`); + } +}); + +test('property: 2-arg legacy call NEVER removes a non-done job (the v3.3.80 canary, fuzzed)', () => { + const rnd = lcg(0xdeadbeef); + const statuses = ['preview', 'error', 'aborted', 'queued', 'skipped']; + const hosters = ['voe.sx', 'byse.sx']; + const names = ['a.mkv', 'b.mp4']; + const pick = (arr) => arr[Math.floor(rnd() * arr.length)]; + + for (let iter = 0; iter < 1000; iter++) { + const jobs = []; + const nJobs = 1 + Math.floor(rnd() * 5); + for (let i = 0; i < nJobs; i++) { + jobs.push({ id: `j${i}`, fileName: pick(names), hoster: pick(hosters), status: pick(statuses), file: `C:/x/${i}` }); + } + const log = []; + const nLog = Math.floor(rnd() * 4); + for (let i = 0; i < nLog; i++) { + log.push({ fileName: pick(names), hoster: pick(hosters), ts: Math.floor(rnd() * 2_000_000_000_000) }); + } + const { removed } = partitionRestoredJobsByLog(jobs, log); + assert.ok(removed.every(j => j.status === 'done'), `iter ${iter}: legacy 2-arg call must never drop a non-done job`); + } +}); diff --git a/tests/queue-dedup.test.js b/tests/queue-dedup.test.js index 4170b0a..c567d5b 100644 --- a/tests/queue-dedup.test.js +++ b/tests/queue-dedup.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { partitionRestoredJobsByLog } = require('../lib/queue-dedup'); +const { partitionRestoredJobsByLog, completedSelectionKeys } = require('../lib/queue-dedup'); function job(status, fileName, hoster) { return { status, fileName, hoster, file: `C:/dl/${fileName}` }; @@ -135,6 +135,45 @@ test('ts-gate: done job uploaded after snapshot is dropped via either rule', () assert.equal(removed.length, 1); }); +test('ts-gate ambiguity guard: a pending same-basename file in a DIFFERENT folder is NOT lost when a sibling completes after the snapshot', () => { + // X (C:/A/clip.mp4) was uploaded after the snapshot and logged. Y is a + // genuinely-different file (C:/B/clip.mp4), same basename + hoster, still + // pending. The log records only basenames, so the ts-rule must not drop Y. + const jobs = [ + { status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' }, + { status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/B/clip.mp4' } + ]; + const savedAt = T('2026-06-19 12:00:00'); + const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt); + assert.equal(removed.length, 0, 'ambiguous key -> ts-rule suppressed, no pending file lost'); + assert.equal(kept.length, 2); +}); + +test('ts-gate ambiguity guard: the done-in-log rule still applies on an ambiguous key', () => { + // Even when the key is ambiguous, a job that is actually 'done' and in the log + // is still decluttered (pre-existing rule, unchanged by the guard). + const jobs = [ + { status: 'done', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' }, + { status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/B/clip.mp4' } + ]; + const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + const savedAt = T('2026-06-19 12:00:00'); + const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt); + assert.equal(removed.length, 1); + assert.equal(removed[0].status, 'done'); + assert.equal(removed[0].file, 'C:/A/clip.mp4'); + assert.ok(kept.some(j => j.file === 'C:/B/clip.mp4'), 'the distinct pending file survives'); +}); + +test('ts-gate: a unique-path ghost still drops (guard does not weaken the common case)', () => { + const jobs = [{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' }]; + const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + const savedAt = T('2026-06-19 12:00:00'); + const { removed } = partitionRestoredJobsByLog(jobs, log, savedAt); + assert.equal(removed.length, 1, 'single job for the key -> unambiguous -> ghost dropped as before'); +}); + test('ts-gate: multi-hoster partial completion — the reported bug shape (drop only the completed hosters)', () => { // One file queued to 4 hosters; close mid-upload. After the snapshot, 2 hosters // completed (logged), 2 never started. On restart all 4 restore as 'preview'. @@ -159,3 +198,56 @@ test('ts-gate: multi-hoster partial completion — the reported bug shape (drop assert.ok(kept.some(j => j.hoster === 'vidmoly.me')); assert.ok(kept.some(j => j.hoster === 'byse.sx')); }); + +test('completedSelectionKeys: a selectedFile that completed after the snapshot yields its full-path|hoster key', () => { + const selectedFiles = [{ path: 'C:/dl/done.mp4', name: 'done.mp4' }, { path: 'C:/dl/pending.mp4', name: 'pending.mp4' }]; + const hosters = ['voe.sx']; + const savedAt = T('2026-06-19 12:00:00'); + const log = [{ fileName: 'done.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt); + assert.deepEqual(keys, ['C:/dl/done.mp4|voe.sx'], 'only the completed file is seeded; pending is not'); +}); + +test('completedSelectionKeys: per-hoster — a file done on voe but not byse only seeds the voe key', () => { + const selectedFiles = [{ path: 'C:/dl/a.mp4', name: 'a.mp4' }]; + const hosters = ['voe.sx', 'byse.sx']; + const savedAt = T('2026-06-19 12:00:00'); + const log = [{ fileName: 'a.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt); + assert.deepEqual(keys, ['C:/dl/a.mp4|voe.sx'], 'the still-pending byse upload is NOT seeded'); +}); + +test('completedSelectionKeys: ambiguous basename across folders seeds NOTHING (no lost re-preview)', () => { + const selectedFiles = [{ path: 'C:/A/clip.mp4', name: 'clip.mp4' }, { path: 'C:/B/clip.mp4', name: 'clip.mp4' }]; + const hosters = ['voe.sx']; + const savedAt = T('2026-06-19 12:00:00'); + const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt); + assert.deepEqual(keys, [], 'ambiguous -> neither path is suppressed, both re-preview (safe direction)'); +}); + +test('completedSelectionKeys: an OLDER completion (pre-snapshot re-queue) is NOT seeded', () => { + const selectedFiles = [{ path: 'C:/dl/reup.mp4', name: 'reup.mp4' }]; + const hosters = ['voe.sx']; + const savedAt = T('2026-06-19 12:00:00'); + const log = [{ fileName: 'reup.mp4', hoster: 'voe.sx', ts: T('2026-06-19 11:00:00') }]; + assert.deepEqual(completedSelectionKeys(selectedFiles, hosters, log, savedAt), []); +}); + +test('completedSelectionKeys: no savedAt / junk inputs -> empty (legacy + robustness)', () => { + const sf = [{ path: 'C:/dl/a.mp4', name: 'a.mp4' }]; + const log = [{ fileName: 'a.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + assert.deepEqual(completedSelectionKeys(sf, ['voe.sx'], log, undefined), []); + assert.deepEqual(completedSelectionKeys(null, ['voe.sx'], log, 1), []); + assert.deepEqual(completedSelectionKeys(sf, null, log, 1), []); + assert.deepEqual(completedSelectionKeys([], [], log, 1), []); + assert.deepEqual(completedSelectionKeys([{ name: 'x' }], ['voe.sx'], log, 1), [], 'entry without path is skipped'); +}); + +test('completedSelectionKeys: derives basename from path when name is missing', () => { + const selectedFiles = [{ path: 'C:/dl/sub/movie.mp4' }]; + const hosters = ['voe.sx']; + const savedAt = T('2026-06-19 12:00:00'); + const log = [{ fileName: 'movie.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }]; + assert.deepEqual(completedSelectionKeys(selectedFiles, hosters, log, savedAt), ['C:/dl/sub/movie.mp4|voe.sx']); +}); diff --git a/tests/queue-persistence-scenario.test.js b/tests/queue-persistence-scenario.test.js new file mode 100644 index 0000000..de02425 --- /dev/null +++ b/tests/queue-persistence-scenario.test.js @@ -0,0 +1,83 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { formatUploadLogLine, parseUploadLogLine } = require('../lib/upload-log'); +const { partitionRestoredJobsByLog } = require('../lib/queue-dedup'); + +function makeJobs(n, hoster, status, offset = 0) { + const jobs = []; + for (let i = 0; i < n; i++) { + const fileName = `clip_${String(i + offset).padStart(4, '0')}.mp4`; + jobs.push({ id: `j-${hoster}-${i + offset}`, file: `D:/inbox/${fileName}`, fileName, hoster, status }); + } + return jobs; +} + +test('user report: 300 queued, ~200 finished mid-session before a hard kill — only the finished drop', () => { + const hoster = 'byse.sx'; + const snapshot = new Date(2026, 5, 19, 22, 0, 0); + const savedAt = snapshot.getTime(); + const restoredJobs = makeJobs(300, hoster, 'preview'); + + const completionBase = new Date(2026, 5, 19, 22, 5, 0).getTime(); + const logEntries = []; + for (let i = 0; i < 200; i++) { + const d = new Date(completionBase + i * 1000); + logEntries.push(parseUploadLogLine( + formatUploadLogLine(d, hoster, `https://byse.sx/d/x${i}`, `clip_${String(i).padStart(4, '0')}.mp4`) + )); + } + + const { kept, removed } = partitionRestoredJobsByLog(restoredJobs, logEntries, savedAt); + assert.equal(removed.length, 200, 'the 200 completed-after-snapshot files are dropped as ghosts'); + assert.equal(kept.length, 100, 'the 100 never-finished files stay queued'); + assert.ok(kept.every(j => Number(j.fileName.slice(5, 9)) >= 200), 'kept are exactly indices 200..299'); + const keptNames = new Set(kept.map(j => j.fileName)); + assert.ok(removed.every(j => !keptNames.has(j.fileName))); +}); + +test('multi-hoster batch: per-hoster completion is independent (a file done on voe but not byse keeps byse)', () => { + const savedAt = new Date(2026, 5, 19, 22, 0, 0).getTime(); + const done = new Date(2026, 5, 19, 22, 3, 0); + const jobs = [ + ...makeJobs(3, 'voe.sx', 'preview'), + ...makeJobs(3, 'byse.sx', 'preview') + ]; + const logEntries = [ + parseUploadLogLine(formatUploadLogLine(done, 'voe.sx', 'l', 'clip_0000.mp4')), + parseUploadLogLine(formatUploadLogLine(done, 'voe.sx', 'l', 'clip_0001.mp4')), + parseUploadLogLine(formatUploadLogLine(done, 'byse.sx', 'l', 'clip_0000.mp4')) + ]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries, savedAt); + assert.equal(removed.length, 3); + assert.ok(removed.some(j => j.hoster === 'voe.sx' && j.fileName === 'clip_0000.mp4')); + assert.ok(removed.some(j => j.hoster === 'voe.sx' && j.fileName === 'clip_0001.mp4')); + assert.ok(removed.some(j => j.hoster === 'byse.sx' && j.fileName === 'clip_0000.mp4')); + assert.ok(kept.some(j => j.hoster === 'byse.sx' && j.fileName === 'clip_0001.mp4'), 'byse clip_0001 not logged -> kept'); +}); + +test('clean idle close (snapshot AFTER completion) keeps an intentional re-queue of an old file', () => { + const hoster = 'voe.sx'; + const yesterday = new Date(2026, 5, 18, 12, 0, 0); + const logEntries = [parseUploadLogLine(formatUploadLogLine(yesterday, hoster, 'link', 'reupload_me.mp4'))]; + const savedAt = new Date(2026, 5, 19, 9, 0, 0).getTime(); + const jobs = [{ id: 'r1', file: 'D:/x/reupload_me.mp4', fileName: 'reupload_me.mp4', hoster, status: 'preview' }]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries, savedAt); + assert.equal(removed.length, 0, 'an upload older than the snapshot is a deliberate re-queue and survives'); + assert.equal(kept.length, 1); +}); + +test('legacy snapshot without savedAt (pre-v3.3.80 config) falls back to done-only dedup', () => { + const hoster = 'voe.sx'; + const logEntries = [ + parseUploadLogLine(formatUploadLogLine(new Date(2026, 5, 19, 12, 0, 0), hoster, 'l', 'done.mp4')), + parseUploadLogLine(formatUploadLogLine(new Date(2026, 5, 19, 12, 1, 0), hoster, 'l', 'preview.mp4')) + ]; + const jobs = [ + { id: 'a', file: 'D:/x/done.mp4', fileName: 'done.mp4', hoster, status: 'done' }, + { id: 'b', file: 'D:/x/preview.mp4', fileName: 'preview.mp4', hoster, status: 'preview' } + ]; + const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries); + assert.equal(removed.length, 1, 'only the done job is decluttered when no savedAt is available'); + assert.equal(removed[0].id, 'a'); + assert.ok(kept.some(j => j.id === 'b'), 'the preview survives the legacy path'); +});