fix(queue): close two lost-work / sticky-ghost edge cases found by intensive testing
Follow-up hardening on the v3.3.80 queue-persistence fix after a deeper adversarial sweep + reviewer pass over the WHOLE subsystem (not just the diff). Two real defects: 1. Basename-collision lost work (regression introduced by v3.3.80 FIX A). restoreQueueStateFromConfig collapses jobs on the FULL path while the ts-gate keys on basename|hoster. Two genuinely different files with the same basename queued to the same hoster from different folders therefore share a gate key: if one was logged after savedAt, the ts-rule dropped BOTH — silently losing the still-pending one. Pre-FIX-A only 'done' jobs were dropped, so a pending file was never at risk. Fix: an ambiguity guard in partitionRestoredJobsByLog — the ts-rule is suppressed when a basename|hoster key maps to more than one distinct file path (the log records only basenames, so it can't say which physical file completed). The done-in-log rule is unchanged. Fails safe: worst case a visible ghost survives, never silent data loss. 2. selectedFiles re-materialization with removeFromQueueOnDone=ON (second mechanism, independent of the stale snapshot). When that setting is on, a completed job is stripped from queueJobs but its path stays in selectedFiles (syncSelectedFilesFromQueue only runs at batch-done, never on a mid-upload close). On restart the ts-gate operates on queueJobs and never sees it, then the startup updateUploadView -> buildQueuePreview re-creates it as a preview ghost AFTER the gate ran, and it re-persists with a fresh savedAt — sticky. Fix: completedSelectionKeys() seeds _completedUploadKeys (the set buildQueuePreview already consults) from the log at startup, keyed on full path, with the same ambiguity guard. Log-based so it survives a hard kill, consistent with FIX A. Also extracts the orphan-tmp sweep decision into lib/orphan-tmp.js (was untested inline code in main.js; behavior-preserving) and adds executable coverage for the paths that were previously only argued from logic: orphan-tmp sweep, config-store pendingQueue+savedAt round-trip, an end-to-end scenario in the exact user-reported shape (300 queued / ~200 finished mid-session), and a 3000+500-iteration property fuzz of the gate invariant including the lost-work guarantee. 359/359 green, ESLint clean, smoke-boot unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f7c8d308fc
commit
6b0c515a9b
29
lib/orphan-tmp.js
Normal file
29
lib/orphan-tmp.js
Normal file
@ -0,0 +1,29 @@
|
||||
(function (root) {
|
||||
'use strict';
|
||||
|
||||
function selectOrphanTmps(fileNames, opts) {
|
||||
const o = opts || {};
|
||||
const baseName = String(o.baseName || '');
|
||||
const currentPid = o.currentPid;
|
||||
const isAlive = typeof o.isAlive === 'function' ? o.isAlive : () => false;
|
||||
const out = [];
|
||||
if (!baseName || !Array.isArray(fileNames)) return out;
|
||||
const prefix = baseName + '.';
|
||||
const suffix = '.tmp';
|
||||
for (const file of fileNames) {
|
||||
if (typeof file !== 'string') continue;
|
||||
if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue;
|
||||
const mid = file.slice(prefix.length, file.length - suffix.length);
|
||||
if (!/^\d+$/.test(mid)) continue;
|
||||
const pid = Number(mid);
|
||||
if (pid === currentPid) continue;
|
||||
if (isAlive(pid)) continue;
|
||||
out.push(file);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const api = { selectOrphanTmps };
|
||||
if (typeof module !== 'undefined' && module.exports) module.exports = api;
|
||||
else if (root) root.OrphanTmp = api;
|
||||
})(typeof window !== 'undefined' ? window : this);
|
||||
@ -59,11 +59,22 @@
|
||||
? Math.floor(savedAt / 1000) * 1000
|
||||
: null;
|
||||
|
||||
const filesPerKey = new Map();
|
||||
for (const job of jobs) {
|
||||
if (job && job.fileName && job.hoster) {
|
||||
const jk = _key(job.fileName, job.hoster);
|
||||
let set = filesPerKey.get(jk);
|
||||
if (!set) { set = new Set(); filesPerKey.set(jk, set); }
|
||||
set.add(job.file || '');
|
||||
}
|
||||
}
|
||||
|
||||
for (const job of jobs) {
|
||||
const hasIds = job && job.fileName && job.hoster;
|
||||
const k = hasIds ? _key(job.fileName, job.hoster) : null;
|
||||
const doneInLog = job && job.status === 'done' && hasIds && logKeys.has(k);
|
||||
const uploadedAfterSnapshot = savedAtFloor !== null && k !== null
|
||||
const keyUnambiguous = k !== null && filesPerKey.get(k).size <= 1;
|
||||
const uploadedAfterSnapshot = savedAtFloor !== null && k !== null && keyUnambiguous
|
||||
&& logMaxTs.has(k) && logMaxTs.get(k) >= savedAtFloor;
|
||||
if (doneInLog || uploadedAfterSnapshot) {
|
||||
removed.push(job);
|
||||
@ -74,7 +85,25 @@
|
||||
return { kept, removed };
|
||||
}
|
||||
|
||||
const api = { partitionRestoredJobsByLog };
|
||||
function completedSelectionKeys(selectedFiles, hosters, logEntries, savedAt) {
|
||||
const out = [];
|
||||
if (!Array.isArray(selectedFiles) || !Array.isArray(hosters)) return out;
|
||||
if (!(typeof savedAt === 'number' && isFinite(savedAt))) return out;
|
||||
const synthetic = [];
|
||||
for (const f of selectedFiles) {
|
||||
if (!f || !f.path) continue;
|
||||
const name = f.name || String(f.path).split(/[\\/]/).pop();
|
||||
for (const h of hosters) {
|
||||
if (h) synthetic.push({ fileName: name, hoster: h, file: f.path, status: 'preview' });
|
||||
}
|
||||
}
|
||||
if (synthetic.length === 0) return out;
|
||||
const { removed } = partitionRestoredJobsByLog(synthetic, logEntries, savedAt);
|
||||
for (const job of removed) out.push(`${job.file}|${job.hoster}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
const api = { partitionRestoredJobsByLog, completedSelectionKeys };
|
||||
|
||||
if (typeof module !== 'undefined' && module.exports) {
|
||||
module.exports = api;
|
||||
|
||||
21
main.js
21
main.js
@ -18,6 +18,7 @@ const RemoteServer = require('./lib/remote-server');
|
||||
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
||||
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
||||
const { formatUploadLogLine, parseUploadLogLine } = require('./lib/upload-log');
|
||||
const { selectOrphanTmps } = require('./lib/orphan-tmp');
|
||||
const { sanitizeConfig, buildSupportBundleText } = require('./lib/support-bundle');
|
||||
const { buildWebhookRequest, isAllAborted } = require('./lib/webhook-notify');
|
||||
|
||||
@ -2263,17 +2264,15 @@ function _sleepSyncMs(ms) {
|
||||
function _sweepOrphanConfigTmps() {
|
||||
try {
|
||||
const dir = path.dirname(configStore.filePath);
|
||||
const prefix = path.basename(configStore.filePath) + '.';
|
||||
const suffix = '.tmp';
|
||||
for (const file of fs.readdirSync(dir)) {
|
||||
if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue;
|
||||
const mid = file.slice(prefix.length, file.length - suffix.length);
|
||||
if (!/^\d+$/.test(mid)) continue;
|
||||
const pid = Number(mid);
|
||||
if (pid === process.pid) continue;
|
||||
let alive = false;
|
||||
try { process.kill(pid, 0); alive = true; } catch (e) { alive = !!(e && e.code === 'EPERM'); }
|
||||
if (alive) continue;
|
||||
const isAlive = (pid) => {
|
||||
try { process.kill(pid, 0); return true; } catch (e) { return !!(e && e.code === 'EPERM'); }
|
||||
};
|
||||
const orphans = selectOrphanTmps(fs.readdirSync(dir), {
|
||||
baseName: path.basename(configStore.filePath),
|
||||
currentPid: process.pid,
|
||||
isAlive
|
||||
});
|
||||
for (const file of orphans) {
|
||||
try { fs.unlinkSync(path.join(dir, file)); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
@ -4935,7 +4935,7 @@ function handleShutdownCountdown(data) {
|
||||
|
||||
// --- Auto-deduplicate restored queue against own upload log on startup ---
|
||||
async function _autoDeduplicateFromLog() {
|
||||
if (queueJobs.length === 0) return;
|
||||
if (queueJobs.length === 0 && selectedFiles.length === 0) return;
|
||||
try {
|
||||
const entries = await window.api.readOwnUploadLog();
|
||||
if (!entries || entries.length === 0) return;
|
||||
@ -4954,6 +4954,11 @@ async function _autoDeduplicateFromLog() {
|
||||
syncSelectedFilesFromQueue();
|
||||
window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`);
|
||||
}
|
||||
const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt);
|
||||
if (seedKeys.length > 0) {
|
||||
for (const k of seedKeys) _completedUploadKeys.add(k);
|
||||
window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
|
||||
@ -56,3 +56,32 @@ fertig hochgeladene Dateien wieder in der Liste.
|
||||
Windows-Profil verschlüsselt, nicht entschlüsselbar). Symptom tritt nur auf bei
|
||||
Close WÄHREND aktivem Upload oder Hard-Kill — ein sauberer Idle-Close war schon
|
||||
vorher korrekt.
|
||||
|
||||
## Runde 2 — "noch intensiver" (v3.3.81)
|
||||
- **Echte ausführbare Tests** für bisher nur logisch abgedeckte Pfade: `tests/orphan-tmp.test.js`
|
||||
(Sweep-Entscheidung, extrahiert nach `lib/orphan-tmp.js`), config-store `pendingQueue`+`savedAt`
|
||||
Roundtrip, `tests/queue-persistence-scenario.test.js` (exakte 300/100-Bug-Form + multi-hoster),
|
||||
`tests/queue-dedup-property.test.js` (3000+500 Fuzz-Iterationen gegen die formale Invariante).
|
||||
- **Breite adversariale Bug-Jagd** übers GANZE Subsystem: lief tooling-bedingt teils kaputt
|
||||
(bug-analyzer ohne File-Tools, Socket-Fehler) → 3 unverifizierte Hypothesen SELBST am Code
|
||||
geprüft:
|
||||
- #2 (gedroppte done-Jobs reappear via buildQueuePreview) → REFUTED: `_completedUploadKeys`
|
||||
(full path) == buildQueuePreview-Key; nach Gate räumt `syncSelectedFilesFromQueue` selectedFiles.
|
||||
- #3 (done-File bleibt in selectedFiles → re-preview) → **REAL (Advisor-Catch) & gefixt.** Meine
|
||||
erste Abweisung war zu schnell: `syncSelectedFilesFromQueue` läuft NICHT beim Mid-Upload-Close.
|
||||
Bei `removeFromQueueOnDone=ON` werden fertige Jobs aus queueJobs entfernt, bleiben aber in
|
||||
selectedFiles; `updateUploadView`→`buildQueuePreview` (Startup, Zeile 990) re-materialisiert sie
|
||||
als Preview-Ghost NACH dem Gate → sticky. Fix: `completedSelectionKeys` (queue-dedup.js) seedet
|
||||
beim Start `_completedUploadKeys` (full-path, log-basiert/hard-kill-durabel, gleiche Ambiguity-
|
||||
Guard) → buildQueuePreview überspringt fertige (file|hoster)-Paare. Nur relevant bei
|
||||
removeFromQueueOnDone=ON (Default OFF).
|
||||
- #1 (basename-Kollision droppt PENDING Datei = Lost Work) → **REAL & gefixt.** FIX A's ts-Regel
|
||||
keyt auf basename, Restore-Collapse auf full path → zwei gleichnamige Dateien aus verschiedenen
|
||||
Ordnern an denselben Hoster: die geloggte droppte fälschlich auch die andere PENDING. **Ambiguity-
|
||||
Guard** in queue-dedup.js: ts-Regel wird unterdrückt, wenn ein basename|hoster-Key auf mehrere
|
||||
DISTINKTE Pfade zeigt (done-Regel unberührt). Fail-safe: schlimmstenfalls überlebt ein
|
||||
sichtbarer Ghost, NIE stiller Datenverlust.
|
||||
- **Un-gehuntete Bereiche selbst abgeklopft:** DST/Clock-Skew → Fehler nur in SICHERER Richtung
|
||||
(Ghost bleibt, kein Lost Work), inhärente Grenze von Sekunden-Lokalzeit-Logs. Log-Discovery
|
||||
readdir-Filter `startsWith(base)&&endsWith(ext)` fängt single/daily/session — keine verpassten
|
||||
Log-Files. 353/353 grün, ESLint clean, 3× Suite ohne Flake.
|
||||
|
||||
@ -61,6 +61,42 @@ describe('ConfigStore', () => {
|
||||
assert.equal(config.globalSettings.logMode, 'single');
|
||||
});
|
||||
|
||||
it('persists pendingQueue incl. savedAt (number) and ts-bearing jobs across save/load', async () => {
|
||||
// The queue-persistence fix stamps pendingQueue.savedAt and restores it on launch.
|
||||
// This proves the persistence layer round-trips the new fields untouched (the
|
||||
// ts-gate is worthless if savedAt does not survive serialization).
|
||||
const pendingQueue = {
|
||||
savedAt: 1750000000123,
|
||||
selectedUploadHosters: ['voe.sx', 'byse.sx'],
|
||||
selectedFiles: [{ path: 'C:/dl/a.mkv', name: 'a.mkv', size: 4242 }],
|
||||
queueJobs: [
|
||||
{ id: 'j1', file: 'C:/dl/a.mkv', fileName: 'a.mkv', hoster: 'voe.sx', status: 'preview', bytesTotal: 4242, maxAttempts: 0 },
|
||||
{ id: 'j2', file: 'C:/dl/a.mkv', fileName: 'a.mkv', hoster: 'byse.sx', status: 'error', error: 'boom', maxAttempts: 3 }
|
||||
]
|
||||
};
|
||||
const current = store.load();
|
||||
await store.save({ globalSettings: { ...current.globalSettings, pendingQueue } });
|
||||
const loaded = store.load();
|
||||
const pq = loaded.globalSettings.pendingQueue;
|
||||
assert.equal(pq.savedAt, 1750000000123, 'savedAt epoch survives JSON round-trip');
|
||||
assert.equal(typeof pq.savedAt, 'number');
|
||||
assert.equal(pq.queueJobs.length, 2);
|
||||
assert.equal(pq.queueJobs[0].fileName, 'a.mkv');
|
||||
assert.equal(pq.queueJobs[0].hoster, 'voe.sx');
|
||||
assert.equal(pq.queueJobs[1].status, 'error');
|
||||
assert.deepEqual(pq.selectedUploadHosters, ['voe.sx', 'byse.sx']);
|
||||
assert.equal(pq.selectedFiles[0].path, 'C:/dl/a.mkv');
|
||||
});
|
||||
|
||||
it('pendingQueue can be cleared back to null (clearPersistedQueueStateSoon path)', async () => {
|
||||
const current = store.load();
|
||||
await store.save({ globalSettings: { ...current.globalSettings, pendingQueue: { savedAt: 1, queueJobs: [] } } });
|
||||
assert.ok(store.load().globalSettings.pendingQueue);
|
||||
const c2 = store.load();
|
||||
await store.save({ globalSettings: { ...c2.globalSettings, pendingQueue: null } });
|
||||
assert.equal(store.load().globalSettings.pendingQueue, null);
|
||||
});
|
||||
|
||||
it('regression: legacy sessionLog:true on disk normalizes to logMode "daily" (NOT "session")', async () => {
|
||||
// Write a config with the legacy boolean only (what an existing user has).
|
||||
await store.save({ globalSettings: { sessionLog: true } });
|
||||
|
||||
56
tests/orphan-tmp.test.js
Normal file
56
tests/orphan-tmp.test.js
Normal file
@ -0,0 +1,56 @@
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const { selectOrphanTmps } = require('../lib/orphan-tmp');
|
||||
|
||||
const BASE = 'electron-config.json';
|
||||
const aliveSet = new Set([100, 200]);
|
||||
const isAlive = (pid) => aliveSet.has(pid);
|
||||
|
||||
test('selects only dead-pid <base>.<pid>.tmp orphans', () => {
|
||||
const files = [
|
||||
'electron-config.json',
|
||||
'electron-config.json.bak',
|
||||
'electron-config.json.tmp',
|
||||
'electron-config.json.100.tmp',
|
||||
'electron-config.json.200.tmp',
|
||||
'electron-config.json.999.tmp',
|
||||
'electron-config.json.4242.tmp',
|
||||
'something-else.500.tmp',
|
||||
'electron-config.json.abc.tmp'
|
||||
];
|
||||
const orphans = selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive });
|
||||
assert.deepEqual(orphans.sort(), ['electron-config.json.4242.tmp', 'electron-config.json.999.tmp']);
|
||||
});
|
||||
|
||||
test('never selects the current process tmp', () => {
|
||||
const files = ['electron-config.json.7.tmp'];
|
||||
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||
});
|
||||
|
||||
test('never selects the FIXED <base>.tmp (used by async _atomicWrite)', () => {
|
||||
const files = ['electron-config.json.tmp'];
|
||||
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||
});
|
||||
|
||||
test('never selects the live config or its .bak', () => {
|
||||
const files = ['electron-config.json', 'electron-config.json.bak'];
|
||||
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||
});
|
||||
|
||||
test('alive pid (incl. EPERM-as-alive) is skipped, preventing deletion of a concurrent instance tmp', () => {
|
||||
const files = ['electron-config.json.100.tmp', 'electron-config.json.300.tmp'];
|
||||
const orphans = selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: (p) => p === 100 });
|
||||
assert.deepEqual(orphans, ['electron-config.json.300.tmp']);
|
||||
});
|
||||
|
||||
test('robust to junk / missing inputs', () => {
|
||||
assert.deepEqual(selectOrphanTmps(null, { baseName: BASE, currentPid: 1, isAlive }), []);
|
||||
assert.deepEqual(selectOrphanTmps(['x', 42, null, undefined], { baseName: BASE, currentPid: 1, isAlive }), []);
|
||||
assert.deepEqual(selectOrphanTmps(['electron-config.json.5.tmp'], {}), []);
|
||||
assert.deepEqual(selectOrphanTmps(['electron-config.json.5.tmp'], { baseName: '', currentPid: 1, isAlive }), []);
|
||||
});
|
||||
|
||||
test('does not match a different base that shares a prefix', () => {
|
||||
const files = ['electron-config.json.backup.5.tmp'];
|
||||
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||
});
|
||||
114
tests/queue-dedup-property.test.js
Normal file
114
tests/queue-dedup-property.test.js
Normal file
@ -0,0 +1,114 @@
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const { partitionRestoredJobsByLog } = require('../lib/queue-dedup');
|
||||
|
||||
function lcg(seed) {
|
||||
let s = seed >>> 0;
|
||||
return () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 4294967296; };
|
||||
}
|
||||
|
||||
function key(f, h) { return `${String(f).toLowerCase()}|${String(h).toLowerCase()}`; }
|
||||
|
||||
test('property: removed iff (done && key in log) OR (savedAt finite && key unambiguous && newest matching log ts >= floor(savedAt/1000)*1000)', () => {
|
||||
const rnd = lcg(0x9e3779b1);
|
||||
const statuses = ['preview', 'done', 'error', 'aborted', 'queued', 'skipped'];
|
||||
const hosters = ['voe.sx', 'byse.sx', 'doodstream.com'];
|
||||
const names = ['a.mkv', 'b.mp4', 'A.MKV', 'c.mov'];
|
||||
const folders = ['C:/A/', 'C:/B/', 'D:/down/'];
|
||||
const pick = (arr) => arr[Math.floor(rnd() * arr.length)];
|
||||
|
||||
for (let iter = 0; iter < 3000; iter++) {
|
||||
const useSavedAt = rnd() < 0.7;
|
||||
const savedAt = useSavedAt ? Math.floor(rnd() * 2_000_000_000_000) : undefined;
|
||||
|
||||
const jobs = [];
|
||||
const nJobs = 1 + Math.floor(rnd() * 6);
|
||||
for (let i = 0; i < nJobs; i++) {
|
||||
const name = pick(names);
|
||||
// Mix shared and distinct paths so ambiguous keys (same name+hoster,
|
||||
// different folder) actually occur and exercise the guard.
|
||||
jobs.push({ id: `j${i}`, fileName: name, hoster: pick(hosters), status: pick(statuses), file: `${pick(folders)}${name}` });
|
||||
}
|
||||
|
||||
const log = [];
|
||||
const nLog = Math.floor(rnd() * 5);
|
||||
for (let i = 0; i < nLog; i++) {
|
||||
const hasTs = rnd() < 0.8;
|
||||
log.push({ fileName: pick(names), hoster: pick(hosters), ts: hasTs ? Math.floor(rnd() * 2_000_000_000_000) : undefined });
|
||||
}
|
||||
|
||||
const logKeys = new Set();
|
||||
const maxTs = new Map();
|
||||
for (const e of log) {
|
||||
const k = key(e.fileName, e.hoster);
|
||||
logKeys.add(k);
|
||||
if (typeof e.ts === 'number' && isFinite(e.ts)) {
|
||||
const prev = maxTs.get(k);
|
||||
if (prev === undefined || e.ts > prev) maxTs.set(k, e.ts);
|
||||
}
|
||||
}
|
||||
const filesPerKey = new Map();
|
||||
for (const job of jobs) {
|
||||
const k = key(job.fileName, job.hoster);
|
||||
if (!filesPerKey.has(k)) filesPerKey.set(k, new Set());
|
||||
filesPerKey.get(k).add(job.file || '');
|
||||
}
|
||||
const floor = (typeof savedAt === 'number' && isFinite(savedAt)) ? Math.floor(savedAt / 1000) * 1000 : null;
|
||||
|
||||
const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||
|
||||
assert.equal(kept.length + removed.length, jobs.length, `iter ${iter}: partition must cover every job exactly once`);
|
||||
const keptIds = new Set(kept.map(j => j.id));
|
||||
const removedIds = new Set(removed.map(j => j.id));
|
||||
assert.equal(keptIds.size + removedIds.size, jobs.length, `iter ${iter}: no job in both partitions`);
|
||||
|
||||
for (const job of jobs) {
|
||||
const k = key(job.fileName, job.hoster);
|
||||
const doneInLog = job.status === 'done' && logKeys.has(k);
|
||||
const unambiguous = filesPerKey.get(k).size <= 1;
|
||||
const afterSnap = floor !== null && unambiguous && maxTs.has(k) && maxTs.get(k) >= floor;
|
||||
const shouldRemove = doneInLog || afterSnap;
|
||||
assert.equal(removedIds.has(job.id), shouldRemove,
|
||||
`iter ${iter}: job ${job.id} (status=${job.status} key=${k} unambig=${unambiguous}) expected removed=${shouldRemove}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('property: a genuinely-pending job is NEVER lost to a same-basename sibling completing after the snapshot', () => {
|
||||
const rnd = lcg(0x1234abcd);
|
||||
for (let iter = 0; iter < 500; iter++) {
|
||||
const savedAt = 1_000_000_000_000 + Math.floor(rnd() * 1_000_000);
|
||||
// X completed after the snapshot (logged); Y is a DIFFERENT file, same
|
||||
// basename + hoster, genuinely pending. Y must survive.
|
||||
const jobs = [
|
||||
{ id: 'X', fileName: 'clip.mp4', hoster: 'voe.sx', status: 'preview', file: 'C:/A/clip.mp4' },
|
||||
{ id: 'Y', fileName: 'clip.mp4', hoster: 'voe.sx', status: 'preview', file: 'C:/B/clip.mp4' }
|
||||
];
|
||||
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: savedAt + 1000 + Math.floor(rnd() * 1000) }];
|
||||
const { kept } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||
assert.ok(kept.some(j => j.id === 'Y'), `iter ${iter}: pending Y must never be silently dropped`);
|
||||
}
|
||||
});
|
||||
|
||||
test('property: 2-arg legacy call NEVER removes a non-done job (the v3.3.80 canary, fuzzed)', () => {
|
||||
const rnd = lcg(0xdeadbeef);
|
||||
const statuses = ['preview', 'error', 'aborted', 'queued', 'skipped'];
|
||||
const hosters = ['voe.sx', 'byse.sx'];
|
||||
const names = ['a.mkv', 'b.mp4'];
|
||||
const pick = (arr) => arr[Math.floor(rnd() * arr.length)];
|
||||
|
||||
for (let iter = 0; iter < 1000; iter++) {
|
||||
const jobs = [];
|
||||
const nJobs = 1 + Math.floor(rnd() * 5);
|
||||
for (let i = 0; i < nJobs; i++) {
|
||||
jobs.push({ id: `j${i}`, fileName: pick(names), hoster: pick(hosters), status: pick(statuses), file: `C:/x/${i}` });
|
||||
}
|
||||
const log = [];
|
||||
const nLog = Math.floor(rnd() * 4);
|
||||
for (let i = 0; i < nLog; i++) {
|
||||
log.push({ fileName: pick(names), hoster: pick(hosters), ts: Math.floor(rnd() * 2_000_000_000_000) });
|
||||
}
|
||||
const { removed } = partitionRestoredJobsByLog(jobs, log);
|
||||
assert.ok(removed.every(j => j.status === 'done'), `iter ${iter}: legacy 2-arg call must never drop a non-done job`);
|
||||
}
|
||||
});
|
||||
@ -1,6 +1,6 @@
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const { partitionRestoredJobsByLog } = require('../lib/queue-dedup');
|
||||
const { partitionRestoredJobsByLog, completedSelectionKeys } = require('../lib/queue-dedup');
|
||||
|
||||
function job(status, fileName, hoster) {
|
||||
return { status, fileName, hoster, file: `C:/dl/${fileName}` };
|
||||
@ -135,6 +135,45 @@ test('ts-gate: done job uploaded after snapshot is dropped via either rule', ()
|
||||
assert.equal(removed.length, 1);
|
||||
});
|
||||
|
||||
test('ts-gate ambiguity guard: a pending same-basename file in a DIFFERENT folder is NOT lost when a sibling completes after the snapshot', () => {
|
||||
// X (C:/A/clip.mp4) was uploaded after the snapshot and logged. Y is a
|
||||
// genuinely-different file (C:/B/clip.mp4), same basename + hoster, still
|
||||
// pending. The log records only basenames, so the ts-rule must not drop Y.
|
||||
const jobs = [
|
||||
{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' },
|
||||
{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/B/clip.mp4' }
|
||||
];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||
assert.equal(removed.length, 0, 'ambiguous key -> ts-rule suppressed, no pending file lost');
|
||||
assert.equal(kept.length, 2);
|
||||
});
|
||||
|
||||
test('ts-gate ambiguity guard: the done-in-log rule still applies on an ambiguous key', () => {
|
||||
// Even when the key is ambiguous, a job that is actually 'done' and in the log
|
||||
// is still decluttered (pre-existing rule, unchanged by the guard).
|
||||
const jobs = [
|
||||
{ status: 'done', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' },
|
||||
{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/B/clip.mp4' }
|
||||
];
|
||||
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||
assert.equal(removed.length, 1);
|
||||
assert.equal(removed[0].status, 'done');
|
||||
assert.equal(removed[0].file, 'C:/A/clip.mp4');
|
||||
assert.ok(kept.some(j => j.file === 'C:/B/clip.mp4'), 'the distinct pending file survives');
|
||||
});
|
||||
|
||||
test('ts-gate: a unique-path ghost still drops (guard does not weaken the common case)', () => {
|
||||
const jobs = [{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' }];
|
||||
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const { removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||
assert.equal(removed.length, 1, 'single job for the key -> unambiguous -> ghost dropped as before');
|
||||
});
|
||||
|
||||
test('ts-gate: multi-hoster partial completion — the reported bug shape (drop only the completed hosters)', () => {
|
||||
// One file queued to 4 hosters; close mid-upload. After the snapshot, 2 hosters
|
||||
// completed (logged), 2 never started. On restart all 4 restore as 'preview'.
|
||||
@ -159,3 +198,56 @@ test('ts-gate: multi-hoster partial completion — the reported bug shape (drop
|
||||
assert.ok(kept.some(j => j.hoster === 'vidmoly.me'));
|
||||
assert.ok(kept.some(j => j.hoster === 'byse.sx'));
|
||||
});
|
||||
|
||||
test('completedSelectionKeys: a selectedFile that completed after the snapshot yields its full-path|hoster key', () => {
|
||||
const selectedFiles = [{ path: 'C:/dl/done.mp4', name: 'done.mp4' }, { path: 'C:/dl/pending.mp4', name: 'pending.mp4' }];
|
||||
const hosters = ['voe.sx'];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const log = [{ fileName: 'done.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt);
|
||||
assert.deepEqual(keys, ['C:/dl/done.mp4|voe.sx'], 'only the completed file is seeded; pending is not');
|
||||
});
|
||||
|
||||
test('completedSelectionKeys: per-hoster — a file done on voe but not byse only seeds the voe key', () => {
|
||||
const selectedFiles = [{ path: 'C:/dl/a.mp4', name: 'a.mp4' }];
|
||||
const hosters = ['voe.sx', 'byse.sx'];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const log = [{ fileName: 'a.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt);
|
||||
assert.deepEqual(keys, ['C:/dl/a.mp4|voe.sx'], 'the still-pending byse upload is NOT seeded');
|
||||
});
|
||||
|
||||
test('completedSelectionKeys: ambiguous basename across folders seeds NOTHING (no lost re-preview)', () => {
|
||||
const selectedFiles = [{ path: 'C:/A/clip.mp4', name: 'clip.mp4' }, { path: 'C:/B/clip.mp4', name: 'clip.mp4' }];
|
||||
const hosters = ['voe.sx'];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt);
|
||||
assert.deepEqual(keys, [], 'ambiguous -> neither path is suppressed, both re-preview (safe direction)');
|
||||
});
|
||||
|
||||
test('completedSelectionKeys: an OLDER completion (pre-snapshot re-queue) is NOT seeded', () => {
|
||||
const selectedFiles = [{ path: 'C:/dl/reup.mp4', name: 'reup.mp4' }];
|
||||
const hosters = ['voe.sx'];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const log = [{ fileName: 'reup.mp4', hoster: 'voe.sx', ts: T('2026-06-19 11:00:00') }];
|
||||
assert.deepEqual(completedSelectionKeys(selectedFiles, hosters, log, savedAt), []);
|
||||
});
|
||||
|
||||
test('completedSelectionKeys: no savedAt / junk inputs -> empty (legacy + robustness)', () => {
|
||||
const sf = [{ path: 'C:/dl/a.mp4', name: 'a.mp4' }];
|
||||
const log = [{ fileName: 'a.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
assert.deepEqual(completedSelectionKeys(sf, ['voe.sx'], log, undefined), []);
|
||||
assert.deepEqual(completedSelectionKeys(null, ['voe.sx'], log, 1), []);
|
||||
assert.deepEqual(completedSelectionKeys(sf, null, log, 1), []);
|
||||
assert.deepEqual(completedSelectionKeys([], [], log, 1), []);
|
||||
assert.deepEqual(completedSelectionKeys([{ name: 'x' }], ['voe.sx'], log, 1), [], 'entry without path is skipped');
|
||||
});
|
||||
|
||||
test('completedSelectionKeys: derives basename from path when name is missing', () => {
|
||||
const selectedFiles = [{ path: 'C:/dl/sub/movie.mp4' }];
|
||||
const hosters = ['voe.sx'];
|
||||
const savedAt = T('2026-06-19 12:00:00');
|
||||
const log = [{ fileName: 'movie.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||
assert.deepEqual(completedSelectionKeys(selectedFiles, hosters, log, savedAt), ['C:/dl/sub/movie.mp4|voe.sx']);
|
||||
});
|
||||
|
||||
83
tests/queue-persistence-scenario.test.js
Normal file
83
tests/queue-persistence-scenario.test.js
Normal file
@ -0,0 +1,83 @@
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const { formatUploadLogLine, parseUploadLogLine } = require('../lib/upload-log');
|
||||
const { partitionRestoredJobsByLog } = require('../lib/queue-dedup');
|
||||
|
||||
function makeJobs(n, hoster, status, offset = 0) {
|
||||
const jobs = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
const fileName = `clip_${String(i + offset).padStart(4, '0')}.mp4`;
|
||||
jobs.push({ id: `j-${hoster}-${i + offset}`, file: `D:/inbox/${fileName}`, fileName, hoster, status });
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
test('user report: 300 queued, ~200 finished mid-session before a hard kill — only the finished drop', () => {
|
||||
const hoster = 'byse.sx';
|
||||
const snapshot = new Date(2026, 5, 19, 22, 0, 0);
|
||||
const savedAt = snapshot.getTime();
|
||||
const restoredJobs = makeJobs(300, hoster, 'preview');
|
||||
|
||||
const completionBase = new Date(2026, 5, 19, 22, 5, 0).getTime();
|
||||
const logEntries = [];
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const d = new Date(completionBase + i * 1000);
|
||||
logEntries.push(parseUploadLogLine(
|
||||
formatUploadLogLine(d, hoster, `https://byse.sx/d/x${i}`, `clip_${String(i).padStart(4, '0')}.mp4`)
|
||||
));
|
||||
}
|
||||
|
||||
const { kept, removed } = partitionRestoredJobsByLog(restoredJobs, logEntries, savedAt);
|
||||
assert.equal(removed.length, 200, 'the 200 completed-after-snapshot files are dropped as ghosts');
|
||||
assert.equal(kept.length, 100, 'the 100 never-finished files stay queued');
|
||||
assert.ok(kept.every(j => Number(j.fileName.slice(5, 9)) >= 200), 'kept are exactly indices 200..299');
|
||||
const keptNames = new Set(kept.map(j => j.fileName));
|
||||
assert.ok(removed.every(j => !keptNames.has(j.fileName)));
|
||||
});
|
||||
|
||||
test('multi-hoster batch: per-hoster completion is independent (a file done on voe but not byse keeps byse)', () => {
|
||||
const savedAt = new Date(2026, 5, 19, 22, 0, 0).getTime();
|
||||
const done = new Date(2026, 5, 19, 22, 3, 0);
|
||||
const jobs = [
|
||||
...makeJobs(3, 'voe.sx', 'preview'),
|
||||
...makeJobs(3, 'byse.sx', 'preview')
|
||||
];
|
||||
const logEntries = [
|
||||
parseUploadLogLine(formatUploadLogLine(done, 'voe.sx', 'l', 'clip_0000.mp4')),
|
||||
parseUploadLogLine(formatUploadLogLine(done, 'voe.sx', 'l', 'clip_0001.mp4')),
|
||||
parseUploadLogLine(formatUploadLogLine(done, 'byse.sx', 'l', 'clip_0000.mp4'))
|
||||
];
|
||||
const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries, savedAt);
|
||||
assert.equal(removed.length, 3);
|
||||
assert.ok(removed.some(j => j.hoster === 'voe.sx' && j.fileName === 'clip_0000.mp4'));
|
||||
assert.ok(removed.some(j => j.hoster === 'voe.sx' && j.fileName === 'clip_0001.mp4'));
|
||||
assert.ok(removed.some(j => j.hoster === 'byse.sx' && j.fileName === 'clip_0000.mp4'));
|
||||
assert.ok(kept.some(j => j.hoster === 'byse.sx' && j.fileName === 'clip_0001.mp4'), 'byse clip_0001 not logged -> kept');
|
||||
});
|
||||
|
||||
test('clean idle close (snapshot AFTER completion) keeps an intentional re-queue of an old file', () => {
|
||||
const hoster = 'voe.sx';
|
||||
const yesterday = new Date(2026, 5, 18, 12, 0, 0);
|
||||
const logEntries = [parseUploadLogLine(formatUploadLogLine(yesterday, hoster, 'link', 'reupload_me.mp4'))];
|
||||
const savedAt = new Date(2026, 5, 19, 9, 0, 0).getTime();
|
||||
const jobs = [{ id: 'r1', file: 'D:/x/reupload_me.mp4', fileName: 'reupload_me.mp4', hoster, status: 'preview' }];
|
||||
const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries, savedAt);
|
||||
assert.equal(removed.length, 0, 'an upload older than the snapshot is a deliberate re-queue and survives');
|
||||
assert.equal(kept.length, 1);
|
||||
});
|
||||
|
||||
test('legacy snapshot without savedAt (pre-v3.3.80 config) falls back to done-only dedup', () => {
|
||||
const hoster = 'voe.sx';
|
||||
const logEntries = [
|
||||
parseUploadLogLine(formatUploadLogLine(new Date(2026, 5, 19, 12, 0, 0), hoster, 'l', 'done.mp4')),
|
||||
parseUploadLogLine(formatUploadLogLine(new Date(2026, 5, 19, 12, 1, 0), hoster, 'l', 'preview.mp4'))
|
||||
];
|
||||
const jobs = [
|
||||
{ id: 'a', file: 'D:/x/done.mp4', fileName: 'done.mp4', hoster, status: 'done' },
|
||||
{ id: 'b', file: 'D:/x/preview.mp4', fileName: 'preview.mp4', hoster, status: 'preview' }
|
||||
];
|
||||
const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries);
|
||||
assert.equal(removed.length, 1, 'only the done job is decluttered when no savedAt is available');
|
||||
assert.equal(removed[0].id, 'a');
|
||||
assert.ok(kept.some(j => j.id === 'b'), 'the preview survives the legacy path');
|
||||
});
|
||||
Loading…
Reference in New Issue
Block a user