fix(queue): close two lost-work / sticky-ghost edge cases found by intensive testing
Follow-up hardening on the v3.3.80 queue-persistence fix after a deeper adversarial sweep + reviewer pass over the WHOLE subsystem (not just the diff). Two real defects: 1. Basename-collision lost work (regression introduced by v3.3.80 FIX A). restoreQueueStateFromConfig collapses jobs on the FULL path while the ts-gate keys on basename|hoster. Two genuinely different files with the same basename queued to the same hoster from different folders therefore share a gate key: if one was logged after savedAt, the ts-rule dropped BOTH — silently losing the still-pending one. Pre-FIX-A only 'done' jobs were dropped, so a pending file was never at risk. Fix: an ambiguity guard in partitionRestoredJobsByLog — the ts-rule is suppressed when a basename|hoster key maps to more than one distinct file path (the log records only basenames, so it can't say which physical file completed). The done-in-log rule is unchanged. Fails safe: worst case a visible ghost survives, never silent data loss. 2. selectedFiles re-materialization with removeFromQueueOnDone=ON (second mechanism, independent of the stale snapshot). When that setting is on, a completed job is stripped from queueJobs but its path stays in selectedFiles (syncSelectedFilesFromQueue only runs at batch-done, never on a mid-upload close). On restart the ts-gate operates on queueJobs and never sees it, then the startup updateUploadView -> buildQueuePreview re-creates it as a preview ghost AFTER the gate ran, and it re-persists with a fresh savedAt — sticky. Fix: completedSelectionKeys() seeds _completedUploadKeys (the set buildQueuePreview already consults) from the log at startup, keyed on full path, with the same ambiguity guard. Log-based so it survives a hard kill, consistent with FIX A. Also extracts the orphan-tmp sweep decision into lib/orphan-tmp.js (was untested inline code in main.js; behavior-preserving) and adds executable coverage for the paths that were previously only argued from logic: orphan-tmp sweep, config-store pendingQueue+savedAt round-trip, an end-to-end scenario in the exact user-reported shape (300 queued / ~200 finished mid-session), and a 3000+500-iteration property fuzz of the gate invariant including the lost-work guarantee. 359/359 green, ESLint clean, smoke-boot unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f7c8d308fc
commit
6b0c515a9b
29
lib/orphan-tmp.js
Normal file
29
lib/orphan-tmp.js
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
(function (root) {
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
function selectOrphanTmps(fileNames, opts) {
|
||||||
|
const o = opts || {};
|
||||||
|
const baseName = String(o.baseName || '');
|
||||||
|
const currentPid = o.currentPid;
|
||||||
|
const isAlive = typeof o.isAlive === 'function' ? o.isAlive : () => false;
|
||||||
|
const out = [];
|
||||||
|
if (!baseName || !Array.isArray(fileNames)) return out;
|
||||||
|
const prefix = baseName + '.';
|
||||||
|
const suffix = '.tmp';
|
||||||
|
for (const file of fileNames) {
|
||||||
|
if (typeof file !== 'string') continue;
|
||||||
|
if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue;
|
||||||
|
const mid = file.slice(prefix.length, file.length - suffix.length);
|
||||||
|
if (!/^\d+$/.test(mid)) continue;
|
||||||
|
const pid = Number(mid);
|
||||||
|
if (pid === currentPid) continue;
|
||||||
|
if (isAlive(pid)) continue;
|
||||||
|
out.push(file);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
const api = { selectOrphanTmps };
|
||||||
|
if (typeof module !== 'undefined' && module.exports) module.exports = api;
|
||||||
|
else if (root) root.OrphanTmp = api;
|
||||||
|
})(typeof window !== 'undefined' ? window : this);
|
||||||
@ -59,11 +59,22 @@
|
|||||||
? Math.floor(savedAt / 1000) * 1000
|
? Math.floor(savedAt / 1000) * 1000
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
|
const filesPerKey = new Map();
|
||||||
|
for (const job of jobs) {
|
||||||
|
if (job && job.fileName && job.hoster) {
|
||||||
|
const jk = _key(job.fileName, job.hoster);
|
||||||
|
let set = filesPerKey.get(jk);
|
||||||
|
if (!set) { set = new Set(); filesPerKey.set(jk, set); }
|
||||||
|
set.add(job.file || '');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
const hasIds = job && job.fileName && job.hoster;
|
const hasIds = job && job.fileName && job.hoster;
|
||||||
const k = hasIds ? _key(job.fileName, job.hoster) : null;
|
const k = hasIds ? _key(job.fileName, job.hoster) : null;
|
||||||
const doneInLog = job && job.status === 'done' && hasIds && logKeys.has(k);
|
const doneInLog = job && job.status === 'done' && hasIds && logKeys.has(k);
|
||||||
const uploadedAfterSnapshot = savedAtFloor !== null && k !== null
|
const keyUnambiguous = k !== null && filesPerKey.get(k).size <= 1;
|
||||||
|
const uploadedAfterSnapshot = savedAtFloor !== null && k !== null && keyUnambiguous
|
||||||
&& logMaxTs.has(k) && logMaxTs.get(k) >= savedAtFloor;
|
&& logMaxTs.has(k) && logMaxTs.get(k) >= savedAtFloor;
|
||||||
if (doneInLog || uploadedAfterSnapshot) {
|
if (doneInLog || uploadedAfterSnapshot) {
|
||||||
removed.push(job);
|
removed.push(job);
|
||||||
@ -74,7 +85,25 @@
|
|||||||
return { kept, removed };
|
return { kept, removed };
|
||||||
}
|
}
|
||||||
|
|
||||||
const api = { partitionRestoredJobsByLog };
|
function completedSelectionKeys(selectedFiles, hosters, logEntries, savedAt) {
|
||||||
|
const out = [];
|
||||||
|
if (!Array.isArray(selectedFiles) || !Array.isArray(hosters)) return out;
|
||||||
|
if (!(typeof savedAt === 'number' && isFinite(savedAt))) return out;
|
||||||
|
const synthetic = [];
|
||||||
|
for (const f of selectedFiles) {
|
||||||
|
if (!f || !f.path) continue;
|
||||||
|
const name = f.name || String(f.path).split(/[\\/]/).pop();
|
||||||
|
for (const h of hosters) {
|
||||||
|
if (h) synthetic.push({ fileName: name, hoster: h, file: f.path, status: 'preview' });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (synthetic.length === 0) return out;
|
||||||
|
const { removed } = partitionRestoredJobsByLog(synthetic, logEntries, savedAt);
|
||||||
|
for (const job of removed) out.push(`${job.file}|${job.hoster}`);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
const api = { partitionRestoredJobsByLog, completedSelectionKeys };
|
||||||
|
|
||||||
if (typeof module !== 'undefined' && module.exports) {
|
if (typeof module !== 'undefined' && module.exports) {
|
||||||
module.exports = api;
|
module.exports = api;
|
||||||
|
|||||||
21
main.js
21
main.js
@ -18,6 +18,7 @@ const RemoteServer = require('./lib/remote-server');
|
|||||||
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
||||||
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
||||||
const { formatUploadLogLine, parseUploadLogLine } = require('./lib/upload-log');
|
const { formatUploadLogLine, parseUploadLogLine } = require('./lib/upload-log');
|
||||||
|
const { selectOrphanTmps } = require('./lib/orphan-tmp');
|
||||||
const { sanitizeConfig, buildSupportBundleText } = require('./lib/support-bundle');
|
const { sanitizeConfig, buildSupportBundleText } = require('./lib/support-bundle');
|
||||||
const { buildWebhookRequest, isAllAborted } = require('./lib/webhook-notify');
|
const { buildWebhookRequest, isAllAborted } = require('./lib/webhook-notify');
|
||||||
|
|
||||||
@ -2263,17 +2264,15 @@ function _sleepSyncMs(ms) {
|
|||||||
function _sweepOrphanConfigTmps() {
|
function _sweepOrphanConfigTmps() {
|
||||||
try {
|
try {
|
||||||
const dir = path.dirname(configStore.filePath);
|
const dir = path.dirname(configStore.filePath);
|
||||||
const prefix = path.basename(configStore.filePath) + '.';
|
const isAlive = (pid) => {
|
||||||
const suffix = '.tmp';
|
try { process.kill(pid, 0); return true; } catch (e) { return !!(e && e.code === 'EPERM'); }
|
||||||
for (const file of fs.readdirSync(dir)) {
|
};
|
||||||
if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue;
|
const orphans = selectOrphanTmps(fs.readdirSync(dir), {
|
||||||
const mid = file.slice(prefix.length, file.length - suffix.length);
|
baseName: path.basename(configStore.filePath),
|
||||||
if (!/^\d+$/.test(mid)) continue;
|
currentPid: process.pid,
|
||||||
const pid = Number(mid);
|
isAlive
|
||||||
if (pid === process.pid) continue;
|
});
|
||||||
let alive = false;
|
for (const file of orphans) {
|
||||||
try { process.kill(pid, 0); alive = true; } catch (e) { alive = !!(e && e.code === 'EPERM'); }
|
|
||||||
if (alive) continue;
|
|
||||||
try { fs.unlinkSync(path.join(dir, file)); } catch {}
|
try { fs.unlinkSync(path.join(dir, file)); } catch {}
|
||||||
}
|
}
|
||||||
} catch {}
|
} catch {}
|
||||||
|
|||||||
@ -4935,7 +4935,7 @@ function handleShutdownCountdown(data) {
|
|||||||
|
|
||||||
// --- Auto-deduplicate restored queue against own upload log on startup ---
|
// --- Auto-deduplicate restored queue against own upload log on startup ---
|
||||||
async function _autoDeduplicateFromLog() {
|
async function _autoDeduplicateFromLog() {
|
||||||
if (queueJobs.length === 0) return;
|
if (queueJobs.length === 0 && selectedFiles.length === 0) return;
|
||||||
try {
|
try {
|
||||||
const entries = await window.api.readOwnUploadLog();
|
const entries = await window.api.readOwnUploadLog();
|
||||||
if (!entries || entries.length === 0) return;
|
if (!entries || entries.length === 0) return;
|
||||||
@ -4954,6 +4954,11 @@ async function _autoDeduplicateFromLog() {
|
|||||||
syncSelectedFilesFromQueue();
|
syncSelectedFilesFromQueue();
|
||||||
window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`);
|
window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`);
|
||||||
}
|
}
|
||||||
|
const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt);
|
||||||
|
if (seedKeys.length > 0) {
|
||||||
|
for (const k of seedKeys) _completedUploadKeys.add(k);
|
||||||
|
window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`);
|
||||||
|
}
|
||||||
} catch {}
|
} catch {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -56,3 +56,32 @@ fertig hochgeladene Dateien wieder in der Liste.
|
|||||||
Windows-Profil verschlüsselt, nicht entschlüsselbar). Symptom tritt nur auf bei
|
Windows-Profil verschlüsselt, nicht entschlüsselbar). Symptom tritt nur auf bei
|
||||||
Close WÄHREND aktivem Upload oder Hard-Kill — ein sauberer Idle-Close war schon
|
Close WÄHREND aktivem Upload oder Hard-Kill — ein sauberer Idle-Close war schon
|
||||||
vorher korrekt.
|
vorher korrekt.
|
||||||
|
|
||||||
|
## Runde 2 — "noch intensiver" (v3.3.81)
|
||||||
|
- **Echte ausführbare Tests** für bisher nur logisch abgedeckte Pfade: `tests/orphan-tmp.test.js`
|
||||||
|
(Sweep-Entscheidung, extrahiert nach `lib/orphan-tmp.js`), config-store `pendingQueue`+`savedAt`
|
||||||
|
Roundtrip, `tests/queue-persistence-scenario.test.js` (exakte 300/100-Bug-Form + multi-hoster),
|
||||||
|
`tests/queue-dedup-property.test.js` (3000+500 Fuzz-Iterationen gegen die formale Invariante).
|
||||||
|
- **Breite adversariale Bug-Jagd** übers GANZE Subsystem: lief tooling-bedingt teils kaputt
|
||||||
|
(bug-analyzer ohne File-Tools, Socket-Fehler) → 3 unverifizierte Hypothesen SELBST am Code
|
||||||
|
geprüft:
|
||||||
|
- #2 (gedroppte done-Jobs reappear via buildQueuePreview) → REFUTED: `_completedUploadKeys`
|
||||||
|
(full path) == buildQueuePreview-Key; nach Gate räumt `syncSelectedFilesFromQueue` selectedFiles.
|
||||||
|
- #3 (done-File bleibt in selectedFiles → re-preview) → **REAL (Advisor-Catch) & gefixt.** Meine
|
||||||
|
erste Abweisung war zu schnell: `syncSelectedFilesFromQueue` läuft NICHT beim Mid-Upload-Close.
|
||||||
|
Bei `removeFromQueueOnDone=ON` werden fertige Jobs aus queueJobs entfernt, bleiben aber in
|
||||||
|
selectedFiles; `updateUploadView`→`buildQueuePreview` (Startup, Zeile 990) re-materialisiert sie
|
||||||
|
als Preview-Ghost NACH dem Gate → sticky. Fix: `completedSelectionKeys` (queue-dedup.js) seedet
|
||||||
|
beim Start `_completedUploadKeys` (full-path, log-basiert/hard-kill-durabel, gleiche Ambiguity-
|
||||||
|
Guard) → buildQueuePreview überspringt fertige (file|hoster)-Paare. Nur relevant bei
|
||||||
|
removeFromQueueOnDone=ON (Default OFF).
|
||||||
|
- #1 (basename-Kollision droppt PENDING Datei = Lost Work) → **REAL & gefixt.** FIX A's ts-Regel
|
||||||
|
keyt auf basename, Restore-Collapse auf full path → zwei gleichnamige Dateien aus verschiedenen
|
||||||
|
Ordnern an denselben Hoster: die geloggte droppte fälschlich auch die andere PENDING. **Ambiguity-
|
||||||
|
Guard** in queue-dedup.js: ts-Regel wird unterdrückt, wenn ein basename|hoster-Key auf mehrere
|
||||||
|
DISTINKTE Pfade zeigt (done-Regel unberührt). Fail-safe: schlimmstenfalls überlebt ein
|
||||||
|
sichtbarer Ghost, NIE stiller Datenverlust.
|
||||||
|
- **Un-gehuntete Bereiche selbst abgeklopft:** DST/Clock-Skew → Fehler nur in SICHERER Richtung
|
||||||
|
(Ghost bleibt, kein Lost Work), inhärente Grenze von Sekunden-Lokalzeit-Logs. Log-Discovery
|
||||||
|
readdir-Filter `startsWith(base)&&endsWith(ext)` fängt single/daily/session — keine verpassten
|
||||||
|
Log-Files. 353/353 grün, ESLint clean, 3× Suite ohne Flake.
|
||||||
|
|||||||
@ -61,6 +61,42 @@ describe('ConfigStore', () => {
|
|||||||
assert.equal(config.globalSettings.logMode, 'single');
|
assert.equal(config.globalSettings.logMode, 'single');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('persists pendingQueue incl. savedAt (number) and ts-bearing jobs across save/load', async () => {
|
||||||
|
// The queue-persistence fix stamps pendingQueue.savedAt and restores it on launch.
|
||||||
|
// This proves the persistence layer round-trips the new fields untouched (the
|
||||||
|
// ts-gate is worthless if savedAt does not survive serialization).
|
||||||
|
const pendingQueue = {
|
||||||
|
savedAt: 1750000000123,
|
||||||
|
selectedUploadHosters: ['voe.sx', 'byse.sx'],
|
||||||
|
selectedFiles: [{ path: 'C:/dl/a.mkv', name: 'a.mkv', size: 4242 }],
|
||||||
|
queueJobs: [
|
||||||
|
{ id: 'j1', file: 'C:/dl/a.mkv', fileName: 'a.mkv', hoster: 'voe.sx', status: 'preview', bytesTotal: 4242, maxAttempts: 0 },
|
||||||
|
{ id: 'j2', file: 'C:/dl/a.mkv', fileName: 'a.mkv', hoster: 'byse.sx', status: 'error', error: 'boom', maxAttempts: 3 }
|
||||||
|
]
|
||||||
|
};
|
||||||
|
const current = store.load();
|
||||||
|
await store.save({ globalSettings: { ...current.globalSettings, pendingQueue } });
|
||||||
|
const loaded = store.load();
|
||||||
|
const pq = loaded.globalSettings.pendingQueue;
|
||||||
|
assert.equal(pq.savedAt, 1750000000123, 'savedAt epoch survives JSON round-trip');
|
||||||
|
assert.equal(typeof pq.savedAt, 'number');
|
||||||
|
assert.equal(pq.queueJobs.length, 2);
|
||||||
|
assert.equal(pq.queueJobs[0].fileName, 'a.mkv');
|
||||||
|
assert.equal(pq.queueJobs[0].hoster, 'voe.sx');
|
||||||
|
assert.equal(pq.queueJobs[1].status, 'error');
|
||||||
|
assert.deepEqual(pq.selectedUploadHosters, ['voe.sx', 'byse.sx']);
|
||||||
|
assert.equal(pq.selectedFiles[0].path, 'C:/dl/a.mkv');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('pendingQueue can be cleared back to null (clearPersistedQueueStateSoon path)', async () => {
|
||||||
|
const current = store.load();
|
||||||
|
await store.save({ globalSettings: { ...current.globalSettings, pendingQueue: { savedAt: 1, queueJobs: [] } } });
|
||||||
|
assert.ok(store.load().globalSettings.pendingQueue);
|
||||||
|
const c2 = store.load();
|
||||||
|
await store.save({ globalSettings: { ...c2.globalSettings, pendingQueue: null } });
|
||||||
|
assert.equal(store.load().globalSettings.pendingQueue, null);
|
||||||
|
});
|
||||||
|
|
||||||
it('regression: legacy sessionLog:true on disk normalizes to logMode "daily" (NOT "session")', async () => {
|
it('regression: legacy sessionLog:true on disk normalizes to logMode "daily" (NOT "session")', async () => {
|
||||||
// Write a config with the legacy boolean only (what an existing user has).
|
// Write a config with the legacy boolean only (what an existing user has).
|
||||||
await store.save({ globalSettings: { sessionLog: true } });
|
await store.save({ globalSettings: { sessionLog: true } });
|
||||||
|
|||||||
56
tests/orphan-tmp.test.js
Normal file
56
tests/orphan-tmp.test.js
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
const { test } = require('node:test');
|
||||||
|
const assert = require('node:assert');
|
||||||
|
const { selectOrphanTmps } = require('../lib/orphan-tmp');
|
||||||
|
|
||||||
|
const BASE = 'electron-config.json';
|
||||||
|
const aliveSet = new Set([100, 200]);
|
||||||
|
const isAlive = (pid) => aliveSet.has(pid);
|
||||||
|
|
||||||
|
test('selects only dead-pid <base>.<pid>.tmp orphans', () => {
|
||||||
|
const files = [
|
||||||
|
'electron-config.json',
|
||||||
|
'electron-config.json.bak',
|
||||||
|
'electron-config.json.tmp',
|
||||||
|
'electron-config.json.100.tmp',
|
||||||
|
'electron-config.json.200.tmp',
|
||||||
|
'electron-config.json.999.tmp',
|
||||||
|
'electron-config.json.4242.tmp',
|
||||||
|
'something-else.500.tmp',
|
||||||
|
'electron-config.json.abc.tmp'
|
||||||
|
];
|
||||||
|
const orphans = selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive });
|
||||||
|
assert.deepEqual(orphans.sort(), ['electron-config.json.4242.tmp', 'electron-config.json.999.tmp']);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('never selects the current process tmp', () => {
|
||||||
|
const files = ['electron-config.json.7.tmp'];
|
||||||
|
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('never selects the FIXED <base>.tmp (used by async _atomicWrite)', () => {
|
||||||
|
const files = ['electron-config.json.tmp'];
|
||||||
|
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('never selects the live config or its .bak', () => {
|
||||||
|
const files = ['electron-config.json', 'electron-config.json.bak'];
|
||||||
|
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('alive pid (incl. EPERM-as-alive) is skipped, preventing deletion of a concurrent instance tmp', () => {
|
||||||
|
const files = ['electron-config.json.100.tmp', 'electron-config.json.300.tmp'];
|
||||||
|
const orphans = selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: (p) => p === 100 });
|
||||||
|
assert.deepEqual(orphans, ['electron-config.json.300.tmp']);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('robust to junk / missing inputs', () => {
|
||||||
|
assert.deepEqual(selectOrphanTmps(null, { baseName: BASE, currentPid: 1, isAlive }), []);
|
||||||
|
assert.deepEqual(selectOrphanTmps(['x', 42, null, undefined], { baseName: BASE, currentPid: 1, isAlive }), []);
|
||||||
|
assert.deepEqual(selectOrphanTmps(['electron-config.json.5.tmp'], {}), []);
|
||||||
|
assert.deepEqual(selectOrphanTmps(['electron-config.json.5.tmp'], { baseName: '', currentPid: 1, isAlive }), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('does not match a different base that shares a prefix', () => {
|
||||||
|
const files = ['electron-config.json.backup.5.tmp'];
|
||||||
|
assert.deepEqual(selectOrphanTmps(files, { baseName: BASE, currentPid: 7, isAlive: () => false }), []);
|
||||||
|
});
|
||||||
114
tests/queue-dedup-property.test.js
Normal file
114
tests/queue-dedup-property.test.js
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
const { test } = require('node:test');
|
||||||
|
const assert = require('node:assert');
|
||||||
|
const { partitionRestoredJobsByLog } = require('../lib/queue-dedup');
|
||||||
|
|
||||||
|
function lcg(seed) {
|
||||||
|
let s = seed >>> 0;
|
||||||
|
return () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 4294967296; };
|
||||||
|
}
|
||||||
|
|
||||||
|
function key(f, h) { return `${String(f).toLowerCase()}|${String(h).toLowerCase()}`; }
|
||||||
|
|
||||||
|
test('property: removed iff (done && key in log) OR (savedAt finite && key unambiguous && newest matching log ts >= floor(savedAt/1000)*1000)', () => {
|
||||||
|
const rnd = lcg(0x9e3779b1);
|
||||||
|
const statuses = ['preview', 'done', 'error', 'aborted', 'queued', 'skipped'];
|
||||||
|
const hosters = ['voe.sx', 'byse.sx', 'doodstream.com'];
|
||||||
|
const names = ['a.mkv', 'b.mp4', 'A.MKV', 'c.mov'];
|
||||||
|
const folders = ['C:/A/', 'C:/B/', 'D:/down/'];
|
||||||
|
const pick = (arr) => arr[Math.floor(rnd() * arr.length)];
|
||||||
|
|
||||||
|
for (let iter = 0; iter < 3000; iter++) {
|
||||||
|
const useSavedAt = rnd() < 0.7;
|
||||||
|
const savedAt = useSavedAt ? Math.floor(rnd() * 2_000_000_000_000) : undefined;
|
||||||
|
|
||||||
|
const jobs = [];
|
||||||
|
const nJobs = 1 + Math.floor(rnd() * 6);
|
||||||
|
for (let i = 0; i < nJobs; i++) {
|
||||||
|
const name = pick(names);
|
||||||
|
// Mix shared and distinct paths so ambiguous keys (same name+hoster,
|
||||||
|
// different folder) actually occur and exercise the guard.
|
||||||
|
jobs.push({ id: `j${i}`, fileName: name, hoster: pick(hosters), status: pick(statuses), file: `${pick(folders)}${name}` });
|
||||||
|
}
|
||||||
|
|
||||||
|
const log = [];
|
||||||
|
const nLog = Math.floor(rnd() * 5);
|
||||||
|
for (let i = 0; i < nLog; i++) {
|
||||||
|
const hasTs = rnd() < 0.8;
|
||||||
|
log.push({ fileName: pick(names), hoster: pick(hosters), ts: hasTs ? Math.floor(rnd() * 2_000_000_000_000) : undefined });
|
||||||
|
}
|
||||||
|
|
||||||
|
const logKeys = new Set();
|
||||||
|
const maxTs = new Map();
|
||||||
|
for (const e of log) {
|
||||||
|
const k = key(e.fileName, e.hoster);
|
||||||
|
logKeys.add(k);
|
||||||
|
if (typeof e.ts === 'number' && isFinite(e.ts)) {
|
||||||
|
const prev = maxTs.get(k);
|
||||||
|
if (prev === undefined || e.ts > prev) maxTs.set(k, e.ts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const filesPerKey = new Map();
|
||||||
|
for (const job of jobs) {
|
||||||
|
const k = key(job.fileName, job.hoster);
|
||||||
|
if (!filesPerKey.has(k)) filesPerKey.set(k, new Set());
|
||||||
|
filesPerKey.get(k).add(job.file || '');
|
||||||
|
}
|
||||||
|
const floor = (typeof savedAt === 'number' && isFinite(savedAt)) ? Math.floor(savedAt / 1000) * 1000 : null;
|
||||||
|
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||||
|
|
||||||
|
assert.equal(kept.length + removed.length, jobs.length, `iter ${iter}: partition must cover every job exactly once`);
|
||||||
|
const keptIds = new Set(kept.map(j => j.id));
|
||||||
|
const removedIds = new Set(removed.map(j => j.id));
|
||||||
|
assert.equal(keptIds.size + removedIds.size, jobs.length, `iter ${iter}: no job in both partitions`);
|
||||||
|
|
||||||
|
for (const job of jobs) {
|
||||||
|
const k = key(job.fileName, job.hoster);
|
||||||
|
const doneInLog = job.status === 'done' && logKeys.has(k);
|
||||||
|
const unambiguous = filesPerKey.get(k).size <= 1;
|
||||||
|
const afterSnap = floor !== null && unambiguous && maxTs.has(k) && maxTs.get(k) >= floor;
|
||||||
|
const shouldRemove = doneInLog || afterSnap;
|
||||||
|
assert.equal(removedIds.has(job.id), shouldRemove,
|
||||||
|
`iter ${iter}: job ${job.id} (status=${job.status} key=${k} unambig=${unambiguous}) expected removed=${shouldRemove}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('property: a genuinely-pending job is NEVER lost to a same-basename sibling completing after the snapshot', () => {
|
||||||
|
const rnd = lcg(0x1234abcd);
|
||||||
|
for (let iter = 0; iter < 500; iter++) {
|
||||||
|
const savedAt = 1_000_000_000_000 + Math.floor(rnd() * 1_000_000);
|
||||||
|
// X completed after the snapshot (logged); Y is a DIFFERENT file, same
|
||||||
|
// basename + hoster, genuinely pending. Y must survive.
|
||||||
|
const jobs = [
|
||||||
|
{ id: 'X', fileName: 'clip.mp4', hoster: 'voe.sx', status: 'preview', file: 'C:/A/clip.mp4' },
|
||||||
|
{ id: 'Y', fileName: 'clip.mp4', hoster: 'voe.sx', status: 'preview', file: 'C:/B/clip.mp4' }
|
||||||
|
];
|
||||||
|
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: savedAt + 1000 + Math.floor(rnd() * 1000) }];
|
||||||
|
const { kept } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||||
|
assert.ok(kept.some(j => j.id === 'Y'), `iter ${iter}: pending Y must never be silently dropped`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('property: 2-arg legacy call NEVER removes a non-done job (the v3.3.80 canary, fuzzed)', () => {
|
||||||
|
const rnd = lcg(0xdeadbeef);
|
||||||
|
const statuses = ['preview', 'error', 'aborted', 'queued', 'skipped'];
|
||||||
|
const hosters = ['voe.sx', 'byse.sx'];
|
||||||
|
const names = ['a.mkv', 'b.mp4'];
|
||||||
|
const pick = (arr) => arr[Math.floor(rnd() * arr.length)];
|
||||||
|
|
||||||
|
for (let iter = 0; iter < 1000; iter++) {
|
||||||
|
const jobs = [];
|
||||||
|
const nJobs = 1 + Math.floor(rnd() * 5);
|
||||||
|
for (let i = 0; i < nJobs; i++) {
|
||||||
|
jobs.push({ id: `j${i}`, fileName: pick(names), hoster: pick(hosters), status: pick(statuses), file: `C:/x/${i}` });
|
||||||
|
}
|
||||||
|
const log = [];
|
||||||
|
const nLog = Math.floor(rnd() * 4);
|
||||||
|
for (let i = 0; i < nLog; i++) {
|
||||||
|
log.push({ fileName: pick(names), hoster: pick(hosters), ts: Math.floor(rnd() * 2_000_000_000_000) });
|
||||||
|
}
|
||||||
|
const { removed } = partitionRestoredJobsByLog(jobs, log);
|
||||||
|
assert.ok(removed.every(j => j.status === 'done'), `iter ${iter}: legacy 2-arg call must never drop a non-done job`);
|
||||||
|
}
|
||||||
|
});
|
||||||
@ -1,6 +1,6 @@
|
|||||||
const { test } = require('node:test');
|
const { test } = require('node:test');
|
||||||
const assert = require('node:assert');
|
const assert = require('node:assert');
|
||||||
const { partitionRestoredJobsByLog } = require('../lib/queue-dedup');
|
const { partitionRestoredJobsByLog, completedSelectionKeys } = require('../lib/queue-dedup');
|
||||||
|
|
||||||
function job(status, fileName, hoster) {
|
function job(status, fileName, hoster) {
|
||||||
return { status, fileName, hoster, file: `C:/dl/${fileName}` };
|
return { status, fileName, hoster, file: `C:/dl/${fileName}` };
|
||||||
@ -135,6 +135,45 @@ test('ts-gate: done job uploaded after snapshot is dropped via either rule', ()
|
|||||||
assert.equal(removed.length, 1);
|
assert.equal(removed.length, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ts-gate ambiguity guard: a pending same-basename file in a DIFFERENT folder is NOT lost when a sibling completes after the snapshot', () => {
|
||||||
|
// X (C:/A/clip.mp4) was uploaded after the snapshot and logged. Y is a
|
||||||
|
// genuinely-different file (C:/B/clip.mp4), same basename + hoster, still
|
||||||
|
// pending. The log records only basenames, so the ts-rule must not drop Y.
|
||||||
|
const jobs = [
|
||||||
|
{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' },
|
||||||
|
{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/B/clip.mp4' }
|
||||||
|
];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||||
|
assert.equal(removed.length, 0, 'ambiguous key -> ts-rule suppressed, no pending file lost');
|
||||||
|
assert.equal(kept.length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ts-gate ambiguity guard: the done-in-log rule still applies on an ambiguous key', () => {
|
||||||
|
// Even when the key is ambiguous, a job that is actually 'done' and in the log
|
||||||
|
// is still decluttered (pre-existing rule, unchanged by the guard).
|
||||||
|
const jobs = [
|
||||||
|
{ status: 'done', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' },
|
||||||
|
{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/B/clip.mp4' }
|
||||||
|
];
|
||||||
|
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||||
|
assert.equal(removed.length, 1);
|
||||||
|
assert.equal(removed[0].status, 'done');
|
||||||
|
assert.equal(removed[0].file, 'C:/A/clip.mp4');
|
||||||
|
assert.ok(kept.some(j => j.file === 'C:/B/clip.mp4'), 'the distinct pending file survives');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ts-gate: a unique-path ghost still drops (guard does not weaken the common case)', () => {
|
||||||
|
const jobs = [{ status: 'preview', fileName: 'clip.mp4', hoster: 'voe.sx', file: 'C:/A/clip.mp4' }];
|
||||||
|
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const { removed } = partitionRestoredJobsByLog(jobs, log, savedAt);
|
||||||
|
assert.equal(removed.length, 1, 'single job for the key -> unambiguous -> ghost dropped as before');
|
||||||
|
});
|
||||||
|
|
||||||
test('ts-gate: multi-hoster partial completion — the reported bug shape (drop only the completed hosters)', () => {
|
test('ts-gate: multi-hoster partial completion — the reported bug shape (drop only the completed hosters)', () => {
|
||||||
// One file queued to 4 hosters; close mid-upload. After the snapshot, 2 hosters
|
// One file queued to 4 hosters; close mid-upload. After the snapshot, 2 hosters
|
||||||
// completed (logged), 2 never started. On restart all 4 restore as 'preview'.
|
// completed (logged), 2 never started. On restart all 4 restore as 'preview'.
|
||||||
@ -159,3 +198,56 @@ test('ts-gate: multi-hoster partial completion — the reported bug shape (drop
|
|||||||
assert.ok(kept.some(j => j.hoster === 'vidmoly.me'));
|
assert.ok(kept.some(j => j.hoster === 'vidmoly.me'));
|
||||||
assert.ok(kept.some(j => j.hoster === 'byse.sx'));
|
assert.ok(kept.some(j => j.hoster === 'byse.sx'));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('completedSelectionKeys: a selectedFile that completed after the snapshot yields its full-path|hoster key', () => {
|
||||||
|
const selectedFiles = [{ path: 'C:/dl/done.mp4', name: 'done.mp4' }, { path: 'C:/dl/pending.mp4', name: 'pending.mp4' }];
|
||||||
|
const hosters = ['voe.sx'];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const log = [{ fileName: 'done.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt);
|
||||||
|
assert.deepEqual(keys, ['C:/dl/done.mp4|voe.sx'], 'only the completed file is seeded; pending is not');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completedSelectionKeys: per-hoster — a file done on voe but not byse only seeds the voe key', () => {
|
||||||
|
const selectedFiles = [{ path: 'C:/dl/a.mp4', name: 'a.mp4' }];
|
||||||
|
const hosters = ['voe.sx', 'byse.sx'];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const log = [{ fileName: 'a.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt);
|
||||||
|
assert.deepEqual(keys, ['C:/dl/a.mp4|voe.sx'], 'the still-pending byse upload is NOT seeded');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completedSelectionKeys: ambiguous basename across folders seeds NOTHING (no lost re-preview)', () => {
|
||||||
|
const selectedFiles = [{ path: 'C:/A/clip.mp4', name: 'clip.mp4' }, { path: 'C:/B/clip.mp4', name: 'clip.mp4' }];
|
||||||
|
const hosters = ['voe.sx'];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const log = [{ fileName: 'clip.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
const keys = completedSelectionKeys(selectedFiles, hosters, log, savedAt);
|
||||||
|
assert.deepEqual(keys, [], 'ambiguous -> neither path is suppressed, both re-preview (safe direction)');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completedSelectionKeys: an OLDER completion (pre-snapshot re-queue) is NOT seeded', () => {
|
||||||
|
const selectedFiles = [{ path: 'C:/dl/reup.mp4', name: 'reup.mp4' }];
|
||||||
|
const hosters = ['voe.sx'];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const log = [{ fileName: 'reup.mp4', hoster: 'voe.sx', ts: T('2026-06-19 11:00:00') }];
|
||||||
|
assert.deepEqual(completedSelectionKeys(selectedFiles, hosters, log, savedAt), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completedSelectionKeys: no savedAt / junk inputs -> empty (legacy + robustness)', () => {
|
||||||
|
const sf = [{ path: 'C:/dl/a.mp4', name: 'a.mp4' }];
|
||||||
|
const log = [{ fileName: 'a.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
assert.deepEqual(completedSelectionKeys(sf, ['voe.sx'], log, undefined), []);
|
||||||
|
assert.deepEqual(completedSelectionKeys(null, ['voe.sx'], log, 1), []);
|
||||||
|
assert.deepEqual(completedSelectionKeys(sf, null, log, 1), []);
|
||||||
|
assert.deepEqual(completedSelectionKeys([], [], log, 1), []);
|
||||||
|
assert.deepEqual(completedSelectionKeys([{ name: 'x' }], ['voe.sx'], log, 1), [], 'entry without path is skipped');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completedSelectionKeys: derives basename from path when name is missing', () => {
|
||||||
|
const selectedFiles = [{ path: 'C:/dl/sub/movie.mp4' }];
|
||||||
|
const hosters = ['voe.sx'];
|
||||||
|
const savedAt = T('2026-06-19 12:00:00');
|
||||||
|
const log = [{ fileName: 'movie.mp4', hoster: 'voe.sx', ts: T('2026-06-19 12:00:05') }];
|
||||||
|
assert.deepEqual(completedSelectionKeys(selectedFiles, hosters, log, savedAt), ['C:/dl/sub/movie.mp4|voe.sx']);
|
||||||
|
});
|
||||||
|
|||||||
83
tests/queue-persistence-scenario.test.js
Normal file
83
tests/queue-persistence-scenario.test.js
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
const { test } = require('node:test');
|
||||||
|
const assert = require('node:assert');
|
||||||
|
const { formatUploadLogLine, parseUploadLogLine } = require('../lib/upload-log');
|
||||||
|
const { partitionRestoredJobsByLog } = require('../lib/queue-dedup');
|
||||||
|
|
||||||
|
function makeJobs(n, hoster, status, offset = 0) {
|
||||||
|
const jobs = [];
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
const fileName = `clip_${String(i + offset).padStart(4, '0')}.mp4`;
|
||||||
|
jobs.push({ id: `j-${hoster}-${i + offset}`, file: `D:/inbox/${fileName}`, fileName, hoster, status });
|
||||||
|
}
|
||||||
|
return jobs;
|
||||||
|
}
|
||||||
|
|
||||||
|
test('user report: 300 queued, ~200 finished mid-session before a hard kill — only the finished drop', () => {
|
||||||
|
const hoster = 'byse.sx';
|
||||||
|
const snapshot = new Date(2026, 5, 19, 22, 0, 0);
|
||||||
|
const savedAt = snapshot.getTime();
|
||||||
|
const restoredJobs = makeJobs(300, hoster, 'preview');
|
||||||
|
|
||||||
|
const completionBase = new Date(2026, 5, 19, 22, 5, 0).getTime();
|
||||||
|
const logEntries = [];
|
||||||
|
for (let i = 0; i < 200; i++) {
|
||||||
|
const d = new Date(completionBase + i * 1000);
|
||||||
|
logEntries.push(parseUploadLogLine(
|
||||||
|
formatUploadLogLine(d, hoster, `https://byse.sx/d/x${i}`, `clip_${String(i).padStart(4, '0')}.mp4`)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(restoredJobs, logEntries, savedAt);
|
||||||
|
assert.equal(removed.length, 200, 'the 200 completed-after-snapshot files are dropped as ghosts');
|
||||||
|
assert.equal(kept.length, 100, 'the 100 never-finished files stay queued');
|
||||||
|
assert.ok(kept.every(j => Number(j.fileName.slice(5, 9)) >= 200), 'kept are exactly indices 200..299');
|
||||||
|
const keptNames = new Set(kept.map(j => j.fileName));
|
||||||
|
assert.ok(removed.every(j => !keptNames.has(j.fileName)));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('multi-hoster batch: per-hoster completion is independent (a file done on voe but not byse keeps byse)', () => {
|
||||||
|
const savedAt = new Date(2026, 5, 19, 22, 0, 0).getTime();
|
||||||
|
const done = new Date(2026, 5, 19, 22, 3, 0);
|
||||||
|
const jobs = [
|
||||||
|
...makeJobs(3, 'voe.sx', 'preview'),
|
||||||
|
...makeJobs(3, 'byse.sx', 'preview')
|
||||||
|
];
|
||||||
|
const logEntries = [
|
||||||
|
parseUploadLogLine(formatUploadLogLine(done, 'voe.sx', 'l', 'clip_0000.mp4')),
|
||||||
|
parseUploadLogLine(formatUploadLogLine(done, 'voe.sx', 'l', 'clip_0001.mp4')),
|
||||||
|
parseUploadLogLine(formatUploadLogLine(done, 'byse.sx', 'l', 'clip_0000.mp4'))
|
||||||
|
];
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries, savedAt);
|
||||||
|
assert.equal(removed.length, 3);
|
||||||
|
assert.ok(removed.some(j => j.hoster === 'voe.sx' && j.fileName === 'clip_0000.mp4'));
|
||||||
|
assert.ok(removed.some(j => j.hoster === 'voe.sx' && j.fileName === 'clip_0001.mp4'));
|
||||||
|
assert.ok(removed.some(j => j.hoster === 'byse.sx' && j.fileName === 'clip_0000.mp4'));
|
||||||
|
assert.ok(kept.some(j => j.hoster === 'byse.sx' && j.fileName === 'clip_0001.mp4'), 'byse clip_0001 not logged -> kept');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('clean idle close (snapshot AFTER completion) keeps an intentional re-queue of an old file', () => {
|
||||||
|
const hoster = 'voe.sx';
|
||||||
|
const yesterday = new Date(2026, 5, 18, 12, 0, 0);
|
||||||
|
const logEntries = [parseUploadLogLine(formatUploadLogLine(yesterday, hoster, 'link', 'reupload_me.mp4'))];
|
||||||
|
const savedAt = new Date(2026, 5, 19, 9, 0, 0).getTime();
|
||||||
|
const jobs = [{ id: 'r1', file: 'D:/x/reupload_me.mp4', fileName: 'reupload_me.mp4', hoster, status: 'preview' }];
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries, savedAt);
|
||||||
|
assert.equal(removed.length, 0, 'an upload older than the snapshot is a deliberate re-queue and survives');
|
||||||
|
assert.equal(kept.length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('legacy snapshot without savedAt (pre-v3.3.80 config) falls back to done-only dedup', () => {
|
||||||
|
const hoster = 'voe.sx';
|
||||||
|
const logEntries = [
|
||||||
|
parseUploadLogLine(formatUploadLogLine(new Date(2026, 5, 19, 12, 0, 0), hoster, 'l', 'done.mp4')),
|
||||||
|
parseUploadLogLine(formatUploadLogLine(new Date(2026, 5, 19, 12, 1, 0), hoster, 'l', 'preview.mp4'))
|
||||||
|
];
|
||||||
|
const jobs = [
|
||||||
|
{ id: 'a', file: 'D:/x/done.mp4', fileName: 'done.mp4', hoster, status: 'done' },
|
||||||
|
{ id: 'b', file: 'D:/x/preview.mp4', fileName: 'preview.mp4', hoster, status: 'preview' }
|
||||||
|
];
|
||||||
|
const { kept, removed } = partitionRestoredJobsByLog(jobs, logEntries);
|
||||||
|
assert.equal(removed.length, 1, 'only the done job is decluttered when no savedAt is available');
|
||||||
|
assert.equal(removed[0].id, 'a');
|
||||||
|
assert.ok(kept.some(j => j.id === 'b'), 'the preview survives the legacy path');
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue
Block a user