Delete each uploaded source as soon as its links and queue state are saved

Previously sources were only removed when the whole upload batch ended. In
automation mode a batch keeps growing for hours, so hundreds of finished
files stayed on disk. A file is now removed once every required hoster
succeeded, its links are flushed to the upload log, the automation ledger
is written and the renderer confirmed the queue save. Without that evidence
the file still waits for the batch end as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
SucukdeluxeandClaude Opus 5.5 committed 2026-10-21 02:59:54 +02:00
1 parent 58da562e65
commit d12a5de45b
6 files changed
+201 -6

No files matched your search

+63
View File
@@ -0,0 +1,63 @@
function createEarlySourceCleanup(options) {
if (!options || !options.cleanup || typeof options.prepare !== 'function') {
throw new TypeError('createEarlySourceCleanup requires cleanup and prepare');
}
const cleanup = options.cleanup;
const prepare = options.prepare;
const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, options.delayMs) : 3000;
const schedule = typeof options.schedule === 'function' ? options.schedule : (callback, ms) => setTimeout(callback, ms);
const cancel = typeof options.cancel === 'function' ? options.cancel : (timer) => clearTimeout(timer);
const onError = typeof options.onError === 'function' ? options.onError : () => {};
const pending = new Set();
let timer = null;
let running = null;
let closed = false;
function arm() {
if (closed || timer || running || pending.size === 0) return;
timer = schedule(() => {
timer = null;
running = run().finally(() => {
running = null;
arm();
});
}, delayMs);
}
async function run() {
const tokens = [...pending];
pending.clear();
if (closed || tokens.length === 0) return;
let evidence = null;
try {
evidence = await prepare(tokens);
} catch (error) {
onError(error);
return;
}
if (closed || !evidence || evidence.queuePersisted !== true) return;
try {
await cleanup.finalizeReady(tokens, evidence);
} catch (error) {
onError(error);
}
}
function request(token) {
if (closed || typeof token !== 'string' || token.length === 0) return;
pending.add(token);
arm();
}
async function close() {
closed = true;
if (timer) cancel(timer);
timer = null;
pending.clear();
if (running) await running;
}
return Object.freeze({ request, close });
}
module.exports = { createEarlySourceCleanup };
+46 -5
View File
@@ -13,6 +13,7 @@ function createSourceFileCleanup(options) {
const isEnabled = typeof options.isEnabled === 'function' ? options.isEnabled : () => false; const isEnabled = typeof options.isEnabled === 'function' ? options.isEnabled : () => false;
const audit = typeof options.audit === 'function' ? options.audit : () => {}; const audit = typeof options.audit === 'function' ? options.audit : () => {};
const journal = options.journal || null; const journal = options.journal || null;
const onReady = typeof options.onReady === 'function' ? options.onReady : null;
const wait = typeof options.wait === 'function' const wait = typeof options.wait === 'function'
? options.wait ? options.wait
: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); : (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
@@ -107,6 +108,7 @@ function createSourceFileCleanup(options) {
canonicalFile, canonicalFile,
requiredHosters: Object.freeze(requiredHosters), requiredHosters: Object.freeze(requiredHosters),
completedHosters, completedHosters,
previouslyCompletedHosters: new Set(completedHosters),
jobs, jobs,
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null, suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
fingerprint: null, fingerprint: null,
@@ -153,7 +155,9 @@ function createSourceFileCleanup(options) {
...input.requiredHosters ...input.requiredHosters
])); ]));
for (const hoster of uniqueStrings(input.completedHosters)) { for (const hoster of uniqueStrings(input.completedHosters)) {
if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster); if (!existing.requiredHosters.includes(hoster)) continue;
existing.completedHosters.add(hoster);
existing.previouslyCompletedHosters.add(hoster);
} }
for (const job of Array.isArray(input.jobs) ? input.jobs : []) { for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue; if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
@@ -198,9 +202,44 @@ function createSourceFileCleanup(options) {
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false; if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status })); manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
if (event.status === 'done') manifest.completedHosters.add(event.hoster); if (event.status === 'done') manifest.completedHosters.add(event.hoster);
if (onReady && isReady(manifest)) {
try { onReady(manifest.token); } catch {}
}
return true; return true;
} }
function isReady(manifest) {
return !manifest.finalizationPromise
&& !manifest.registrationOutcome
&& manifest.requiredHosters.length > 0
&& blockingStatuses(manifest).length === 0;
}
function hasDurableResults(manifest, isDurable) {
for (const hoster of manifest.requiredHosters) {
if (manifest.previouslyCompletedHosters.has(hoster)) continue;
const doneJobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster && job.status === 'done');
if (!doneJobs.some((job) => isDurable(job))) return false;
}
return true;
}
function finalizeReady(tokens, evidence = {}) {
const isDurable = typeof evidence.isDurable === 'function' ? evidence.isDurable : () => false;
const pending = [];
for (const token of Array.isArray(tokens) ? tokens : []) {
const manifest = groups.get(token);
if (!manifest || !isReady(manifest) || !hasDurableResults(manifest, isDurable)) continue;
manifest.finalizationPromise = finalize(
manifest,
{ historyPersisted: true, queuePersisted: evidence.queuePersisted === true },
{ scope: 'file' }
);
pending.push(manifest.finalizationPromise);
}
return Promise.all(pending);
}
function markSkipped(jobId) { function markSkipped(jobId) {
let changed = false; let changed = false;
for (const manifest of groups.values()) { for (const manifest of groups.values()) {
@@ -341,7 +380,7 @@ function createSourceFileCleanup(options) {
} }
} }
async function finalize(manifest, barriers) { async function finalize(manifest, barriers, scope = {}) {
try { try {
let enabled; let enabled;
try { try {
@@ -383,7 +422,8 @@ function createSourceFileCleanup(options) {
return sourceOutcome.outcome; return sourceOutcome.outcome;
} }
const auditPersisted = await writeAudit(manifest, 'delete-approved', { const auditPersisted = await writeAudit(manifest, 'delete-approved', {
trigger: 'all-selected-hosters-succeeded' trigger: 'all-selected-hosters-succeeded',
...scope
}); });
if (!auditPersisted) { if (!auditPersisted) {
await emitAudit(manifest, 'blocked', { trigger: 'audit-write-failed' }); await emitAudit(manifest, 'blocked', { trigger: 'audit-write-failed' });
@@ -412,7 +452,8 @@ function createSourceFileCleanup(options) {
await clearJournal(manifest); await clearJournal(manifest);
await emitAudit(manifest, 'deleted', { await emitAudit(manifest, 'deleted', {
trigger: 'all-selected-hosters-succeeded', trigger: 'all-selected-hosters-succeeded',
attempts: result.attempts attempts: result.attempts,
...scope
}); });
return 'deleted'; return 'deleted';
} catch (error) { } catch (error) {
@@ -449,7 +490,7 @@ function createSourceFileCleanup(options) {
return Promise.all(pending); return Promise.all(pending);
} }
return Object.freeze({ registerGroups, settle, markSkipped, finishBatch }); return Object.freeze({ registerGroups, settle, markSkipped, finalizeReady, finishBatch });
} }
module.exports = { createSourceFileCleanup }; module.exports = { createSourceFileCleanup };
+71 -1
View File
@@ -18,6 +18,7 @@ app.once('before-quit', () => startupDiagnostics.write('process-quit-requested')
const ConfigStore = require('./lib/config-store'); const ConfigStore = require('./lib/config-store');
const UploadManager = require('./lib/upload-manager'); const UploadManager = require('./lib/upload-manager');
const { createSourceFileCleanup } = require('./lib/source-file-cleanup'); const { createSourceFileCleanup } = require('./lib/source-file-cleanup');
const { createEarlySourceCleanup } = require('./lib/early-source-cleanup');
const SourceDeleteJournal = require('./lib/source-delete-journal'); const SourceDeleteJournal = require('./lib/source-delete-journal');
const { HOSTER_CONFIGS } = require('./lib/hosters'); const { HOSTER_CONFIGS } = require('./lib/hosters');
const VidmolyUploader = require('./lib/vidmoly-upload'); const VidmolyUploader = require('./lib/vidmoly-upload');
@@ -240,6 +241,7 @@ let uploadManager = null;
let lastSessionSummary = null; let lastSessionSummary = null;
let sourceDeleteJournal = null; let sourceDeleteJournal = null;
const pendingUploadFinalizations = new Map(); const pendingUploadFinalizations = new Map();
const pendingQueuePersists = new Map();
async function waitForUploadManagerRelease(manager, timeoutMs = 300000) { async function waitForUploadManagerRelease(manager, timeoutMs = 300000) {
const deadline = Date.now() + timeoutMs; const deadline = Date.now() + timeoutMs;
@@ -267,6 +269,21 @@ function requestUploadFinalization(summary, preserveQueue = false) {
safeSend('upload-batch-done', { summary, finalizationId, preserveQueue }); safeSend('upload-batch-done', { summary, finalizationId, preserveQueue });
}); });
} }
function requestQueuePersist(timeoutMs = 15000) {
const requestId = `queue-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
return new Promise((resolve) => {
const timer = setTimeout(() => {
pendingQueuePersists.delete(requestId);
resolve(false);
}, timeoutMs);
pendingQueuePersists.set(requestId, (value) => {
clearTimeout(timer);
pendingQueuePersists.delete(requestId);
resolve(value === true);
});
if (!safeSend('persist-upload-queue', { requestId })) pendingQueuePersists.get(requestId)(false);
});
}
const activeUploadProducerTrackers = new Set(); const activeUploadProducerTrackers = new Set();
const settingsImportGate = createSettingsImportGate(() => !!(uploadManager && uploadManager.running)); const settingsImportGate = createSettingsImportGate(() => !!(uploadManager && uploadManager.running));
@@ -1129,6 +1146,21 @@ function _flushUploadLog() {
return _uploadLogWrite; return _uploadLogWrite;
} }
async function flushUploadLogDurably() {
for (let attempt = 0; attempt < 5; attempt++) {
if (_uploadLogFlushTimer) {
clearTimeout(_uploadLogFlushTimer);
_uploadLogFlushTimer = null;
}
await _flushUploadLog();
if (!_uploadLogWriting && _uploadLogBuffer.length === 0) return true;
}
if (_uploadLogBuffer.length && !_uploadLogFlushTimer) {
_uploadLogFlushTimer = setTimeout(() => { _uploadLogFlushTimer = null; _flushUploadLog(); }, 1000);
}
return false;
}
async function drainRuntimeLogs() { async function drainRuntimeLogs() {
await Promise.all([_flushDebugLog(), _flushUploadLog(), _rotLogWriter.drain(), _uploadAuditWriter.drain(), startupDiagnostics.drain()]); await Promise.all([_flushDebugLog(), _flushUploadLog(), _rotLogWriter.drain(), _uploadAuditWriter.drain(), startupDiagnostics.drain()]);
await _rotLogFlusher.flush('rot-log'); await _rotLogFlusher.flush('rot-log');
@@ -2687,13 +2719,29 @@ ipcMain.handle('start-upload', async (_event, payload) => {
// new upload; addJobs during a running batch keeps them). // new upload; addJobs during a running batch keeps them).
_jobLogCollector.clear(); _jobLogCollector.clear();
const _loggedLinkJobIds = new Set();
let earlySourceCleanup = null;
const sourceCleanup = createSourceFileCleanup({ const sourceCleanup = createSourceFileCleanup({
fs, fs,
path, path,
platform: process.platform, platform: process.platform,
isEnabled: () => configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') === true, isEnabled: () => configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') === true,
audit: appendSourceCleanupAudit, audit: appendSourceCleanupAudit,
journal: sourceDeleteJournal journal: sourceDeleteJournal,
onReady: (token) => { if (earlySourceCleanup) earlySourceCleanup.request(token); }
});
earlySourceCleanup = createEarlySourceCleanup({
cleanup: sourceCleanup,
prepare: async () => {
if (configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') !== true) return null;
const durableJobIds = new Set(_loggedLinkJobIds);
await _thisManager._automationCompletionWriter?.flush();
if (!(await flushUploadLogDurably())) return null;
if (!_flushProgressToRendererNow()) return null;
const queuePersisted = await requestQueuePersist();
return { queuePersisted, isDurable: (job) => durableJobIds.has(job.jobId) };
},
onError: (error) => debugLog(`source-cleanup per file skipped: ${error && error.message ? error.message : error}`)
}); });
let sourceCleanupFingerprints; let sourceCleanupFingerprints;
try { try {
@@ -2734,6 +2782,19 @@ ipcMain.handle('start-upload', async (_event, payload) => {
}, PROGRESS_BATCH_INTERVAL_MS); }, PROGRESS_BATCH_INTERVAL_MS);
} }
function _flushProgressToRendererNow() {
if (_progressFlushTimer) {
clearTimeout(_progressFlushTimer);
_progressFlushTimer = null;
}
if (!mainWindow || mainWindow.isDestroyed()) return false;
const batch = _progressTerminalQueue.splice(0);
for (const value of _progressByJob.values()) batch.push(value);
_progressByJob.clear();
if (batch.length) safeSend('upload-progress-batch', batch);
return true;
}
function _queueProgressForRenderer(data) { function _queueProgressForRenderer(data) {
const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped'; const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped';
if (isTerminal) { if (isTerminal) {
@@ -2778,6 +2839,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
if (link) { if (link) {
if (shouldLogHosterToFile(data.hoster)) { if (shouldLogHosterToFile(data.hoster)) {
appendUploadLog(data.hoster || '', link, data.fileName || ''); appendUploadLog(data.hoster || '', link, data.fileName || '');
if (data.jobId) _loggedLinkJobIds.add(data.jobId);
} else { } else {
debugLog(`upload-log: skip ${data.fileName} @ ${data.hoster} (logToFile disabled for hoster)`); debugLog(`upload-log: skip ${data.fileName} @ ${data.hoster} (logToFile disabled for hoster)`);
} }
@@ -2932,6 +2994,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); } try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
} }
if (!finalizationPersisted) debugLog('upload finalization blocked: queue or automation completion evidence was not persisted'); if (!finalizationPersisted) debugLog('upload finalization blocked: queue or automation completion evidence was not persisted');
await earlySourceCleanup.close();
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted: finalizationPersisted }); await sourceCleanup.finishBatch({ historyPersisted, queuePersisted: finalizationPersisted });
_producerTracker.finish(); _producerTracker.finish();
@@ -3745,6 +3808,13 @@ ipcMain.handle('save-pending-queue', async (_event, pendingQueue) => {
return true; return true;
}); });
ipcMain.handle('complete-queue-persist', async (_event, payload) => {
const complete = payload && pendingQueuePersists.get(payload.requestId);
if (!complete) return false;
complete(payload.ok === true);
return true;
});
ipcMain.handle('complete-upload-finalization', async (_event, payload) => { ipcMain.handle('complete-upload-finalization', async (_event, payload) => {
const finalizationId = payload && payload.finalizationId; const finalizationId = payload && payload.finalizationId;
const pending = finalizationId && pendingUploadFinalizations.get(finalizationId); const pending = finalizationId && pendingUploadFinalizations.get(finalizationId);
+5
View File
@@ -90,6 +90,7 @@ contextBridge.exposeInMainWorld('api', {
cancelSelectedJobs: (jobIds) => ipcRenderer.invoke('cancel-selected-jobs', jobIds), cancelSelectedJobs: (jobIds) => ipcRenderer.invoke('cancel-selected-jobs', jobIds),
addJobsToBatch: (payload) => ipcRenderer.invoke('add-jobs-to-batch', payload), addJobsToBatch: (payload) => ipcRenderer.invoke('add-jobs-to-batch', payload),
completeUploadFinalization: (payload) => ipcRenderer.invoke('complete-upload-finalization', payload), completeUploadFinalization: (payload) => ipcRenderer.invoke('complete-upload-finalization', payload),
completeQueuePersist: (payload) => ipcRenderer.invoke('complete-queue-persist', payload),
finishAfterActive: () => ipcRenderer.invoke('finish-after-active'), finishAfterActive: () => ipcRenderer.invoke('finish-after-active'),
runHealthCheck: (payload) => ipcRenderer.invoke('run-health-check', payload), runHealthCheck: (payload) => ipcRenderer.invoke('run-health-check', payload),
onHealthCheckResult: (callback) => { onHealthCheckResult: (callback) => {
@@ -184,6 +185,9 @@ contextBridge.exposeInMainWorld('api', {
onUploadBatchDone: (callback) => { onUploadBatchDone: (callback) => {
ipcRenderer.on('upload-batch-done', (_event, data) => callback(data)); ipcRenderer.on('upload-batch-done', (_event, data) => callback(data));
}, },
onPersistUploadQueue: (callback) => {
ipcRenderer.on('persist-upload-queue', (_event, data) => callback(data));
},
onUploadStats: (callback) => { onUploadStats: (callback) => {
ipcRenderer.on('upload-stats', (_event, data) => callback(data)); ipcRenderer.on('upload-stats', (_event, data) => callback(data));
}, },
@@ -223,6 +227,7 @@ contextBridge.exposeInMainWorld('api', {
ipcRenderer.removeAllListeners('folder-monitor:renderer-generation'); ipcRenderer.removeAllListeners('folder-monitor:renderer-generation');
ipcRenderer.removeAllListeners('upload-progress'); ipcRenderer.removeAllListeners('upload-progress');
ipcRenderer.removeAllListeners('upload-batch-done'); ipcRenderer.removeAllListeners('upload-batch-done');
ipcRenderer.removeAllListeners('persist-upload-queue');
ipcRenderer.removeAllListeners('upload-stats'); ipcRenderer.removeAllListeners('upload-stats');
ipcRenderer.removeAllListeners('app:update-available'); ipcRenderer.removeAllListeners('app:update-available');
ipcRenderer.removeAllListeners('app:update-progress'); ipcRenderer.removeAllListeners('app:update-progress');
+15
View File
@@ -1573,6 +1573,21 @@ async function init() {
}); });
} }
}); });
if (window.api.onPersistUploadQueue) {
window.api.onPersistUploadQueue(async (data) => {
const requestId = data && data.requestId;
if (!requestId) return;
let ok = false;
if (closePreparationState === 'open') {
try {
queuePersistThrottle.cancel();
await persistQueueStateNow();
ok = true;
} catch {}
}
try { await window.api.completeQueuePersist({ requestId, ok }); } catch {}
});
}
window.api.onUploadStats((data) => { window.api.onUploadStats((data) => {
handleStats(data); handleStats(data);
}); });
+1
View File
@@ -29,6 +29,7 @@ const sourceFiles = [
'lib/diagnostics-agent.js', 'lib/diagnostics-agent.js',
'lib/diagnostics-collectors.js', 'lib/diagnostics-collectors.js',
'lib/doodstream-upload.js', 'lib/doodstream-upload.js',
'lib/early-source-cleanup.js',
'lib/file-probe.js', 'lib/file-probe.js',
'lib/file-discovery.js', 'lib/file-discovery.js',
'lib/folder-monitor.js', 'lib/folder-monitor.js',