Delete each uploaded source as soon as its links and queue state are saved
Previously sources were only removed when the whole upload batch ended. In automation mode a batch keeps growing for hours, so hundreds of finished files stayed on disk. A file is now removed once every required hoster succeeded, its links are flushed to the upload log, the automation ledger is written and the renderer confirmed the queue save. Without that evidence the file still waits for the batch end as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
58da562e65
commit
d12a5de45b
6 files changed
+201
-6
No files matched your search
@@ -0,0 +1,63 @@
|
||||
function createEarlySourceCleanup(options) {
|
||||
if (!options || !options.cleanup || typeof options.prepare !== 'function') {
|
||||
throw new TypeError('createEarlySourceCleanup requires cleanup and prepare');
|
||||
}
|
||||
const cleanup = options.cleanup;
|
||||
const prepare = options.prepare;
|
||||
const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, options.delayMs) : 3000;
|
||||
const schedule = typeof options.schedule === 'function' ? options.schedule : (callback, ms) => setTimeout(callback, ms);
|
||||
const cancel = typeof options.cancel === 'function' ? options.cancel : (timer) => clearTimeout(timer);
|
||||
const onError = typeof options.onError === 'function' ? options.onError : () => {};
|
||||
const pending = new Set();
|
||||
let timer = null;
|
||||
let running = null;
|
||||
let closed = false;
|
||||
|
||||
function arm() {
|
||||
if (closed || timer || running || pending.size === 0) return;
|
||||
timer = schedule(() => {
|
||||
timer = null;
|
||||
running = run().finally(() => {
|
||||
running = null;
|
||||
arm();
|
||||
});
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const tokens = [...pending];
|
||||
pending.clear();
|
||||
if (closed || tokens.length === 0) return;
|
||||
let evidence = null;
|
||||
try {
|
||||
evidence = await prepare(tokens);
|
||||
} catch (error) {
|
||||
onError(error);
|
||||
return;
|
||||
}
|
||||
if (closed || !evidence || evidence.queuePersisted !== true) return;
|
||||
try {
|
||||
await cleanup.finalizeReady(tokens, evidence);
|
||||
} catch (error) {
|
||||
onError(error);
|
||||
}
|
||||
}
|
||||
|
||||
function request(token) {
|
||||
if (closed || typeof token !== 'string' || token.length === 0) return;
|
||||
pending.add(token);
|
||||
arm();
|
||||
}
|
||||
|
||||
async function close() {
|
||||
closed = true;
|
||||
if (timer) cancel(timer);
|
||||
timer = null;
|
||||
pending.clear();
|
||||
if (running) await running;
|
||||
}
|
||||
|
||||
return Object.freeze({ request, close });
|
||||
}
|
||||
|
||||
module.exports = { createEarlySourceCleanup };
|
||||
@@ -13,6 +13,7 @@ function createSourceFileCleanup(options) {
|
||||
const isEnabled = typeof options.isEnabled === 'function' ? options.isEnabled : () => false;
|
||||
const audit = typeof options.audit === 'function' ? options.audit : () => {};
|
||||
const journal = options.journal || null;
|
||||
const onReady = typeof options.onReady === 'function' ? options.onReady : null;
|
||||
const wait = typeof options.wait === 'function'
|
||||
? options.wait
|
||||
: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
@@ -107,6 +108,7 @@ function createSourceFileCleanup(options) {
|
||||
canonicalFile,
|
||||
requiredHosters: Object.freeze(requiredHosters),
|
||||
completedHosters,
|
||||
previouslyCompletedHosters: new Set(completedHosters),
|
||||
jobs,
|
||||
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
|
||||
fingerprint: null,
|
||||
@@ -153,7 +155,9 @@ function createSourceFileCleanup(options) {
|
||||
...input.requiredHosters
|
||||
]));
|
||||
for (const hoster of uniqueStrings(input.completedHosters)) {
|
||||
if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster);
|
||||
if (!existing.requiredHosters.includes(hoster)) continue;
|
||||
existing.completedHosters.add(hoster);
|
||||
existing.previouslyCompletedHosters.add(hoster);
|
||||
}
|
||||
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
||||
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
||||
@@ -198,9 +202,44 @@ function createSourceFileCleanup(options) {
|
||||
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
|
||||
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
|
||||
if (event.status === 'done') manifest.completedHosters.add(event.hoster);
|
||||
if (onReady && isReady(manifest)) {
|
||||
try { onReady(manifest.token); } catch {}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function isReady(manifest) {
|
||||
return !manifest.finalizationPromise
|
||||
&& !manifest.registrationOutcome
|
||||
&& manifest.requiredHosters.length > 0
|
||||
&& blockingStatuses(manifest).length === 0;
|
||||
}
|
||||
|
||||
function hasDurableResults(manifest, isDurable) {
|
||||
for (const hoster of manifest.requiredHosters) {
|
||||
if (manifest.previouslyCompletedHosters.has(hoster)) continue;
|
||||
const doneJobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster && job.status === 'done');
|
||||
if (!doneJobs.some((job) => isDurable(job))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function finalizeReady(tokens, evidence = {}) {
|
||||
const isDurable = typeof evidence.isDurable === 'function' ? evidence.isDurable : () => false;
|
||||
const pending = [];
|
||||
for (const token of Array.isArray(tokens) ? tokens : []) {
|
||||
const manifest = groups.get(token);
|
||||
if (!manifest || !isReady(manifest) || !hasDurableResults(manifest, isDurable)) continue;
|
||||
manifest.finalizationPromise = finalize(
|
||||
manifest,
|
||||
{ historyPersisted: true, queuePersisted: evidence.queuePersisted === true },
|
||||
{ scope: 'file' }
|
||||
);
|
||||
pending.push(manifest.finalizationPromise);
|
||||
}
|
||||
return Promise.all(pending);
|
||||
}
|
||||
|
||||
function markSkipped(jobId) {
|
||||
let changed = false;
|
||||
for (const manifest of groups.values()) {
|
||||
@@ -341,7 +380,7 @@ function createSourceFileCleanup(options) {
|
||||
}
|
||||
}
|
||||
|
||||
async function finalize(manifest, barriers) {
|
||||
async function finalize(manifest, barriers, scope = {}) {
|
||||
try {
|
||||
let enabled;
|
||||
try {
|
||||
@@ -383,7 +422,8 @@ function createSourceFileCleanup(options) {
|
||||
return sourceOutcome.outcome;
|
||||
}
|
||||
const auditPersisted = await writeAudit(manifest, 'delete-approved', {
|
||||
trigger: 'all-selected-hosters-succeeded'
|
||||
trigger: 'all-selected-hosters-succeeded',
|
||||
...scope
|
||||
});
|
||||
if (!auditPersisted) {
|
||||
await emitAudit(manifest, 'blocked', { trigger: 'audit-write-failed' });
|
||||
@@ -412,7 +452,8 @@ function createSourceFileCleanup(options) {
|
||||
await clearJournal(manifest);
|
||||
await emitAudit(manifest, 'deleted', {
|
||||
trigger: 'all-selected-hosters-succeeded',
|
||||
attempts: result.attempts
|
||||
attempts: result.attempts,
|
||||
...scope
|
||||
});
|
||||
return 'deleted';
|
||||
} catch (error) {
|
||||
@@ -449,7 +490,7 @@ function createSourceFileCleanup(options) {
|
||||
return Promise.all(pending);
|
||||
}
|
||||
|
||||
return Object.freeze({ registerGroups, settle, markSkipped, finishBatch });
|
||||
return Object.freeze({ registerGroups, settle, markSkipped, finalizeReady, finishBatch });
|
||||
}
|
||||
|
||||
module.exports = { createSourceFileCleanup };
|
||||
@@ -18,6 +18,7 @@ app.once('before-quit', () => startupDiagnostics.write('process-quit-requested')
|
||||
const ConfigStore = require('./lib/config-store');
|
||||
const UploadManager = require('./lib/upload-manager');
|
||||
const { createSourceFileCleanup } = require('./lib/source-file-cleanup');
|
||||
const { createEarlySourceCleanup } = require('./lib/early-source-cleanup');
|
||||
const SourceDeleteJournal = require('./lib/source-delete-journal');
|
||||
const { HOSTER_CONFIGS } = require('./lib/hosters');
|
||||
const VidmolyUploader = require('./lib/vidmoly-upload');
|
||||
@@ -240,6 +241,7 @@ let uploadManager = null;
|
||||
let lastSessionSummary = null;
|
||||
let sourceDeleteJournal = null;
|
||||
const pendingUploadFinalizations = new Map();
|
||||
const pendingQueuePersists = new Map();
|
||||
|
||||
async function waitForUploadManagerRelease(manager, timeoutMs = 300000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
@@ -267,6 +269,21 @@ function requestUploadFinalization(summary, preserveQueue = false) {
|
||||
safeSend('upload-batch-done', { summary, finalizationId, preserveQueue });
|
||||
});
|
||||
}
|
||||
function requestQueuePersist(timeoutMs = 15000) {
|
||||
const requestId = `queue-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
pendingQueuePersists.delete(requestId);
|
||||
resolve(false);
|
||||
}, timeoutMs);
|
||||
pendingQueuePersists.set(requestId, (value) => {
|
||||
clearTimeout(timer);
|
||||
pendingQueuePersists.delete(requestId);
|
||||
resolve(value === true);
|
||||
});
|
||||
if (!safeSend('persist-upload-queue', { requestId })) pendingQueuePersists.get(requestId)(false);
|
||||
});
|
||||
}
|
||||
const activeUploadProducerTrackers = new Set();
|
||||
const settingsImportGate = createSettingsImportGate(() => !!(uploadManager && uploadManager.running));
|
||||
|
||||
@@ -1129,6 +1146,21 @@ function _flushUploadLog() {
|
||||
return _uploadLogWrite;
|
||||
}
|
||||
|
||||
async function flushUploadLogDurably() {
|
||||
for (let attempt = 0; attempt < 5; attempt++) {
|
||||
if (_uploadLogFlushTimer) {
|
||||
clearTimeout(_uploadLogFlushTimer);
|
||||
_uploadLogFlushTimer = null;
|
||||
}
|
||||
await _flushUploadLog();
|
||||
if (!_uploadLogWriting && _uploadLogBuffer.length === 0) return true;
|
||||
}
|
||||
if (_uploadLogBuffer.length && !_uploadLogFlushTimer) {
|
||||
_uploadLogFlushTimer = setTimeout(() => { _uploadLogFlushTimer = null; _flushUploadLog(); }, 1000);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function drainRuntimeLogs() {
|
||||
await Promise.all([_flushDebugLog(), _flushUploadLog(), _rotLogWriter.drain(), _uploadAuditWriter.drain(), startupDiagnostics.drain()]);
|
||||
await _rotLogFlusher.flush('rot-log');
|
||||
@@ -2687,13 +2719,29 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
||||
// new upload; addJobs during a running batch keeps them).
|
||||
_jobLogCollector.clear();
|
||||
|
||||
const _loggedLinkJobIds = new Set();
|
||||
let earlySourceCleanup = null;
|
||||
const sourceCleanup = createSourceFileCleanup({
|
||||
fs,
|
||||
path,
|
||||
platform: process.platform,
|
||||
isEnabled: () => configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') === true,
|
||||
audit: appendSourceCleanupAudit,
|
||||
journal: sourceDeleteJournal
|
||||
journal: sourceDeleteJournal,
|
||||
onReady: (token) => { if (earlySourceCleanup) earlySourceCleanup.request(token); }
|
||||
});
|
||||
earlySourceCleanup = createEarlySourceCleanup({
|
||||
cleanup: sourceCleanup,
|
||||
prepare: async () => {
|
||||
if (configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') !== true) return null;
|
||||
const durableJobIds = new Set(_loggedLinkJobIds);
|
||||
await _thisManager._automationCompletionWriter?.flush();
|
||||
if (!(await flushUploadLogDurably())) return null;
|
||||
if (!_flushProgressToRendererNow()) return null;
|
||||
const queuePersisted = await requestQueuePersist();
|
||||
return { queuePersisted, isDurable: (job) => durableJobIds.has(job.jobId) };
|
||||
},
|
||||
onError: (error) => debugLog(`source-cleanup per file skipped: ${error && error.message ? error.message : error}`)
|
||||
});
|
||||
let sourceCleanupFingerprints;
|
||||
try {
|
||||
@@ -2734,6 +2782,19 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
||||
}, PROGRESS_BATCH_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function _flushProgressToRendererNow() {
|
||||
if (_progressFlushTimer) {
|
||||
clearTimeout(_progressFlushTimer);
|
||||
_progressFlushTimer = null;
|
||||
}
|
||||
if (!mainWindow || mainWindow.isDestroyed()) return false;
|
||||
const batch = _progressTerminalQueue.splice(0);
|
||||
for (const value of _progressByJob.values()) batch.push(value);
|
||||
_progressByJob.clear();
|
||||
if (batch.length) safeSend('upload-progress-batch', batch);
|
||||
return true;
|
||||
}
|
||||
|
||||
function _queueProgressForRenderer(data) {
|
||||
const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped';
|
||||
if (isTerminal) {
|
||||
@@ -2778,6 +2839,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
||||
if (link) {
|
||||
if (shouldLogHosterToFile(data.hoster)) {
|
||||
appendUploadLog(data.hoster || '', link, data.fileName || '');
|
||||
if (data.jobId) _loggedLinkJobIds.add(data.jobId);
|
||||
} else {
|
||||
debugLog(`upload-log: skip ${data.fileName} @ ${data.hoster} (logToFile disabled for hoster)`);
|
||||
}
|
||||
@@ -2932,6 +2994,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
||||
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
|
||||
}
|
||||
if (!finalizationPersisted) debugLog('upload finalization blocked: queue or automation completion evidence was not persisted');
|
||||
await earlySourceCleanup.close();
|
||||
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted: finalizationPersisted });
|
||||
_producerTracker.finish();
|
||||
|
||||
@@ -3745,6 +3808,13 @@ ipcMain.handle('save-pending-queue', async (_event, pendingQueue) => {
|
||||
return true;
|
||||
});
|
||||
|
||||
ipcMain.handle('complete-queue-persist', async (_event, payload) => {
|
||||
const complete = payload && pendingQueuePersists.get(payload.requestId);
|
||||
if (!complete) return false;
|
||||
complete(payload.ok === true);
|
||||
return true;
|
||||
});
|
||||
|
||||
ipcMain.handle('complete-upload-finalization', async (_event, payload) => {
|
||||
const finalizationId = payload && payload.finalizationId;
|
||||
const pending = finalizationId && pendingUploadFinalizations.get(finalizationId);
|
||||
|
||||
@@ -90,6 +90,7 @@ contextBridge.exposeInMainWorld('api', {
|
||||
cancelSelectedJobs: (jobIds) => ipcRenderer.invoke('cancel-selected-jobs', jobIds),
|
||||
addJobsToBatch: (payload) => ipcRenderer.invoke('add-jobs-to-batch', payload),
|
||||
completeUploadFinalization: (payload) => ipcRenderer.invoke('complete-upload-finalization', payload),
|
||||
completeQueuePersist: (payload) => ipcRenderer.invoke('complete-queue-persist', payload),
|
||||
finishAfterActive: () => ipcRenderer.invoke('finish-after-active'),
|
||||
runHealthCheck: (payload) => ipcRenderer.invoke('run-health-check', payload),
|
||||
onHealthCheckResult: (callback) => {
|
||||
@@ -184,6 +185,9 @@ contextBridge.exposeInMainWorld('api', {
|
||||
onUploadBatchDone: (callback) => {
|
||||
ipcRenderer.on('upload-batch-done', (_event, data) => callback(data));
|
||||
},
|
||||
onPersistUploadQueue: (callback) => {
|
||||
ipcRenderer.on('persist-upload-queue', (_event, data) => callback(data));
|
||||
},
|
||||
onUploadStats: (callback) => {
|
||||
ipcRenderer.on('upload-stats', (_event, data) => callback(data));
|
||||
},
|
||||
@@ -223,6 +227,7 @@ contextBridge.exposeInMainWorld('api', {
|
||||
ipcRenderer.removeAllListeners('folder-monitor:renderer-generation');
|
||||
ipcRenderer.removeAllListeners('upload-progress');
|
||||
ipcRenderer.removeAllListeners('upload-batch-done');
|
||||
ipcRenderer.removeAllListeners('persist-upload-queue');
|
||||
ipcRenderer.removeAllListeners('upload-stats');
|
||||
ipcRenderer.removeAllListeners('app:update-available');
|
||||
ipcRenderer.removeAllListeners('app:update-progress');
|
||||
|
||||
@@ -1573,6 +1573,21 @@ async function init() {
|
||||
});
|
||||
}
|
||||
});
|
||||
if (window.api.onPersistUploadQueue) {
|
||||
window.api.onPersistUploadQueue(async (data) => {
|
||||
const requestId = data && data.requestId;
|
||||
if (!requestId) return;
|
||||
let ok = false;
|
||||
if (closePreparationState === 'open') {
|
||||
try {
|
||||
queuePersistThrottle.cancel();
|
||||
await persistQueueStateNow();
|
||||
ok = true;
|
||||
} catch {}
|
||||
}
|
||||
try { await window.api.completeQueuePersist({ requestId, ok }); } catch {}
|
||||
});
|
||||
}
|
||||
window.api.onUploadStats((data) => {
|
||||
handleStats(data);
|
||||
});
|
||||
|
||||
@@ -29,6 +29,7 @@ const sourceFiles = [
|
||||
'lib/diagnostics-agent.js',
|
||||
'lib/diagnostics-collectors.js',
|
||||
'lib/doodstream-upload.js',
|
||||
'lib/early-source-cleanup.js',
|
||||
'lib/file-probe.js',
|
||||
'lib/file-discovery.js',
|
||||
'lib/folder-monitor.js',
|
||||
|
||||
Reference in new issue
Block a user