diff --git a/lib/early-source-cleanup.js b/lib/early-source-cleanup.js new file mode 100644 index 0000000..b2d4d1e --- /dev/null +++ b/lib/early-source-cleanup.js @@ -0,0 +1,63 @@ +function createEarlySourceCleanup(options) { + if (!options || !options.cleanup || typeof options.prepare !== 'function') { + throw new TypeError('createEarlySourceCleanup requires cleanup and prepare'); + } + const cleanup = options.cleanup; + const prepare = options.prepare; + const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, options.delayMs) : 3000; + const schedule = typeof options.schedule === 'function' ? options.schedule : (callback, ms) => setTimeout(callback, ms); + const cancel = typeof options.cancel === 'function' ? options.cancel : (timer) => clearTimeout(timer); + const onError = typeof options.onError === 'function' ? options.onError : () => {}; + const pending = new Set(); + let timer = null; + let running = null; + let closed = false; + + function arm() { + if (closed || timer || running || pending.size === 0) return; + timer = schedule(() => { + timer = null; + running = run().finally(() => { + running = null; + arm(); + }); + }, delayMs); + } + + async function run() { + const tokens = [...pending]; + pending.clear(); + if (closed || tokens.length === 0) return; + let evidence = null; + try { + evidence = await prepare(tokens); + } catch (error) { + onError(error); + return; + } + if (closed || !evidence || evidence.queuePersisted !== true) return; + try { + await cleanup.finalizeReady(tokens, evidence); + } catch (error) { + onError(error); + } + } + + function request(token) { + if (closed || typeof token !== 'string' || token.length === 0) return; + pending.add(token); + arm(); + } + + async function close() { + closed = true; + if (timer) cancel(timer); + timer = null; + pending.clear(); + if (running) await running; + } + + return Object.freeze({ request, close }); +} + +module.exports = { createEarlySourceCleanup }; diff --git a/lib/source-file-cleanup.js b/lib/source-file-cleanup.js index e8dc7e0..9847a8b 100644 --- a/lib/source-file-cleanup.js +++ b/lib/source-file-cleanup.js @@ -13,6 +13,7 @@ function createSourceFileCleanup(options) { const isEnabled = typeof options.isEnabled === 'function' ? options.isEnabled : () => false; const audit = typeof options.audit === 'function' ? options.audit : () => {}; const journal = options.journal || null; + const onReady = typeof options.onReady === 'function' ? options.onReady : null; const wait = typeof options.wait === 'function' ? options.wait : (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); @@ -107,6 +108,7 @@ function createSourceFileCleanup(options) { canonicalFile, requiredHosters: Object.freeze(requiredHosters), completedHosters, + previouslyCompletedHosters: new Set(completedHosters), jobs, suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null, fingerprint: null, @@ -153,7 +155,9 @@ function createSourceFileCleanup(options) { ...input.requiredHosters ])); for (const hoster of uniqueStrings(input.completedHosters)) { - if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster); + if (!existing.requiredHosters.includes(hoster)) continue; + existing.completedHosters.add(hoster); + existing.previouslyCompletedHosters.add(hoster); } for (const job of Array.isArray(input.jobs) ? input.jobs : []) { if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue; @@ -198,9 +202,44 @@ function createSourceFileCleanup(options) { if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false; manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status })); if (event.status === 'done') manifest.completedHosters.add(event.hoster); + if (onReady && isReady(manifest)) { + try { onReady(manifest.token); } catch {} + } return true; } + function isReady(manifest) { + return !manifest.finalizationPromise + && !manifest.registrationOutcome + && manifest.requiredHosters.length > 0 + && blockingStatuses(manifest).length === 0; + } + + function hasDurableResults(manifest, isDurable) { + for (const hoster of manifest.requiredHosters) { + if (manifest.previouslyCompletedHosters.has(hoster)) continue; + const doneJobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster && job.status === 'done'); + if (!doneJobs.some((job) => isDurable(job))) return false; + } + return true; + } + + function finalizeReady(tokens, evidence = {}) { + const isDurable = typeof evidence.isDurable === 'function' ? evidence.isDurable : () => false; + const pending = []; + for (const token of Array.isArray(tokens) ? tokens : []) { + const manifest = groups.get(token); + if (!manifest || !isReady(manifest) || !hasDurableResults(manifest, isDurable)) continue; + manifest.finalizationPromise = finalize( + manifest, + { historyPersisted: true, queuePersisted: evidence.queuePersisted === true }, + { scope: 'file' } + ); + pending.push(manifest.finalizationPromise); + } + return Promise.all(pending); + } + function markSkipped(jobId) { let changed = false; for (const manifest of groups.values()) { @@ -341,7 +380,7 @@ function createSourceFileCleanup(options) { } } - async function finalize(manifest, barriers) { + async function finalize(manifest, barriers, scope = {}) { try { let enabled; try { @@ -383,7 +422,8 @@ function createSourceFileCleanup(options) { return sourceOutcome.outcome; } const auditPersisted = await writeAudit(manifest, 'delete-approved', { - trigger: 'all-selected-hosters-succeeded' + trigger: 'all-selected-hosters-succeeded', + ...scope }); if (!auditPersisted) { await emitAudit(manifest, 'blocked', { trigger: 'audit-write-failed' }); @@ -412,7 +452,8 @@ function createSourceFileCleanup(options) { await clearJournal(manifest); await emitAudit(manifest, 'deleted', { trigger: 'all-selected-hosters-succeeded', - attempts: result.attempts + attempts: result.attempts, + ...scope }); return 'deleted'; } catch (error) { @@ -449,7 +490,7 @@ function createSourceFileCleanup(options) { return Promise.all(pending); } - return Object.freeze({ registerGroups, settle, markSkipped, finishBatch }); + return Object.freeze({ registerGroups, settle, markSkipped, finalizeReady, finishBatch }); } module.exports = { createSourceFileCleanup }; diff --git a/main.js b/main.js index 9f9b15e..7ae5bbd 100644 --- a/main.js +++ b/main.js @@ -18,6 +18,7 @@ app.once('before-quit', () => startupDiagnostics.write('process-quit-requested') const ConfigStore = require('./lib/config-store'); const UploadManager = require('./lib/upload-manager'); const { createSourceFileCleanup } = require('./lib/source-file-cleanup'); +const { createEarlySourceCleanup } = require('./lib/early-source-cleanup'); const SourceDeleteJournal = require('./lib/source-delete-journal'); const { HOSTER_CONFIGS } = require('./lib/hosters'); const VidmolyUploader = require('./lib/vidmoly-upload'); @@ -240,6 +241,7 @@ let uploadManager = null; let lastSessionSummary = null; let sourceDeleteJournal = null; const pendingUploadFinalizations = new Map(); +const pendingQueuePersists = new Map(); async function waitForUploadManagerRelease(manager, timeoutMs = 300000) { const deadline = Date.now() + timeoutMs; @@ -267,6 +269,21 @@ function requestUploadFinalization(summary, preserveQueue = false) { safeSend('upload-batch-done', { summary, finalizationId, preserveQueue }); }); } +function requestQueuePersist(timeoutMs = 15000) { + const requestId = `queue-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + return new Promise((resolve) => { + const timer = setTimeout(() => { + pendingQueuePersists.delete(requestId); + resolve(false); + }, timeoutMs); + pendingQueuePersists.set(requestId, (value) => { + clearTimeout(timer); + pendingQueuePersists.delete(requestId); + resolve(value === true); + }); + if (!safeSend('persist-upload-queue', { requestId })) pendingQueuePersists.get(requestId)(false); + }); +} const activeUploadProducerTrackers = new Set(); const settingsImportGate = createSettingsImportGate(() => !!(uploadManager && uploadManager.running)); @@ -1129,6 +1146,21 @@ function _flushUploadLog() { return _uploadLogWrite; } +async function flushUploadLogDurably() { + for (let attempt = 0; attempt < 5; attempt++) { + if (_uploadLogFlushTimer) { + clearTimeout(_uploadLogFlushTimer); + _uploadLogFlushTimer = null; + } + await _flushUploadLog(); + if (!_uploadLogWriting && _uploadLogBuffer.length === 0) return true; + } + if (_uploadLogBuffer.length && !_uploadLogFlushTimer) { + _uploadLogFlushTimer = setTimeout(() => { _uploadLogFlushTimer = null; _flushUploadLog(); }, 1000); + } + return false; +} + async function drainRuntimeLogs() { await Promise.all([_flushDebugLog(), _flushUploadLog(), _rotLogWriter.drain(), _uploadAuditWriter.drain(), startupDiagnostics.drain()]); await _rotLogFlusher.flush('rot-log'); @@ -2687,13 +2719,29 @@ ipcMain.handle('start-upload', async (_event, payload) => { // new upload; addJobs during a running batch keeps them). _jobLogCollector.clear(); + const _loggedLinkJobIds = new Set(); + let earlySourceCleanup = null; const sourceCleanup = createSourceFileCleanup({ fs, path, platform: process.platform, isEnabled: () => configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') === true, audit: appendSourceCleanupAudit, - journal: sourceDeleteJournal + journal: sourceDeleteJournal, + onReady: (token) => { if (earlySourceCleanup) earlySourceCleanup.request(token); } + }); + earlySourceCleanup = createEarlySourceCleanup({ + cleanup: sourceCleanup, + prepare: async () => { + if (configStore.getGlobalSetting('deleteSourceAfterSuccessfulUpload') !== true) return null; + const durableJobIds = new Set(_loggedLinkJobIds); + await _thisManager._automationCompletionWriter?.flush(); + if (!(await flushUploadLogDurably())) return null; + if (!_flushProgressToRendererNow()) return null; + const queuePersisted = await requestQueuePersist(); + return { queuePersisted, isDurable: (job) => durableJobIds.has(job.jobId) }; + }, + onError: (error) => debugLog(`source-cleanup per file skipped: ${error && error.message ? error.message : error}`) }); let sourceCleanupFingerprints; try { @@ -2734,6 +2782,19 @@ ipcMain.handle('start-upload', async (_event, payload) => { }, PROGRESS_BATCH_INTERVAL_MS); } + function _flushProgressToRendererNow() { + if (_progressFlushTimer) { + clearTimeout(_progressFlushTimer); + _progressFlushTimer = null; + } + if (!mainWindow || mainWindow.isDestroyed()) return false; + const batch = _progressTerminalQueue.splice(0); + for (const value of _progressByJob.values()) batch.push(value); + _progressByJob.clear(); + if (batch.length) safeSend('upload-progress-batch', batch); + return true; + } + function _queueProgressForRenderer(data) { const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped'; if (isTerminal) { @@ -2778,6 +2839,7 @@ ipcMain.handle('start-upload', async (_event, payload) => { if (link) { if (shouldLogHosterToFile(data.hoster)) { appendUploadLog(data.hoster || '', link, data.fileName || ''); + if (data.jobId) _loggedLinkJobIds.add(data.jobId); } else { debugLog(`upload-log: skip ${data.fileName} @ ${data.hoster} (logToFile disabled for hoster)`); } @@ -2932,6 +2994,7 @@ ipcMain.handle('start-upload', async (_event, payload) => { try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); } } if (!finalizationPersisted) debugLog('upload finalization blocked: queue or automation completion evidence was not persisted'); + await earlySourceCleanup.close(); await sourceCleanup.finishBatch({ historyPersisted, queuePersisted: finalizationPersisted }); _producerTracker.finish(); @@ -3745,6 +3808,13 @@ ipcMain.handle('save-pending-queue', async (_event, pendingQueue) => { return true; }); +ipcMain.handle('complete-queue-persist', async (_event, payload) => { + const complete = payload && pendingQueuePersists.get(payload.requestId); + if (!complete) return false; + complete(payload.ok === true); + return true; +}); + ipcMain.handle('complete-upload-finalization', async (_event, payload) => { const finalizationId = payload && payload.finalizationId; const pending = finalizationId && pendingUploadFinalizations.get(finalizationId); diff --git a/preload.js b/preload.js index 752e2eb..c405025 100644 --- a/preload.js +++ b/preload.js @@ -90,6 +90,7 @@ contextBridge.exposeInMainWorld('api', { cancelSelectedJobs: (jobIds) => ipcRenderer.invoke('cancel-selected-jobs', jobIds), addJobsToBatch: (payload) => ipcRenderer.invoke('add-jobs-to-batch', payload), completeUploadFinalization: (payload) => ipcRenderer.invoke('complete-upload-finalization', payload), + completeQueuePersist: (payload) => ipcRenderer.invoke('complete-queue-persist', payload), finishAfterActive: () => ipcRenderer.invoke('finish-after-active'), runHealthCheck: (payload) => ipcRenderer.invoke('run-health-check', payload), onHealthCheckResult: (callback) => { @@ -184,6 +185,9 @@ contextBridge.exposeInMainWorld('api', { onUploadBatchDone: (callback) => { ipcRenderer.on('upload-batch-done', (_event, data) => callback(data)); }, + onPersistUploadQueue: (callback) => { + ipcRenderer.on('persist-upload-queue', (_event, data) => callback(data)); + }, onUploadStats: (callback) => { ipcRenderer.on('upload-stats', (_event, data) => callback(data)); }, @@ -223,6 +227,7 @@ contextBridge.exposeInMainWorld('api', { ipcRenderer.removeAllListeners('folder-monitor:renderer-generation'); ipcRenderer.removeAllListeners('upload-progress'); ipcRenderer.removeAllListeners('upload-batch-done'); + ipcRenderer.removeAllListeners('persist-upload-queue'); ipcRenderer.removeAllListeners('upload-stats'); ipcRenderer.removeAllListeners('app:update-available'); ipcRenderer.removeAllListeners('app:update-progress'); diff --git a/renderer/app.js b/renderer/app.js index 09ca585..2364ed1 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -1573,6 +1573,21 @@ async function init() { }); } }); + if (window.api.onPersistUploadQueue) { + window.api.onPersistUploadQueue(async (data) => { + const requestId = data && data.requestId; + if (!requestId) return; + let ok = false; + if (closePreparationState === 'open') { + try { + queuePersistThrottle.cancel(); + await persistQueueStateNow(); + ok = true; + } catch {} + } + try { await window.api.completeQueuePersist({ requestId, ok }); } catch {} + }); + } window.api.onUploadStats((data) => { handleStats(data); }); diff --git a/scripts/verify-public-release.mjs b/scripts/verify-public-release.mjs index dd24570..f9b2a2e 100644 --- a/scripts/verify-public-release.mjs +++ b/scripts/verify-public-release.mjs @@ -29,6 +29,7 @@ const sourceFiles = [ 'lib/diagnostics-agent.js', 'lib/diagnostics-collectors.js', 'lib/doodstream-upload.js', + 'lib/early-source-cleanup.js', 'lib/file-probe.js', 'lib/file-discovery.js', 'lib/folder-monitor.js',