Delete each uploaded source as soon as its links and queue state are saved

Previously sources were only removed when the whole upload batch ended. In
automation mode a batch keeps growing for hours, so hundreds of finished
files stayed on disk. A file is now removed once every required hoster
succeeded, its links are flushed to the upload log, the automation ledger
is written and the renderer confirmed the queue save. Without that evidence
the file still waits for the batch end as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
SucukdeluxeandClaude Opus 5.5 committed 2026-10-21 02:59:54 +02:00
1 parent 58da562e65
commit d12a5de45b
6 files changed
+201 -6

No files matched your search

+63
View File
@@ -0,0 +1,63 @@
function createEarlySourceCleanup(options) {
if (!options || !options.cleanup || typeof options.prepare !== 'function') {
throw new TypeError('createEarlySourceCleanup requires cleanup and prepare');
}
const cleanup = options.cleanup;
const prepare = options.prepare;
const delayMs = Number.isFinite(options.delayMs) ? Math.max(0, options.delayMs) : 3000;
const schedule = typeof options.schedule === 'function' ? options.schedule : (callback, ms) => setTimeout(callback, ms);
const cancel = typeof options.cancel === 'function' ? options.cancel : (timer) => clearTimeout(timer);
const onError = typeof options.onError === 'function' ? options.onError : () => {};
const pending = new Set();
let timer = null;
let running = null;
let closed = false;
function arm() {
if (closed || timer || running || pending.size === 0) return;
timer = schedule(() => {
timer = null;
running = run().finally(() => {
running = null;
arm();
});
}, delayMs);
}
async function run() {
const tokens = [...pending];
pending.clear();
if (closed || tokens.length === 0) return;
let evidence = null;
try {
evidence = await prepare(tokens);
} catch (error) {
onError(error);
return;
}
if (closed || !evidence || evidence.queuePersisted !== true) return;
try {
await cleanup.finalizeReady(tokens, evidence);
} catch (error) {
onError(error);
}
}
function request(token) {
if (closed || typeof token !== 'string' || token.length === 0) return;
pending.add(token);
arm();
}
async function close() {
closed = true;
if (timer) cancel(timer);
timer = null;
pending.clear();
if (running) await running;
}
return Object.freeze({ request, close });
}
module.exports = { createEarlySourceCleanup };
+46 -5
View File
@@ -13,6 +13,7 @@ function createSourceFileCleanup(options) {
const isEnabled = typeof options.isEnabled === 'function' ? options.isEnabled : () => false;
const audit = typeof options.audit === 'function' ? options.audit : () => {};
const journal = options.journal || null;
const onReady = typeof options.onReady === 'function' ? options.onReady : null;
const wait = typeof options.wait === 'function'
? options.wait
: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
@@ -107,6 +108,7 @@ function createSourceFileCleanup(options) {
canonicalFile,
requiredHosters: Object.freeze(requiredHosters),
completedHosters,
previouslyCompletedHosters: new Set(completedHosters),
jobs,
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
fingerprint: null,
@@ -153,7 +155,9 @@ function createSourceFileCleanup(options) {
...input.requiredHosters
]));
for (const hoster of uniqueStrings(input.completedHosters)) {
if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster);
if (!existing.requiredHosters.includes(hoster)) continue;
existing.completedHosters.add(hoster);
existing.previouslyCompletedHosters.add(hoster);
}
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
@@ -198,9 +202,44 @@ function createSourceFileCleanup(options) {
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
if (event.status === 'done') manifest.completedHosters.add(event.hoster);
if (onReady && isReady(manifest)) {
try { onReady(manifest.token); } catch {}
}
return true;
}
function isReady(manifest) {
return !manifest.finalizationPromise
&& !manifest.registrationOutcome
&& manifest.requiredHosters.length > 0
&& blockingStatuses(manifest).length === 0;
}
function hasDurableResults(manifest, isDurable) {
for (const hoster of manifest.requiredHosters) {
if (manifest.previouslyCompletedHosters.has(hoster)) continue;
const doneJobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster && job.status === 'done');
if (!doneJobs.some((job) => isDurable(job))) return false;
}
return true;
}
function finalizeReady(tokens, evidence = {}) {
const isDurable = typeof evidence.isDurable === 'function' ? evidence.isDurable : () => false;
const pending = [];
for (const token of Array.isArray(tokens) ? tokens : []) {
const manifest = groups.get(token);
if (!manifest || !isReady(manifest) || !hasDurableResults(manifest, isDurable)) continue;
manifest.finalizationPromise = finalize(
manifest,
{ historyPersisted: true, queuePersisted: evidence.queuePersisted === true },
{ scope: 'file' }
);
pending.push(manifest.finalizationPromise);
}
return Promise.all(pending);
}
function markSkipped(jobId) {
let changed = false;
for (const manifest of groups.values()) {
@@ -341,7 +380,7 @@ function createSourceFileCleanup(options) {
}
}
async function finalize(manifest, barriers) {
async function finalize(manifest, barriers, scope = {}) {
try {
let enabled;
try {
@@ -383,7 +422,8 @@ function createSourceFileCleanup(options) {
return sourceOutcome.outcome;
}
const auditPersisted = await writeAudit(manifest, 'delete-approved', {
trigger: 'all-selected-hosters-succeeded'
trigger: 'all-selected-hosters-succeeded',
...scope
});
if (!auditPersisted) {
await emitAudit(manifest, 'blocked', { trigger: 'audit-write-failed' });
@@ -412,7 +452,8 @@ function createSourceFileCleanup(options) {
await clearJournal(manifest);
await emitAudit(manifest, 'deleted', {
trigger: 'all-selected-hosters-succeeded',
attempts: result.attempts
attempts: result.attempts,
...scope
});
return 'deleted';
} catch (error) {
@@ -449,7 +490,7 @@ function createSourceFileCleanup(options) {
return Promise.all(pending);
}
return Object.freeze({ registerGroups, settle, markSkipped, finishBatch });
return Object.freeze({ registerGroups, settle, markSkipped, finalizeReady, finishBatch });
}
module.exports = { createSourceFileCleanup };