Delete each uploaded source as soon as its links and queue state are saved
Previously sources were only removed when the whole upload batch ended. In automation mode a batch keeps growing for hours, so hundreds of finished files stayed on disk. A file is now removed once every required hoster succeeded, its links are flushed to the upload log, the automation ledger is written and the renderer confirmed the queue save. Without that evidence the file still waits for the batch end as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
58da562e65
commit
d12a5de45b
6 files changed
+201
-6
No files matched your search
@@ -13,6 +13,7 @@ function createSourceFileCleanup(options) {
|
||||
const isEnabled = typeof options.isEnabled === 'function' ? options.isEnabled : () => false;
|
||||
const audit = typeof options.audit === 'function' ? options.audit : () => {};
|
||||
const journal = options.journal || null;
|
||||
const onReady = typeof options.onReady === 'function' ? options.onReady : null;
|
||||
const wait = typeof options.wait === 'function'
|
||||
? options.wait
|
||||
: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
@@ -107,6 +108,7 @@ function createSourceFileCleanup(options) {
|
||||
canonicalFile,
|
||||
requiredHosters: Object.freeze(requiredHosters),
|
||||
completedHosters,
|
||||
previouslyCompletedHosters: new Set(completedHosters),
|
||||
jobs,
|
||||
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
|
||||
fingerprint: null,
|
||||
@@ -153,7 +155,9 @@ function createSourceFileCleanup(options) {
|
||||
...input.requiredHosters
|
||||
]));
|
||||
for (const hoster of uniqueStrings(input.completedHosters)) {
|
||||
if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster);
|
||||
if (!existing.requiredHosters.includes(hoster)) continue;
|
||||
existing.completedHosters.add(hoster);
|
||||
existing.previouslyCompletedHosters.add(hoster);
|
||||
}
|
||||
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
||||
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
||||
@@ -198,9 +202,44 @@ function createSourceFileCleanup(options) {
|
||||
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
|
||||
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
|
||||
if (event.status === 'done') manifest.completedHosters.add(event.hoster);
|
||||
if (onReady && isReady(manifest)) {
|
||||
try { onReady(manifest.token); } catch {}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function isReady(manifest) {
|
||||
return !manifest.finalizationPromise
|
||||
&& !manifest.registrationOutcome
|
||||
&& manifest.requiredHosters.length > 0
|
||||
&& blockingStatuses(manifest).length === 0;
|
||||
}
|
||||
|
||||
function hasDurableResults(manifest, isDurable) {
|
||||
for (const hoster of manifest.requiredHosters) {
|
||||
if (manifest.previouslyCompletedHosters.has(hoster)) continue;
|
||||
const doneJobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster && job.status === 'done');
|
||||
if (!doneJobs.some((job) => isDurable(job))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function finalizeReady(tokens, evidence = {}) {
|
||||
const isDurable = typeof evidence.isDurable === 'function' ? evidence.isDurable : () => false;
|
||||
const pending = [];
|
||||
for (const token of Array.isArray(tokens) ? tokens : []) {
|
||||
const manifest = groups.get(token);
|
||||
if (!manifest || !isReady(manifest) || !hasDurableResults(manifest, isDurable)) continue;
|
||||
manifest.finalizationPromise = finalize(
|
||||
manifest,
|
||||
{ historyPersisted: true, queuePersisted: evidence.queuePersisted === true },
|
||||
{ scope: 'file' }
|
||||
);
|
||||
pending.push(manifest.finalizationPromise);
|
||||
}
|
||||
return Promise.all(pending);
|
||||
}
|
||||
|
||||
function markSkipped(jobId) {
|
||||
let changed = false;
|
||||
for (const manifest of groups.values()) {
|
||||
@@ -341,7 +380,7 @@ function createSourceFileCleanup(options) {
|
||||
}
|
||||
}
|
||||
|
||||
async function finalize(manifest, barriers) {
|
||||
async function finalize(manifest, barriers, scope = {}) {
|
||||
try {
|
||||
let enabled;
|
||||
try {
|
||||
@@ -383,7 +422,8 @@ function createSourceFileCleanup(options) {
|
||||
return sourceOutcome.outcome;
|
||||
}
|
||||
const auditPersisted = await writeAudit(manifest, 'delete-approved', {
|
||||
trigger: 'all-selected-hosters-succeeded'
|
||||
trigger: 'all-selected-hosters-succeeded',
|
||||
...scope
|
||||
});
|
||||
if (!auditPersisted) {
|
||||
await emitAudit(manifest, 'blocked', { trigger: 'audit-write-failed' });
|
||||
@@ -412,7 +452,8 @@ function createSourceFileCleanup(options) {
|
||||
await clearJournal(manifest);
|
||||
await emitAudit(manifest, 'deleted', {
|
||||
trigger: 'all-selected-hosters-succeeded',
|
||||
attempts: result.attempts
|
||||
attempts: result.attempts,
|
||||
...scope
|
||||
});
|
||||
return 'deleted';
|
||||
} catch (error) {
|
||||
@@ -449,7 +490,7 @@ function createSourceFileCleanup(options) {
|
||||
return Promise.all(pending);
|
||||
}
|
||||
|
||||
return Object.freeze({ registerGroups, settle, markSkipped, finishBatch });
|
||||
return Object.freeze({ registerGroups, settle, markSkipped, finalizeReady, finishBatch });
|
||||
}
|
||||
|
||||
module.exports = { createSourceFileCleanup };
|
||||
Reference in new issue
Block a user