Harden source cleanup durability barriers
Separate provisional upload success from persistently confirmed cleanup state, invalidate prior confirmations before retries, and promote completion only inside the final queue persistence handshake. Preserve source files when history or queue persistence fails and add regression coverage for restart, retry, rollback, and real Electron finalization paths.
This commit is contained in:
+107
-21
@@ -3,6 +3,7 @@
|
||||
|
||||
const pathApi = typeof require === 'function' ? require('path') : null;
|
||||
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
|
||||
const metadataVersion = 2;
|
||||
|
||||
function normalizeFile(file, platform) {
|
||||
const value = typeof file === 'string' ? file.trim() : '';
|
||||
@@ -59,18 +60,26 @@
|
||||
return uniqueHosters(values);
|
||||
}
|
||||
|
||||
function completedHosters(jobs, requiredHosters) {
|
||||
function confirmedHosters(jobs, requiredHosters) {
|
||||
const values = [];
|
||||
for (const job of jobs) {
|
||||
if (Array.isArray(job.sourceCleanupCompletedHosters)) {
|
||||
values.push(...job.sourceCleanupCompletedHosters);
|
||||
if (job.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(job.sourceCleanupConfirmedHosters)) {
|
||||
values.push(...job.sourceCleanupConfirmedHosters);
|
||||
}
|
||||
}
|
||||
const confirmed = new Set(uniqueHosters(values));
|
||||
return requiredHosters.filter((hoster) => confirmed.has(hoster));
|
||||
}
|
||||
|
||||
function provisionalHosters(jobs, requiredHosters) {
|
||||
const values = [];
|
||||
for (const job of jobs) {
|
||||
if (job.status === 'done') values.push(job.hoster);
|
||||
if (job.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(job.sourceCleanupProvisionalHosters)) {
|
||||
values.push(...job.sourceCleanupProvisionalHosters);
|
||||
}
|
||||
}
|
||||
const completed = new Set(uniqueHosters(values));
|
||||
return requiredHosters.filter((hoster) => completed.has(hoster));
|
||||
const provisional = new Set(uniqueHosters(values));
|
||||
return requiredHosters.filter((hoster) => provisional.has(hoster));
|
||||
}
|
||||
|
||||
function storedToken(jobs) {
|
||||
@@ -90,12 +99,15 @@
|
||||
return null;
|
||||
}
|
||||
|
||||
function assignMetadata(jobs, token, requiredHosters, completed, fingerprint, touchedJobs, touchedSet) {
|
||||
function assignMetadata(jobs, token, requiredHosters, confirmed, provisional, fingerprint, touchedJobs, touchedSet) {
|
||||
for (const job of jobs) {
|
||||
job.sourceCleanupMetadataVersion = metadataVersion;
|
||||
job.sourceCleanupToken = token;
|
||||
job.sourceCleanupRequiredHosters = [...requiredHosters];
|
||||
job.sourceCleanupCompletedHosters = [...completed];
|
||||
job.sourceCleanupConfirmedHosters = [...confirmed];
|
||||
job.sourceCleanupProvisionalHosters = [...provisional];
|
||||
job.sourceCleanupFingerprint = cloneFingerprint(fingerprint);
|
||||
delete job.sourceCleanupCompletedHosters;
|
||||
if (!touchedSet.has(job)) {
|
||||
touchedSet.add(job);
|
||||
touchedJobs.push(job);
|
||||
@@ -108,7 +120,10 @@
|
||||
const touchedJobs = [];
|
||||
const touchedSet = new Set();
|
||||
const preparedFiles = new Set();
|
||||
if (!Array.isArray(queueJobs) || !Array.isArray(jobsToStart)) return { groups, touchedJobs };
|
||||
const revokedHosters = [];
|
||||
const revokedSet = new Set();
|
||||
if (!Array.isArray(queueJobs) || !Array.isArray(jobsToStart)) return { groups, touchedJobs, revokedHosters };
|
||||
const currentRoundJobs = new Set(jobsToStart);
|
||||
|
||||
for (const selectedJob of jobsToStart) {
|
||||
const file = normalizeFile(selectedJob && selectedJob.file, platform);
|
||||
@@ -124,14 +139,26 @@
|
||||
...persistedRequired,
|
||||
...siblings.map((job) => job.hoster)
|
||||
]);
|
||||
const completed = completedHosters(siblings, requiredHosters);
|
||||
const startedHosters = new Set(uniqueHosters(
|
||||
siblings.filter((job) => currentRoundJobs.has(job)).map((job) => job.hoster)
|
||||
));
|
||||
const storedConfirmed = confirmedHosters(siblings, requiredHosters);
|
||||
const confirmed = storedConfirmed.filter((hoster) => !startedHosters.has(hoster));
|
||||
const provisional = provisionalHosters(siblings, requiredHosters)
|
||||
.filter((hoster) => !startedHosters.has(hoster));
|
||||
for (const hoster of storedConfirmed) {
|
||||
if (!startedHosters.has(hoster) || revokedSet.has(hoster)) continue;
|
||||
revokedSet.add(hoster);
|
||||
revokedHosters.push(hoster);
|
||||
}
|
||||
const fingerprint = storedFingerprint(siblings);
|
||||
|
||||
assignMetadata(
|
||||
siblings,
|
||||
token,
|
||||
requiredHosters,
|
||||
completed,
|
||||
confirmed,
|
||||
provisional,
|
||||
fingerprint,
|
||||
touchedJobs,
|
||||
touchedSet
|
||||
@@ -141,33 +168,84 @@
|
||||
token,
|
||||
file: selectedJob.file,
|
||||
requiredHosters: [...requiredHosters],
|
||||
completedHosters: [...completed],
|
||||
confirmedHosters: [...confirmed],
|
||||
fingerprint: cloneFingerprint(fingerprint),
|
||||
jobs: siblings.map((job) => ({
|
||||
jobId: job.id,
|
||||
hoster: normalizeHoster(job.hoster),
|
||||
status: job.status
|
||||
status: job.status,
|
||||
currentRound: currentRoundJobs.has(job)
|
||||
}))
|
||||
});
|
||||
}
|
||||
|
||||
return { groups, touchedJobs };
|
||||
return { groups, touchedJobs, revokedHosters };
|
||||
}
|
||||
|
||||
function markCompleted(queueJobs, job, platform) {
|
||||
const siblings = relatedJobs(queueJobs, job, platform);
|
||||
if (siblings.length === 0) return [];
|
||||
const requiredHosters = storedRequiredHosters(siblings);
|
||||
const completed = new Set(completedHosters(siblings, requiredHosters));
|
||||
const provisional = new Set(provisionalHosters(siblings, requiredHosters));
|
||||
const hoster = normalizeHoster(job.hoster);
|
||||
if (requiredHosters.includes(hoster)) completed.add(hoster);
|
||||
const orderedCompleted = requiredHosters.filter((required) => completed.has(required));
|
||||
if (requiredHosters.includes(hoster)) provisional.add(hoster);
|
||||
const orderedConfirmed = confirmedHosters(siblings, requiredHosters);
|
||||
const orderedProvisional = requiredHosters.filter((required) => provisional.has(required));
|
||||
for (const sibling of siblings) {
|
||||
sibling.sourceCleanupCompletedHosters = [...orderedCompleted];
|
||||
sibling.sourceCleanupMetadataVersion = metadataVersion;
|
||||
sibling.sourceCleanupConfirmedHosters = [...orderedConfirmed];
|
||||
sibling.sourceCleanupProvisionalHosters = [...orderedProvisional];
|
||||
delete sibling.sourceCleanupCompletedHosters;
|
||||
}
|
||||
return siblings;
|
||||
}
|
||||
|
||||
async function persistRoundCompletions(queueJobs, options = {}) {
|
||||
if (!Array.isArray(queueJobs) || typeof options.persist !== 'function') return false;
|
||||
const historyPersisted = options.historyPersisted === true;
|
||||
const groupsByToken = new Map();
|
||||
for (const job of queueJobs) {
|
||||
if (!job || typeof job.sourceCleanupToken !== 'string' || !job.sourceCleanupToken) continue;
|
||||
if (!groupsByToken.has(job.sourceCleanupToken)) groupsByToken.set(job.sourceCleanupToken, []);
|
||||
groupsByToken.get(job.sourceCleanupToken).push(job);
|
||||
}
|
||||
const snapshots = [];
|
||||
for (const jobs of groupsByToken.values()) {
|
||||
const requiredHosters = storedRequiredHosters(jobs);
|
||||
const confirmed = confirmedHosters(jobs, requiredHosters);
|
||||
const provisional = provisionalHosters(jobs, requiredHosters);
|
||||
const promoted = historyPersisted
|
||||
? uniqueHosters([...confirmed, ...provisional])
|
||||
: confirmed;
|
||||
const orderedPromoted = requiredHosters.filter((hoster) => promoted.includes(hoster));
|
||||
for (const job of jobs) {
|
||||
snapshots.push({
|
||||
job,
|
||||
confirmedHosters: job.sourceCleanupMetadataVersion === metadataVersion
|
||||
? uniqueHosters(job.sourceCleanupConfirmedHosters)
|
||||
: []
|
||||
});
|
||||
job.sourceCleanupMetadataVersion = metadataVersion;
|
||||
job.sourceCleanupConfirmedHosters = [...orderedPromoted];
|
||||
job.sourceCleanupProvisionalHosters = [];
|
||||
delete job.sourceCleanupCompletedHosters;
|
||||
}
|
||||
}
|
||||
let persisted = false;
|
||||
try {
|
||||
persisted = (await options.persist()) === true;
|
||||
} catch {}
|
||||
if (!persisted) {
|
||||
for (const snapshot of snapshots) {
|
||||
snapshot.job.sourceCleanupMetadataVersion = metadataVersion;
|
||||
snapshot.job.sourceCleanupConfirmedHosters = [...snapshot.confirmedHosters];
|
||||
snapshot.job.sourceCleanupProvisionalHosters = [];
|
||||
delete snapshot.job.sourceCleanupCompletedHosters;
|
||||
}
|
||||
}
|
||||
return persisted;
|
||||
}
|
||||
|
||||
function removeRequirement(queueJobs, job, platform) {
|
||||
if (!job || protectedStatuses.has(job.status)) return [];
|
||||
const siblings = relatedJobs(queueJobs, job, platform);
|
||||
@@ -176,13 +254,20 @@
|
||||
const required = Array.isArray(sibling.sourceCleanupRequiredHosters)
|
||||
? sibling.sourceCleanupRequiredHosters
|
||||
: [];
|
||||
const completed = Array.isArray(sibling.sourceCleanupCompletedHosters)
|
||||
? sibling.sourceCleanupCompletedHosters
|
||||
const confirmed = sibling.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(sibling.sourceCleanupConfirmedHosters)
|
||||
? sibling.sourceCleanupConfirmedHosters
|
||||
: [];
|
||||
const provisional = sibling.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(sibling.sourceCleanupProvisionalHosters)
|
||||
? sibling.sourceCleanupProvisionalHosters
|
||||
: [];
|
||||
sibling.sourceCleanupMetadataVersion = metadataVersion;
|
||||
sibling.sourceCleanupRequiredHosters = uniqueHosters(required)
|
||||
.filter((hoster) => hoster !== removedHoster);
|
||||
sibling.sourceCleanupCompletedHosters = uniqueHosters(completed)
|
||||
sibling.sourceCleanupConfirmedHosters = uniqueHosters(confirmed)
|
||||
.filter((hoster) => hoster !== removedHoster);
|
||||
sibling.sourceCleanupProvisionalHosters = uniqueHosters(provisional)
|
||||
.filter((hoster) => hoster !== removedHoster);
|
||||
delete sibling.sourceCleanupCompletedHosters;
|
||||
}
|
||||
return siblings;
|
||||
}
|
||||
@@ -211,6 +296,7 @@
|
||||
const api = {
|
||||
prepareGroups,
|
||||
markCompleted,
|
||||
persistRoundCompletions,
|
||||
removeRequirement,
|
||||
applyFingerprints
|
||||
};
|
||||
|
||||
+43
-23
@@ -83,30 +83,29 @@ function createSourceFileCleanup(options) {
|
||||
|
||||
function createManifest(input, canonicalFile) {
|
||||
const requiredHosters = uniqueStrings(input.requiredHosters);
|
||||
const completedHosters = new Set(uniqueStrings(input.completedHosters));
|
||||
const confirmedHosters = new Set(
|
||||
uniqueStrings(input.confirmedHosters).filter((hoster) => requiredHosters.includes(hoster))
|
||||
);
|
||||
const jobs = new Map();
|
||||
|
||||
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
||||
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
||||
const status = completedHosters.has(job.hoster) ? 'done' : normalizeStatus(job.status);
|
||||
const currentRound = job.currentRound !== false;
|
||||
const status = currentRound ? 'pending' : normalizeStatus(job.status);
|
||||
jobs.set(job.jobId, Object.freeze({
|
||||
jobId: job.jobId,
|
||||
hoster: job.hoster,
|
||||
status
|
||||
status,
|
||||
currentRound
|
||||
}));
|
||||
}
|
||||
|
||||
for (const hoster of completedHosters) {
|
||||
if (requiredHosters.includes(hoster)) continue;
|
||||
completedHosters.delete(hoster);
|
||||
}
|
||||
|
||||
return {
|
||||
token: input.token || input.sourceCleanupToken,
|
||||
file: path.resolve(input.file),
|
||||
canonicalFile,
|
||||
requiredHosters: Object.freeze(requiredHosters),
|
||||
completedHosters,
|
||||
confirmedHosters,
|
||||
jobs,
|
||||
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
|
||||
fingerprint: null,
|
||||
@@ -152,16 +151,32 @@ function createSourceFileCleanup(options) {
|
||||
...existing.requiredHosters,
|
||||
...input.requiredHosters
|
||||
]));
|
||||
for (const hoster of uniqueStrings(input.completedHosters)) {
|
||||
if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster);
|
||||
const incomingJobs = Array.isArray(input.jobs) ? input.jobs : [];
|
||||
const currentRoundHosters = new Set(
|
||||
incomingJobs
|
||||
.filter((job) => job && typeof job.hoster === 'string' && job.currentRound !== false)
|
||||
.map((job) => job.hoster)
|
||||
);
|
||||
for (const hoster of currentRoundHosters) existing.confirmedHosters.delete(hoster);
|
||||
for (const hoster of uniqueStrings(input.confirmedHosters)) {
|
||||
if (existing.requiredHosters.includes(hoster) && !currentRoundHosters.has(hoster)) {
|
||||
existing.confirmedHosters.add(hoster);
|
||||
}
|
||||
}
|
||||
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
||||
for (const job of incomingJobs) {
|
||||
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
||||
const previous = existing.jobs.get(job.jobId);
|
||||
const status = existing.completedHosters.has(job.hoster)
|
||||
? 'done'
|
||||
const incomingCurrentRound = job.currentRound !== false;
|
||||
const currentRound = Boolean((previous && previous.currentRound) || incomingCurrentRound);
|
||||
const status = incomingCurrentRound
|
||||
? 'pending'
|
||||
: normalizeStatus(previous ? previous.status : job.status);
|
||||
existing.jobs.set(job.jobId, Object.freeze({ jobId: job.jobId, hoster: job.hoster, status }));
|
||||
existing.jobs.set(job.jobId, Object.freeze({
|
||||
jobId: job.jobId,
|
||||
hoster: job.hoster,
|
||||
status,
|
||||
currentRound
|
||||
}));
|
||||
}
|
||||
fingerprints[token] = cloneFingerprint(existing.fingerprint);
|
||||
continue;
|
||||
@@ -194,10 +209,9 @@ function createSourceFileCleanup(options) {
|
||||
const manifest = groups.get(token);
|
||||
if (!manifest || manifest.finalizationPromise) return false;
|
||||
const job = manifest.jobs.get(event.jobId);
|
||||
if (!job || job.hoster !== event.hoster) return false;
|
||||
if (!job || !job.currentRound || job.hoster !== event.hoster) return false;
|
||||
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
|
||||
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
|
||||
if (event.status === 'done') manifest.completedHosters.add(event.hoster);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -206,7 +220,7 @@ function createSourceFileCleanup(options) {
|
||||
for (const manifest of groups.values()) {
|
||||
if (manifest.finalizationPromise) continue;
|
||||
const job = manifest.jobs.get(jobId);
|
||||
if (!job) continue;
|
||||
if (!job || !job.currentRound) continue;
|
||||
manifest.jobs.set(jobId, Object.freeze({ ...job, status: 'skipped' }));
|
||||
changed = true;
|
||||
}
|
||||
@@ -216,11 +230,17 @@ function createSourceFileCleanup(options) {
|
||||
function blockingStatuses(manifest) {
|
||||
const blocking = [];
|
||||
for (const hoster of manifest.requiredHosters) {
|
||||
if (manifest.completedHosters.has(hoster)) continue;
|
||||
const statuses = [...manifest.jobs.values()]
|
||||
.filter((job) => job.hoster === hoster)
|
||||
.map((job) => job.status);
|
||||
if (statuses.includes('done')) continue;
|
||||
const jobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster);
|
||||
const currentJobs = jobs.filter((job) => job.currentRound);
|
||||
if (currentJobs.length > 0) {
|
||||
const currentBlocker = currentJobs.find((job) => job.status !== 'done');
|
||||
if (!currentBlocker) continue;
|
||||
const status = currentJobs.map((job) => job.status).find((value) => value !== 'pending') || 'pending';
|
||||
blocking.push({ hoster, status });
|
||||
continue;
|
||||
}
|
||||
if (manifest.confirmedHosters.has(hoster)) continue;
|
||||
const statuses = jobs.map((job) => job.status).filter((status) => status !== 'done');
|
||||
const status = statuses.find((value) => value !== 'pending') || 'pending';
|
||||
blocking.push({ hoster, status });
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ let lastSessionSummary = null;
|
||||
let sourceDeleteJournal = null;
|
||||
const pendingUploadFinalizations = new Map();
|
||||
|
||||
function requestUploadFinalization(summary) {
|
||||
function requestUploadFinalization(summary, historyPersisted) {
|
||||
const finalizationId = `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
@@ -136,7 +136,7 @@ function requestUploadFinalization(summary) {
|
||||
resolve(value);
|
||||
}
|
||||
});
|
||||
safeSend('upload-batch-done', { summary, finalizationId });
|
||||
safeSend('upload-batch-done', { summary, finalizationId, historyPersisted: historyPersisted === true });
|
||||
});
|
||||
}
|
||||
const activeUploadProducerTrackers = new Set();
|
||||
@@ -2276,7 +2276,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
||||
for (const value of _progressByJob.values()) finalProgressBatch.push(value);
|
||||
_progressByJob.clear();
|
||||
if (finalProgressBatch.length) safeSend('upload-progress-batch', finalProgressBatch);
|
||||
const queuePersisted = await requestUploadFinalization(summary);
|
||||
const queuePersisted = await requestUploadFinalization(summary, historyPersisted);
|
||||
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
|
||||
if (!queuePersisted) debugLog('upload finalization blocked: renderer queue acknowledgement missing');
|
||||
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted });
|
||||
@@ -2978,6 +2978,10 @@ ipcMain.handle('complete-upload-finalization', async (_event, payload) => {
|
||||
const finalizationId = payload && payload.finalizationId;
|
||||
const pending = finalizationId && pendingUploadFinalizations.get(finalizationId);
|
||||
if (!pending) return false;
|
||||
if (payload.ready === false) {
|
||||
pending.resolve(false);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await configStore.savePendingQueue(payload.pendingQueue ?? null);
|
||||
pending.resolve(true);
|
||||
|
||||
+88
-22
@@ -60,6 +60,8 @@ let selectedUploadHosters = [];
|
||||
let config = { hosters: {}, hosterSettings: {}, globalSettings: {} };
|
||||
let hosterSettings = {};
|
||||
let uploading = false;
|
||||
let sourceCleanupFinalizationPending = false;
|
||||
let sourceCleanupRevocationPending = false;
|
||||
let healthCheckRunning = false;
|
||||
|
||||
let _rLongTasks = 0, _rLongTaskMax = 0, _rFrameLast = 0, _rFrameWorst = 0, _rFrameCount = 0, _rFrameJank = 0, _rPerfLastLog = 0, _rPerfWindowStart = 0;
|
||||
@@ -445,16 +447,18 @@ async function init() {
|
||||
}
|
||||
window.api.onUploadBatchDone(async (data) => {
|
||||
const summary = data && data.summary ? data.summary : data;
|
||||
handleBatchDone(summary);
|
||||
if (data && data.finalizationId && window.api.completeUploadFinalization) {
|
||||
if (_doneRemovalCoalescer) _doneRemovalCoalescer.drainSync();
|
||||
queuePersistThrottle.cancel();
|
||||
await window.api.completeUploadFinalization({
|
||||
finalizationId: data.finalizationId,
|
||||
pendingQueue: queueJobs.some((job) => !['done', 'skipped'].includes(job.status))
|
||||
? buildPersistedQueueState()
|
||||
: null
|
||||
});
|
||||
const requiresFinalization = Boolean(data && data.finalizationId && window.api.completeUploadFinalization);
|
||||
if (!requiresFinalization) {
|
||||
handleBatchDone(summary);
|
||||
return;
|
||||
}
|
||||
sourceCleanupFinalizationPending = true;
|
||||
try {
|
||||
handleBatchDone(summary, { deferPersistence: true });
|
||||
await completeSourceCleanupFinalization(data);
|
||||
} finally {
|
||||
sourceCleanupFinalizationPending = false;
|
||||
updateQueueActionButtons();
|
||||
}
|
||||
});
|
||||
window.api.onUploadStats((data) => {
|
||||
@@ -1234,9 +1238,13 @@ function restoreQueueStateFromConfig() {
|
||||
failureDetails: job.failureDetails || null,
|
||||
interrupted: interruptedJobIds.size > 0 && !['done', 'error', 'skipped'].includes(job.status),
|
||||
result: job.result || null,
|
||||
sourceCleanupMetadataVersion: job.sourceCleanupMetadataVersion === 2 ? 2 : null,
|
||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||
sourceCleanupRequiredHosters: Array.isArray(job.sourceCleanupRequiredHosters) ? [...job.sourceCleanupRequiredHosters] : [],
|
||||
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
||||
sourceCleanupConfirmedHosters: job.sourceCleanupMetadataVersion === 2 && Array.isArray(job.sourceCleanupConfirmedHosters)
|
||||
? [...job.sourceCleanupConfirmedHosters]
|
||||
: [],
|
||||
sourceCleanupProvisionalHosters: [],
|
||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||
attempt: 0,
|
||||
maxAttempts: job.maxAttempts || 0,
|
||||
@@ -1311,9 +1319,12 @@ function buildPersistedQueueState() {
|
||||
error: isTerminal ? (job.error || null) : null,
|
||||
failureDetails: isTerminal ? (job.failureDetails || null) : null,
|
||||
result: isTerminal ? (job.result || null) : null,
|
||||
sourceCleanupMetadataVersion: job.sourceCleanupToken ? 2 : null,
|
||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||
sourceCleanupRequiredHosters: Array.isArray(job.sourceCleanupRequiredHosters) ? [...job.sourceCleanupRequiredHosters] : [],
|
||||
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
||||
sourceCleanupConfirmedHosters: job.sourceCleanupMetadataVersion === 2 && Array.isArray(job.sourceCleanupConfirmedHosters)
|
||||
? [...job.sourceCleanupConfirmedHosters]
|
||||
: [],
|
||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||
maxAttempts: job.maxAttempts || 0
|
||||
};
|
||||
@@ -1527,7 +1538,7 @@ function updateStartButton() {
|
||||
const hosters = getSelectedHosters();
|
||||
const hasQueuedJobs = queueJobs.some(isStartableQueueJob);
|
||||
const canBuildQueueFromSelection = selectedFiles.length > 0 && hosters.length > 0;
|
||||
btn.disabled = uploading || !(hasQueuedJobs || canBuildQueueFromSelection);
|
||||
btn.disabled = uploading || sourceCleanupFinalizationPending || !(hasQueuedJobs || canBuildQueueFromSelection);
|
||||
}
|
||||
|
||||
const _UPLOAD_SELECTION_STATUSES = new Set(['done', 'error', 'aborted', 'skipped']);
|
||||
@@ -1563,8 +1574,8 @@ function updateQueueActionButtons() {
|
||||
const moveDownBtn = document.getElementById('moveDownBtn');
|
||||
const moveBottomBtn = document.getElementById('moveBottomBtn');
|
||||
|
||||
if (startSelectedBtn) startSelectedBtn.disabled = uploading || !hasStartableSelection;
|
||||
if (reuploadBtn) reuploadBtn.disabled = !hasUploadSelection;
|
||||
if (startSelectedBtn) startSelectedBtn.disabled = uploading || sourceCleanupFinalizationPending || !hasStartableSelection;
|
||||
if (reuploadBtn) reuploadBtn.disabled = sourceCleanupFinalizationPending || !hasUploadSelection;
|
||||
if (abortSelectedBtn) abortSelectedBtn.disabled = !hasAbortSelection;
|
||||
if (finishStopBtn) finishStopBtn.disabled = !uploading;
|
||||
if (abortAllBtn) abortAllBtn.disabled = !uploading;
|
||||
@@ -2894,8 +2905,54 @@ function getSelectedJobLinks() {
|
||||
|
||||
// --- Upload ---
|
||||
function prepareSourceCleanup(jobs) {
|
||||
if (!config.globalSettings?.deleteSourceAfterSuccessfulUpload || !window.SourceCleanupPolicy) return { groups: [] };
|
||||
return window.SourceCleanupPolicy.prepareGroups(queueJobs, jobs, () => window.crypto.randomUUID(), 'win32');
|
||||
if (!window.SourceCleanupPolicy) return { groups: [] };
|
||||
const enabled = config.globalSettings?.deleteSourceAfterSuccessfulUpload === true;
|
||||
const hasMetadata = Array.isArray(jobs) && jobs.some((job) => job?.sourceCleanupToken);
|
||||
if (!enabled && !hasMetadata) return { groups: [] };
|
||||
const preparation = window.SourceCleanupPolicy.prepareGroups(queueJobs, jobs, () => window.crypto.randomUUID(), 'win32');
|
||||
return enabled ? preparation : { ...preparation, groups: [] };
|
||||
}
|
||||
|
||||
async function persistSourceCleanupRevocations(preparation) {
|
||||
if (Array.isArray(preparation?.revokedHosters) && preparation.revokedHosters.length > 0) {
|
||||
sourceCleanupRevocationPending = true;
|
||||
}
|
||||
if (!sourceCleanupRevocationPending) return;
|
||||
queuePersistThrottle.cancel();
|
||||
await persistQueueStateNow();
|
||||
await flushConfigWrites();
|
||||
sourceCleanupRevocationPending = false;
|
||||
}
|
||||
|
||||
async function completeSourceCleanupFinalization(data) {
|
||||
if (!data?.finalizationId || !window.api.completeUploadFinalization) return false;
|
||||
if (_doneRemovalCoalescer) _doneRemovalCoalescer.drainSync();
|
||||
queuePersistThrottle.cancel();
|
||||
let writesReady = true;
|
||||
try {
|
||||
await flushConfigWrites();
|
||||
} catch {
|
||||
writesReady = false;
|
||||
}
|
||||
const persist = async () => {
|
||||
if (!writesReady) {
|
||||
try {
|
||||
await window.api.completeUploadFinalization({ finalizationId: data.finalizationId, ready: false });
|
||||
} catch {}
|
||||
return false;
|
||||
}
|
||||
return window.api.completeUploadFinalization({
|
||||
finalizationId: data.finalizationId,
|
||||
pendingQueue: queueJobs.some((job) => !['done', 'skipped'].includes(job.status))
|
||||
? buildPersistedQueueState()
|
||||
: null
|
||||
});
|
||||
};
|
||||
if (!window.SourceCleanupPolicy?.persistRoundCompletions) return Boolean(await persist());
|
||||
return window.SourceCleanupPolicy.persistRoundCompletions(queueJobs, {
|
||||
historyPersisted: data.historyPersisted === true,
|
||||
persist
|
||||
});
|
||||
}
|
||||
|
||||
function serializeUploadJob(job) {
|
||||
@@ -2904,15 +2961,16 @@ function serializeUploadJob(job) {
|
||||
file: job.file,
|
||||
fileName: job.fileName,
|
||||
hoster: job.hoster,
|
||||
sourceCleanupMetadataVersion: job.sourceCleanupToken ? 2 : null,
|
||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||
sourceCleanupRequiredHosters: job.sourceCleanupRequiredHosters || [],
|
||||
sourceCleanupCompletedHosters: job.sourceCleanupCompletedHosters || [],
|
||||
sourceCleanupConfirmedHosters: job.sourceCleanupConfirmedHosters || [],
|
||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null
|
||||
};
|
||||
}
|
||||
|
||||
async function startUpload(opts) {
|
||||
if (uploading) return;
|
||||
if (uploading || sourceCleanupFinalizationPending) return;
|
||||
if (!(opts && opts._restoredAutoStart)) cancelStartupQueueAutoStart();
|
||||
if (!(opts && opts._autoRetry)) _cancelAutoRetry(true);
|
||||
else _cancelAutoRetry(false);
|
||||
@@ -2950,6 +3008,7 @@ async function startUpload(opts) {
|
||||
updateQueueActionButtons();
|
||||
renderQueueTable();
|
||||
updateStatusBar();
|
||||
await persistSourceCleanupRevocations(cleanupPreparation);
|
||||
|
||||
const uploadPayload = {
|
||||
hosters,
|
||||
@@ -2991,6 +3050,7 @@ function _markSkippedJobs(result) {
|
||||
}
|
||||
|
||||
async function startSelectedUpload(explicitJobs) {
|
||||
if (sourceCleanupFinalizationPending) return;
|
||||
const scopedJobs = Array.isArray(explicitJobs) ? explicitJobs : _getVisibleSelectedQueueJobs();
|
||||
if (uploading) {
|
||||
_hydrateMissingJobSizes();
|
||||
@@ -3008,6 +3068,7 @@ async function startSelectedUpload(explicitJobs) {
|
||||
renderQueueTable();
|
||||
let result = null;
|
||||
try {
|
||||
await persistSourceCleanupRevocations(cleanupPreparation);
|
||||
result = await window.api.addJobsToBatch({
|
||||
jobs: addable.map(serializeUploadJob),
|
||||
sourceCleanupGroups: cleanupPreparation.groups
|
||||
@@ -3071,6 +3132,7 @@ async function startSelectedUpload(explicitJobs) {
|
||||
updateQueueActionButtons();
|
||||
renderQueueTable();
|
||||
updateStatusBar();
|
||||
await persistSourceCleanupRevocations(cleanupPreparation);
|
||||
|
||||
const uploadPayload = {
|
||||
hosters,
|
||||
@@ -3237,7 +3299,7 @@ function _handleProgressImpl(data) {
|
||||
persistQueueStateSoon();
|
||||
}
|
||||
|
||||
function handleBatchDone(summary) {
|
||||
function handleBatchDone(summary, options = {}) {
|
||||
uploading = false;
|
||||
applySummaryResults(summary);
|
||||
_deletedJobIds.clear(); // Free memory — stale IDs no longer needed after batch completes
|
||||
@@ -3312,8 +3374,10 @@ function handleBatchDone(summary) {
|
||||
}
|
||||
}
|
||||
|
||||
if (queueJobs.some((job) => !['done', 'skipped'].includes(job.status))) persistQueueStateSoon(true);
|
||||
else clearPersistedQueueStateSoon();
|
||||
if (!options.deferPersistence) {
|
||||
if (queueJobs.some((job) => !['done', 'skipped'].includes(job.status))) persistQueueStateSoon(true);
|
||||
else clearPersistedQueueStateSoon();
|
||||
}
|
||||
|
||||
lastUploadStats = { state: 'idle', globalSpeedKbs: 0, totalBytes: lastUploadStats.totalBytes, elapsed: lastUploadStats.elapsed, activeJobs: 0 };
|
||||
updateStatusBar();
|
||||
@@ -3408,6 +3472,7 @@ function _maybeShowBatchSummary(summary) {
|
||||
}
|
||||
|
||||
function _retryFailedFromBuckets(buckets, transientOnly) {
|
||||
if (sourceCleanupFinalizationPending) return;
|
||||
const cats = transientOnly ? ['hoster-transient', 'network', 'unknown'] : ['hoster-transient', 'network', 'unknown', 'file-rejected', 'account-error'];
|
||||
const toRetry = [];
|
||||
for (const cat of cats) {
|
||||
@@ -3540,6 +3605,7 @@ async function copyJobLogToClipboard() {
|
||||
|
||||
// --- Retry ---
|
||||
async function retrySelectedJobs() {
|
||||
if (sourceCleanupFinalizationPending) return;
|
||||
_normalizeQueueSelectionToVisible();
|
||||
const retryJobs = [];
|
||||
// Build a Set for O(1) selectedFiles dedup below.
|
||||
|
||||
@@ -48,13 +48,13 @@ test('prepareGroups creates one Windows manifest with a stable token and immutab
|
||||
token: 'cleanup-1',
|
||||
file: 'C:\\Uploads\\Movie.MKV',
|
||||
requiredHosters: ['doodstream.com', 'voe.sx', 'vidmoly.me', 'byse.sx'],
|
||||
completedHosters: ['doodstream.com', 'voe.sx'],
|
||||
confirmedHosters: [],
|
||||
fingerprint: null,
|
||||
jobs: [
|
||||
{ jobId: 'job-doodstream', hoster: 'doodstream.com', status: 'done' },
|
||||
{ jobId: 'job-voe', hoster: 'voe.sx', status: 'done' },
|
||||
{ jobId: 'job-vidmoly', hoster: 'vidmoly.me', status: 'error' },
|
||||
{ jobId: 'job-byse', hoster: 'byse.sx', status: 'preview' }
|
||||
{ jobId: 'job-doodstream', hoster: 'doodstream.com', status: 'done', currentRound: false },
|
||||
{ jobId: 'job-voe', hoster: 'voe.sx', status: 'done', currentRound: false },
|
||||
{ jobId: 'job-vidmoly', hoster: 'vidmoly.me', status: 'error', currentRound: true },
|
||||
{ jobId: 'job-byse', hoster: 'byse.sx', status: 'preview', currentRound: false }
|
||||
]
|
||||
});
|
||||
assert.deepEqual(queueJobs.map((job) => job.sourceCleanupToken), [
|
||||
@@ -74,11 +74,13 @@ test('prepareGroups creates one Windows manifest with a stable token and immutab
|
||||
assert.doesNotThrow(() => JSON.stringify(prepared.groups));
|
||||
});
|
||||
|
||||
test('prepareGroups reuses persisted metadata across a partial retry', () => {
|
||||
test('prepareGroups reuses confirmed metadata across a partial retry', () => {
|
||||
const queueJobs = queueFixture();
|
||||
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-1', 'win32');
|
||||
queueJobs[0].sourceCleanupCompletedHosters = ['doodstream.com', 'voe.sx'];
|
||||
queueJobs[1].sourceCleanupCompletedHosters = ['doodstream.com', 'voe.sx'];
|
||||
for (const job of queueJobs) {
|
||||
job.sourceCleanupMetadataVersion = 2;
|
||||
job.sourceCleanupConfirmedHosters = ['doodstream.com', 'voe.sx'];
|
||||
}
|
||||
queueJobs[0].status = 'preview';
|
||||
queueJobs[1].status = 'preview';
|
||||
|
||||
@@ -93,7 +95,7 @@ test('prepareGroups reuses persisted metadata across a partial retry', () => {
|
||||
'vidmoly.me',
|
||||
'byse.sx'
|
||||
]);
|
||||
assert.deepEqual(prepared.groups[0].completedHosters, ['doodstream.com', 'voe.sx']);
|
||||
assert.deepEqual(prepared.groups[0].confirmedHosters, ['doodstream.com', 'voe.sx']);
|
||||
assert.equal(prepared.groups[0].jobs.find((job) => job.hoster === 'vidmoly.me').status, 'error');
|
||||
});
|
||||
|
||||
@@ -163,7 +165,7 @@ test('removeRequirement drops only an explicitly discarded unstarted hoster', ()
|
||||
}
|
||||
});
|
||||
|
||||
test('markCompleted preserves successful hosters for later retries', () => {
|
||||
test('markCompleted keeps successful hosters provisional for the current round', () => {
|
||||
const queueJobs = queueFixture();
|
||||
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-1', 'win32');
|
||||
|
||||
@@ -171,11 +173,8 @@ test('markCompleted preserves successful hosters for later retries', () => {
|
||||
policy.markCompleted(queueJobs, queueJobs[2], 'win32');
|
||||
|
||||
for (const job of queueJobs) {
|
||||
assert.deepEqual(job.sourceCleanupCompletedHosters, [
|
||||
'doodstream.com',
|
||||
'voe.sx',
|
||||
'vidmoly.me'
|
||||
]);
|
||||
assert.deepEqual(job.sourceCleanupConfirmedHosters, []);
|
||||
assert.deepEqual(job.sourceCleanupProvisionalHosters, ['vidmoly.me']);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -203,3 +202,127 @@ test('applyFingerprints attaches Main fingerprints by token and includes them in
|
||||
assert.notEqual(job.sourceCleanupFingerprint, fingerprint);
|
||||
}
|
||||
});
|
||||
|
||||
test('keeps round successes provisional until history and queue persistence succeed', async () => {
|
||||
const queueJobs = [
|
||||
{ id: 'job-voe', file: 'C:\\Uploads\\Round.bin', hoster: 'voe.sx', status: 'preview' },
|
||||
{ id: 'job-byse', file: 'C:\\Uploads\\Round.bin', hoster: 'byse.sx', status: 'error' }
|
||||
];
|
||||
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-round', 'win32');
|
||||
queueJobs[0].status = 'done';
|
||||
|
||||
policy.markCompleted(queueJobs, queueJobs[0], 'win32');
|
||||
|
||||
for (const job of queueJobs) {
|
||||
assert.deepEqual(job.sourceCleanupConfirmedHosters, []);
|
||||
assert.deepEqual(job.sourceCleanupProvisionalHosters, ['voe.sx']);
|
||||
}
|
||||
|
||||
let persistedHosters = null;
|
||||
const queuePersisted = await policy.persistRoundCompletions(queueJobs, {
|
||||
historyPersisted: true,
|
||||
persist: async () => {
|
||||
persistedHosters = queueJobs.map((job) => [...job.sourceCleanupConfirmedHosters]);
|
||||
return true;
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(queuePersisted, true);
|
||||
assert.deepEqual(persistedHosters, [['voe.sx'], ['voe.sx']]);
|
||||
for (const job of queueJobs) {
|
||||
assert.deepEqual(job.sourceCleanupConfirmedHosters, ['voe.sx']);
|
||||
assert.deepEqual(job.sourceCleanupProvisionalHosters, []);
|
||||
}
|
||||
});
|
||||
|
||||
test('rolls back provisional promotion when the final queue save fails', async () => {
|
||||
const queueJobs = [
|
||||
{
|
||||
id: 'job-voe',
|
||||
file: 'C:\\Uploads\\Partial.bin',
|
||||
hoster: 'voe.sx',
|
||||
status: 'done',
|
||||
sourceCleanupMetadataVersion: 2,
|
||||
sourceCleanupToken: 'cleanup-partial',
|
||||
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||
sourceCleanupConfirmedHosters: ['voe.sx']
|
||||
},
|
||||
{
|
||||
id: 'job-byse',
|
||||
file: 'C:\\Uploads\\Partial.bin',
|
||||
hoster: 'byse.sx',
|
||||
status: 'preview',
|
||||
sourceCleanupMetadataVersion: 2,
|
||||
sourceCleanupToken: 'cleanup-partial',
|
||||
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||
sourceCleanupConfirmedHosters: ['voe.sx']
|
||||
}
|
||||
];
|
||||
policy.prepareGroups(queueJobs, [queueJobs[1]], () => 'unused', 'win32');
|
||||
queueJobs[1].status = 'done';
|
||||
policy.markCompleted(queueJobs, queueJobs[1], 'win32');
|
||||
|
||||
const queuePersisted = await policy.persistRoundCompletions(queueJobs, {
|
||||
historyPersisted: true,
|
||||
persist: async () => false
|
||||
});
|
||||
|
||||
assert.equal(queuePersisted, false);
|
||||
for (const job of queueJobs) {
|
||||
assert.deepEqual(job.sourceCleanupConfirmedHosters, ['voe.sx']);
|
||||
assert.deepEqual(job.sourceCleanupProvisionalHosters, []);
|
||||
}
|
||||
});
|
||||
|
||||
test('ignores legacy v2.1.19 completed markers when preparing a retry', () => {
|
||||
const queueJobs = [{
|
||||
id: 'job-voe',
|
||||
file: 'C:\\Uploads\\Legacy.bin',
|
||||
hoster: 'voe.sx',
|
||||
status: 'error',
|
||||
sourceCleanupToken: 'cleanup-legacy',
|
||||
sourceCleanupRequiredHosters: ['voe.sx'],
|
||||
sourceCleanupCompletedHosters: ['voe.sx']
|
||||
}];
|
||||
|
||||
const prepared = policy.prepareGroups(queueJobs, queueJobs, () => 'unused', 'win32');
|
||||
|
||||
assert.deepEqual(prepared.groups[0].confirmedHosters, []);
|
||||
assert.equal(queueJobs[0].sourceCleanupMetadataVersion, 2);
|
||||
assert.deepEqual(queueJobs[0].sourceCleanupConfirmedHosters, []);
|
||||
assert.equal(Object.prototype.hasOwnProperty.call(queueJobs[0], 'sourceCleanupCompletedHosters'), false);
|
||||
});
|
||||
|
||||
test('selecting a confirmed hoster for retry revokes its durable completion', () => {
|
||||
const queueJobs = [
|
||||
{
|
||||
id: 'job-voe',
|
||||
file: 'C:\\Uploads\\Retry.bin',
|
||||
hoster: 'voe.sx',
|
||||
status: 'preview',
|
||||
sourceCleanupMetadataVersion: 2,
|
||||
sourceCleanupToken: 'cleanup-retry',
|
||||
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||
sourceCleanupConfirmedHosters: ['voe.sx', 'byse.sx']
|
||||
},
|
||||
{
|
||||
id: 'job-byse',
|
||||
file: 'C:\\Uploads\\Retry.bin',
|
||||
hoster: 'byse.sx',
|
||||
status: 'done',
|
||||
sourceCleanupMetadataVersion: 2,
|
||||
sourceCleanupToken: 'cleanup-retry',
|
||||
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||
sourceCleanupConfirmedHosters: ['voe.sx', 'byse.sx']
|
||||
}
|
||||
];
|
||||
|
||||
const prepared = policy.prepareGroups(queueJobs, [queueJobs[0]], () => 'unused', 'win32');
|
||||
|
||||
assert.deepEqual(prepared.groups[0].confirmedHosters, ['byse.sx']);
|
||||
assert.equal(prepared.groups[0].jobs[0].currentRound, true);
|
||||
assert.equal(prepared.groups[0].jobs[1].currentRound, false);
|
||||
for (const job of queueJobs) {
|
||||
assert.deepEqual(job.sourceCleanupConfirmedHosters, ['byse.sx']);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -19,7 +19,7 @@ function group(file, overrides = {}) {
|
||||
token: 'cleanup-1',
|
||||
file,
|
||||
requiredHosters: ['voe.sx', 'byse.sx'],
|
||||
completedHosters: [],
|
||||
confirmedHosters: [],
|
||||
jobs: [
|
||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending' },
|
||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending' }
|
||||
@@ -81,7 +81,7 @@ test('fingerprints a regular file and keeps the registered manifest immutable',
|
||||
assert.equal(typeof fingerprints['cleanup-1'].ino, 'number');
|
||||
|
||||
manifest.requiredHosters.splice(1, 1);
|
||||
manifest.completedHosters.push('byse.sx');
|
||||
manifest.confirmedHosters.push('byse.sx');
|
||||
manifest.jobs[1].hoster = 'voe.sx';
|
||||
await cleanup.settle({
|
||||
token: 'cleanup-1',
|
||||
@@ -182,7 +182,13 @@ test('combines previous successes with a successful retry without relaxing other
|
||||
await t.test('last retry completes the immutable group', async (subtest) => {
|
||||
const { file } = await makeSource(subtest, 'retry-completes.bin');
|
||||
const { cleanup } = makeCleanup();
|
||||
const manifest = group(file, { completedHosters: ['voe.sx'] });
|
||||
const manifest = group(file, {
|
||||
confirmedHosters: ['voe.sx'],
|
||||
jobs: [
|
||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: false },
|
||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true }
|
||||
]
|
||||
});
|
||||
await cleanup.registerGroups([manifest]);
|
||||
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'done' });
|
||||
|
||||
@@ -196,11 +202,11 @@ test('combines previous successes with a successful retry without relaxing other
|
||||
const { cleanup, audits } = makeCleanup();
|
||||
const manifest = group(file, {
|
||||
requiredHosters: ['voe.sx', 'byse.sx', 'vidmoly.me'],
|
||||
completedHosters: ['voe.sx'],
|
||||
confirmedHosters: ['voe.sx'],
|
||||
jobs: [
|
||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' },
|
||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending' },
|
||||
{ jobId: 'job-vidmoly', file, hoster: 'vidmoly.me', status: 'error' }
|
||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: false },
|
||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true },
|
||||
{ jobId: 'job-vidmoly', file, hoster: 'vidmoly.me', status: 'error', currentRound: false }
|
||||
]
|
||||
});
|
||||
await cleanup.registerGroups([manifest]);
|
||||
@@ -467,3 +473,118 @@ test('rechecks the staged file and restores a replacement without deleting it',
|
||||
assert.equal((await fs.promises.readFile(file, 'utf-8')), 'replacement data');
|
||||
assert.equal(audits.at(-1).outcome, 'source-changed');
|
||||
});
|
||||
|
||||
test('keeps the source after done, failed history persistence, and an aborted retry of the same hoster', async (t) => {
|
||||
const { file } = await makeSource(t, 'history-barrier-retry.bin');
|
||||
const firstRound = makeCleanup();
|
||||
const firstManifest = group(file, {
|
||||
requiredHosters: ['voe.sx'],
|
||||
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||
});
|
||||
await firstRound.cleanup.registerGroups([firstManifest]);
|
||||
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||
|
||||
assert.deepEqual(await firstRound.cleanup.finishBatch({ historyPersisted: false, queuePersisted: true }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
|
||||
const retryRound = makeCleanup();
|
||||
const retryManifest = group(file, {
|
||||
requiredHosters: ['voe.sx'],
|
||||
completedHosters: ['voe.sx'],
|
||||
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||
});
|
||||
await retryRound.cleanup.registerGroups([retryManifest]);
|
||||
await retryRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'aborted' });
|
||||
|
||||
assert.deepEqual(await retryRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
});
|
||||
|
||||
test('deletes after a persisted partial round and a successful retry of the remaining hoster', async (t) => {
|
||||
const { file } = await makeSource(t, 'partial-round-retry.bin');
|
||||
const firstRound = makeCleanup();
|
||||
const firstManifest = group(file, {
|
||||
jobs: [
|
||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true },
|
||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true }
|
||||
]
|
||||
});
|
||||
await firstRound.cleanup.registerGroups([firstManifest]);
|
||||
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'error' });
|
||||
|
||||
assert.deepEqual(await firstRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
|
||||
const retryRound = makeCleanup();
|
||||
const retryManifest = group(file, {
|
||||
confirmedHosters: ['voe.sx'],
|
||||
jobs: [
|
||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: false },
|
||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true }
|
||||
]
|
||||
});
|
||||
await retryRound.cleanup.registerGroups([retryManifest]);
|
||||
await retryRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'done' });
|
||||
|
||||
assert.deepEqual(await retryRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['deleted']);
|
||||
assert.equal(await exists(file), false);
|
||||
});
|
||||
|
||||
test('does not trust a legacy completion after a failed queue barrier and restart', async (t) => {
|
||||
const { file } = await makeSource(t, 'queue-barrier-restart.bin');
|
||||
const firstRound = makeCleanup();
|
||||
const firstManifest = group(file, {
|
||||
requiredHosters: ['voe.sx'],
|
||||
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||
});
|
||||
await firstRound.cleanup.registerGroups([firstManifest]);
|
||||
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||
|
||||
assert.deepEqual(await firstRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: false }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
|
||||
const restartedRound = makeCleanup();
|
||||
const restartedManifest = group(file, {
|
||||
requiredHosters: ['voe.sx'],
|
||||
completedHosters: ['voe.sx'],
|
||||
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||
});
|
||||
await restartedRound.cleanup.registerGroups([restartedManifest]);
|
||||
await restartedRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'error' });
|
||||
|
||||
assert.deepEqual(await restartedRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
});
|
||||
|
||||
test('lets a current non-done retry override an earlier confirmed completion', async (t) => {
|
||||
const { file } = await makeSource(t, 'confirmed-hoster-retry.bin');
|
||||
const { cleanup, audits } = makeCleanup();
|
||||
const manifest = group(file, {
|
||||
requiredHosters: ['voe.sx'],
|
||||
confirmedHosters: ['voe.sx'],
|
||||
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done', currentRound: true }]
|
||||
});
|
||||
await cleanup.registerGroups([manifest]);
|
||||
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'aborted' });
|
||||
|
||||
assert.deepEqual(await cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
assert.deepEqual(audits[0].blockingStatuses, [{ hoster: 'voe.sx', status: 'aborted' }]);
|
||||
});
|
||||
|
||||
test('re-registering a hoster for the current round invalidates its earlier current success', async (t) => {
|
||||
const { file } = await makeSource(t, 'same-batch-retry.bin');
|
||||
const { cleanup, audits } = makeCleanup();
|
||||
const manifest = group(file, {
|
||||
requiredHosters: ['voe.sx'],
|
||||
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||
});
|
||||
await cleanup.registerGroups([manifest]);
|
||||
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||
await cleanup.registerGroups([manifest]);
|
||||
|
||||
assert.deepEqual(await cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||
assert.equal(await exists(file), true);
|
||||
assert.deepEqual(audits[0].blockingStatuses, [{ hoster: 'voe.sx', status: 'pending' }]);
|
||||
});
|
||||
|
||||
@@ -1711,6 +1711,35 @@ setTimeout(async () => {
|
||||
|
||||
const importedQueueMerge = await wc.executeJavaScript('(() => { queueJobs = [{ id: "ui-live-queue-after-import", file: "C:/ui/live-after-import.bin", fileName: "live-after-import.bin", hoster: "byse.sx", status: "queued", bytesTotal: 12 }]; rebuildJobIndex(); const imported = structuredClone(config); imported.globalSettings = { ...(imported.globalSettings || {}), webhookUrl: "https://queue-merge.invalid/imported", pendingQueue: { savedAt: 1, queueJobs: [{ id: "ui-stale-import-queue" }] } }; applyImportedConfig(imported, "Importiert"); return { webhookUrl: config.globalSettings.webhookUrl, ids: config.globalSettings.pendingQueue?.queueJobs?.map(job => job.id) || [] }; })()');
|
||||
check('Imported settings keep the live local queue in renderer memory', importedQueueMerge.webhookUrl === 'https://queue-merge.invalid/imported' && importedQueueMerge.ids.join('|') === 'ui-live-queue-after-import');
|
||||
const sourceCleanupDisabledRetry = await wc.executeJavaScript('(() => { const previousSetting = config.globalSettings?.deleteSourceAfterSuccessfulUpload; config.globalSettings = { ...(config.globalSettings || {}), deleteSourceAfterSuccessfulUpload: false }; queueJobs = [{ id: "ui-cleanup-disabled-retry", file: "C:/ui/cleanup-disabled.bin", fileName: "cleanup-disabled.bin", hoster: "voe.sx", status: "preview", bytesTotal: 10, sourceCleanupMetadataVersion: 2, sourceCleanupToken: "ui-cleanup-disabled-token", sourceCleanupRequiredHosters: ["voe.sx"], sourceCleanupConfirmedHosters: ["voe.sx"] }]; rebuildJobIndex(); const preparation = prepareSourceCleanup(queueJobs); config.globalSettings.deleteSourceAfterSuccessfulUpload = previousSetting; return { groups: preparation.groups.length, revokedHosters: preparation.revokedHosters || [], confirmedHosters: queueJobs[0].sourceCleanupConfirmedHosters || [] }; })()');
|
||||
check('Retry revokes a confirmed cleanup hoster even while source deletion is disabled', sourceCleanupDisabledRetry.groups === 0 && sourceCleanupDisabledRetry.revokedHosters.join('|') === 'voe.sx' && sourceCleanupDisabledRetry.confirmedHosters.length === 0);
|
||||
let sourceCleanupRevocationSaveCalls = 0;
|
||||
ipcMain.removeHandler('save-pending-queue');
|
||||
ipcMain.handle('save-pending-queue', async () => {
|
||||
sourceCleanupRevocationSaveCalls++;
|
||||
if (sourceCleanupRevocationSaveCalls === 1) throw new Error('injected cleanup revocation save failure');
|
||||
return true;
|
||||
});
|
||||
const sourceCleanupRevocationRetry = await wc.executeJavaScript('(async () => { queueJobs = [{ id: "ui-cleanup-revocation-retry", file: "C:/ui/cleanup-revocation-retry.bin", fileName: "cleanup-revocation-retry.bin", hoster: "voe.sx", status: "preview", bytesTotal: 10, sourceCleanupMetadataVersion: 2, sourceCleanupToken: "ui-cleanup-revocation-retry-token", sourceCleanupRequiredHosters: ["voe.sx"], sourceCleanupConfirmedHosters: ["voe.sx"] }]; rebuildJobIndex(); const firstPreparation = prepareSourceCleanup(queueJobs); let firstFailed = false; try { await persistSourceCleanupRevocations(firstPreparation); } catch { firstFailed = true; } const secondPreparation = prepareSourceCleanup(queueJobs); let secondSucceeded = true; try { await persistSourceCleanupRevocations(secondPreparation); } catch { secondSucceeded = false; } return { firstFailed, secondSucceeded, secondRevocations: secondPreparation.revokedHosters || [] }; })()');
|
||||
const sourceCleanupRevocationCallsBeforeRecovery = sourceCleanupRevocationSaveCalls;
|
||||
restoreInitialIpcHandler('save-pending-queue');
|
||||
await wc.executeJavaScript('flushConfigWrites()');
|
||||
check('A failed cleanup revocation save stays mandatory for the next start attempt', sourceCleanupRevocationRetry.firstFailed === true && sourceCleanupRevocationRetry.secondSucceeded === true && sourceCleanupRevocationRetry.secondRevocations.length === 0 && sourceCleanupRevocationCallsBeforeRecovery >= 2);
|
||||
const sourceCleanupFinalizationGate = await wc.executeJavaScript('(async () => { if (typeof sourceCleanupFinalizationPending === "undefined") return { available: false }; queueJobs = [{ id: "ui-cleanup-finalizing", file: "C:/ui/cleanup-finalizing.bin", fileName: "cleanup-finalizing.bin", hoster: "voe.sx", status: "done", bytesTotal: 10, result: { download_url: "https://example.invalid/finalizing" } }]; rebuildJobIndex(); selectedJobIds.clear(); selectedJobIds.add(queueJobs[0].id); sourceCleanupFinalizationPending = true; updateQueueActionButtons(); const disabled = document.getElementById("reuploadSelectedBtn")?.disabled === true; await retrySelectedJobs(); const status = queueJobs[0].status; sourceCleanupFinalizationPending = false; selectedJobIds.clear(); updateQueueActionButtons(); return { available: true, disabled, status }; })()');
|
||||
check('Final cleanup persistence blocks a new retry until the handshake settles', sourceCleanupFinalizationGate.available === true && sourceCleanupFinalizationGate.disabled === true && sourceCleanupFinalizationGate.status === 'done');
|
||||
let sourceCleanupFinalizationPayload = null;
|
||||
ipcMain.removeHandler('complete-upload-finalization');
|
||||
ipcMain.handle('complete-upload-finalization', (_event, payload) => {
|
||||
sourceCleanupFinalizationPayload = payload;
|
||||
return false;
|
||||
});
|
||||
const sourceCleanupRollback = await wc.executeJavaScript('(async () => { queuePersistThrottle.cancel(); await flushConfigWrites(); queueJobs = [{ id: "ui-cleanup-voe", file: "C:/ui/cleanup-round.bin", fileName: "cleanup-round.bin", hoster: "voe.sx", status: "preview", bytesTotal: 10 }, { id: "ui-cleanup-byse", file: "C:/ui/cleanup-round.bin", fileName: "cleanup-round.bin", hoster: "byse.sx", status: "error", bytesTotal: 10 }]; rebuildJobIndex(); window.SourceCleanupPolicy.prepareGroups(queueJobs, queueJobs, () => "ui-cleanup-token", "win32"); queueJobs[0].status = "done"; window.SourceCleanupPolicy.markCompleted(queueJobs, queueJobs[0], "win32"); const before = buildPersistedQueueState(); if (typeof completeSourceCleanupFinalization !== "function") return { available: false, before }; const result = await completeSourceCleanupFinalization({ finalizationId: "ui-cleanup-finalization", historyPersisted: true }); const after = buildPersistedQueueState(); return { available: true, result, before, after }; })()');
|
||||
const sourceCleanupBeforeJobs = sourceCleanupRollback.before?.queueJobs || [];
|
||||
const sourceCleanupAfterJobs = sourceCleanupRollback.after?.queueJobs || [];
|
||||
const sourceCleanupPromotedJobs = sourceCleanupFinalizationPayload?.pendingQueue?.queueJobs || [];
|
||||
const sourceCleanupRollbackOk = sourceCleanupRollback.available === true && sourceCleanupRollback.result === false && sourceCleanupBeforeJobs.length === 2 && sourceCleanupBeforeJobs.every(job => !Object.prototype.hasOwnProperty.call(job, 'sourceCleanupProvisionalHosters') && !Object.prototype.hasOwnProperty.call(job, 'sourceCleanupCompletedHosters') && (job.sourceCleanupConfirmedHosters || []).length === 0) && sourceCleanupPromotedJobs.length === 2 && sourceCleanupPromotedJobs.every(job => (job.sourceCleanupConfirmedHosters || []).join('|') === 'voe.sx') && sourceCleanupAfterJobs.length === 2 && sourceCleanupAfterJobs.every(job => (job.sourceCleanupConfirmedHosters || []).length === 0);
|
||||
check('Final queue persistence promotes only inside the handshake and rolls back failed saves', sourceCleanupRollbackOk);
|
||||
restoreInitialIpcHandler('complete-upload-finalization');
|
||||
await wc.executeJavaScript('document.getElementById("copyToast")?.classList.remove("show")');
|
||||
|
||||
console.log('\\n=== History View ===');
|
||||
|
||||
Reference in New Issue
Block a user