Harden source cleanup durability barriers
Separate provisional upload success from persistently confirmed cleanup state, invalidate prior confirmations before retries, and promote completion only inside the final queue persistence handshake. Preserve source files when history or queue persistence fails and add regression coverage for restart, retry, rollback, and real Electron finalization paths.
This commit is contained in:
+107
-21
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
const pathApi = typeof require === 'function' ? require('path') : null;
|
const pathApi = typeof require === 'function' ? require('path') : null;
|
||||||
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
|
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
|
||||||
|
const metadataVersion = 2;
|
||||||
|
|
||||||
function normalizeFile(file, platform) {
|
function normalizeFile(file, platform) {
|
||||||
const value = typeof file === 'string' ? file.trim() : '';
|
const value = typeof file === 'string' ? file.trim() : '';
|
||||||
@@ -59,18 +60,26 @@
|
|||||||
return uniqueHosters(values);
|
return uniqueHosters(values);
|
||||||
}
|
}
|
||||||
|
|
||||||
function completedHosters(jobs, requiredHosters) {
|
function confirmedHosters(jobs, requiredHosters) {
|
||||||
const values = [];
|
const values = [];
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
if (Array.isArray(job.sourceCleanupCompletedHosters)) {
|
if (job.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(job.sourceCleanupConfirmedHosters)) {
|
||||||
values.push(...job.sourceCleanupCompletedHosters);
|
values.push(...job.sourceCleanupConfirmedHosters);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
const confirmed = new Set(uniqueHosters(values));
|
||||||
|
return requiredHosters.filter((hoster) => confirmed.has(hoster));
|
||||||
|
}
|
||||||
|
|
||||||
|
function provisionalHosters(jobs, requiredHosters) {
|
||||||
|
const values = [];
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
if (job.status === 'done') values.push(job.hoster);
|
if (job.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(job.sourceCleanupProvisionalHosters)) {
|
||||||
|
values.push(...job.sourceCleanupProvisionalHosters);
|
||||||
}
|
}
|
||||||
const completed = new Set(uniqueHosters(values));
|
}
|
||||||
return requiredHosters.filter((hoster) => completed.has(hoster));
|
const provisional = new Set(uniqueHosters(values));
|
||||||
|
return requiredHosters.filter((hoster) => provisional.has(hoster));
|
||||||
}
|
}
|
||||||
|
|
||||||
function storedToken(jobs) {
|
function storedToken(jobs) {
|
||||||
@@ -90,12 +99,15 @@
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function assignMetadata(jobs, token, requiredHosters, completed, fingerprint, touchedJobs, touchedSet) {
|
function assignMetadata(jobs, token, requiredHosters, confirmed, provisional, fingerprint, touchedJobs, touchedSet) {
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
|
job.sourceCleanupMetadataVersion = metadataVersion;
|
||||||
job.sourceCleanupToken = token;
|
job.sourceCleanupToken = token;
|
||||||
job.sourceCleanupRequiredHosters = [...requiredHosters];
|
job.sourceCleanupRequiredHosters = [...requiredHosters];
|
||||||
job.sourceCleanupCompletedHosters = [...completed];
|
job.sourceCleanupConfirmedHosters = [...confirmed];
|
||||||
|
job.sourceCleanupProvisionalHosters = [...provisional];
|
||||||
job.sourceCleanupFingerprint = cloneFingerprint(fingerprint);
|
job.sourceCleanupFingerprint = cloneFingerprint(fingerprint);
|
||||||
|
delete job.sourceCleanupCompletedHosters;
|
||||||
if (!touchedSet.has(job)) {
|
if (!touchedSet.has(job)) {
|
||||||
touchedSet.add(job);
|
touchedSet.add(job);
|
||||||
touchedJobs.push(job);
|
touchedJobs.push(job);
|
||||||
@@ -108,7 +120,10 @@
|
|||||||
const touchedJobs = [];
|
const touchedJobs = [];
|
||||||
const touchedSet = new Set();
|
const touchedSet = new Set();
|
||||||
const preparedFiles = new Set();
|
const preparedFiles = new Set();
|
||||||
if (!Array.isArray(queueJobs) || !Array.isArray(jobsToStart)) return { groups, touchedJobs };
|
const revokedHosters = [];
|
||||||
|
const revokedSet = new Set();
|
||||||
|
if (!Array.isArray(queueJobs) || !Array.isArray(jobsToStart)) return { groups, touchedJobs, revokedHosters };
|
||||||
|
const currentRoundJobs = new Set(jobsToStart);
|
||||||
|
|
||||||
for (const selectedJob of jobsToStart) {
|
for (const selectedJob of jobsToStart) {
|
||||||
const file = normalizeFile(selectedJob && selectedJob.file, platform);
|
const file = normalizeFile(selectedJob && selectedJob.file, platform);
|
||||||
@@ -124,14 +139,26 @@
|
|||||||
...persistedRequired,
|
...persistedRequired,
|
||||||
...siblings.map((job) => job.hoster)
|
...siblings.map((job) => job.hoster)
|
||||||
]);
|
]);
|
||||||
const completed = completedHosters(siblings, requiredHosters);
|
const startedHosters = new Set(uniqueHosters(
|
||||||
|
siblings.filter((job) => currentRoundJobs.has(job)).map((job) => job.hoster)
|
||||||
|
));
|
||||||
|
const storedConfirmed = confirmedHosters(siblings, requiredHosters);
|
||||||
|
const confirmed = storedConfirmed.filter((hoster) => !startedHosters.has(hoster));
|
||||||
|
const provisional = provisionalHosters(siblings, requiredHosters)
|
||||||
|
.filter((hoster) => !startedHosters.has(hoster));
|
||||||
|
for (const hoster of storedConfirmed) {
|
||||||
|
if (!startedHosters.has(hoster) || revokedSet.has(hoster)) continue;
|
||||||
|
revokedSet.add(hoster);
|
||||||
|
revokedHosters.push(hoster);
|
||||||
|
}
|
||||||
const fingerprint = storedFingerprint(siblings);
|
const fingerprint = storedFingerprint(siblings);
|
||||||
|
|
||||||
assignMetadata(
|
assignMetadata(
|
||||||
siblings,
|
siblings,
|
||||||
token,
|
token,
|
||||||
requiredHosters,
|
requiredHosters,
|
||||||
completed,
|
confirmed,
|
||||||
|
provisional,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
touchedJobs,
|
touchedJobs,
|
||||||
touchedSet
|
touchedSet
|
||||||
@@ -141,33 +168,84 @@
|
|||||||
token,
|
token,
|
||||||
file: selectedJob.file,
|
file: selectedJob.file,
|
||||||
requiredHosters: [...requiredHosters],
|
requiredHosters: [...requiredHosters],
|
||||||
completedHosters: [...completed],
|
confirmedHosters: [...confirmed],
|
||||||
fingerprint: cloneFingerprint(fingerprint),
|
fingerprint: cloneFingerprint(fingerprint),
|
||||||
jobs: siblings.map((job) => ({
|
jobs: siblings.map((job) => ({
|
||||||
jobId: job.id,
|
jobId: job.id,
|
||||||
hoster: normalizeHoster(job.hoster),
|
hoster: normalizeHoster(job.hoster),
|
||||||
status: job.status
|
status: job.status,
|
||||||
|
currentRound: currentRoundJobs.has(job)
|
||||||
}))
|
}))
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return { groups, touchedJobs };
|
return { groups, touchedJobs, revokedHosters };
|
||||||
}
|
}
|
||||||
|
|
||||||
function markCompleted(queueJobs, job, platform) {
|
function markCompleted(queueJobs, job, platform) {
|
||||||
const siblings = relatedJobs(queueJobs, job, platform);
|
const siblings = relatedJobs(queueJobs, job, platform);
|
||||||
if (siblings.length === 0) return [];
|
if (siblings.length === 0) return [];
|
||||||
const requiredHosters = storedRequiredHosters(siblings);
|
const requiredHosters = storedRequiredHosters(siblings);
|
||||||
const completed = new Set(completedHosters(siblings, requiredHosters));
|
const provisional = new Set(provisionalHosters(siblings, requiredHosters));
|
||||||
const hoster = normalizeHoster(job.hoster);
|
const hoster = normalizeHoster(job.hoster);
|
||||||
if (requiredHosters.includes(hoster)) completed.add(hoster);
|
if (requiredHosters.includes(hoster)) provisional.add(hoster);
|
||||||
const orderedCompleted = requiredHosters.filter((required) => completed.has(required));
|
const orderedConfirmed = confirmedHosters(siblings, requiredHosters);
|
||||||
|
const orderedProvisional = requiredHosters.filter((required) => provisional.has(required));
|
||||||
for (const sibling of siblings) {
|
for (const sibling of siblings) {
|
||||||
sibling.sourceCleanupCompletedHosters = [...orderedCompleted];
|
sibling.sourceCleanupMetadataVersion = metadataVersion;
|
||||||
|
sibling.sourceCleanupConfirmedHosters = [...orderedConfirmed];
|
||||||
|
sibling.sourceCleanupProvisionalHosters = [...orderedProvisional];
|
||||||
|
delete sibling.sourceCleanupCompletedHosters;
|
||||||
}
|
}
|
||||||
return siblings;
|
return siblings;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function persistRoundCompletions(queueJobs, options = {}) {
|
||||||
|
if (!Array.isArray(queueJobs) || typeof options.persist !== 'function') return false;
|
||||||
|
const historyPersisted = options.historyPersisted === true;
|
||||||
|
const groupsByToken = new Map();
|
||||||
|
for (const job of queueJobs) {
|
||||||
|
if (!job || typeof job.sourceCleanupToken !== 'string' || !job.sourceCleanupToken) continue;
|
||||||
|
if (!groupsByToken.has(job.sourceCleanupToken)) groupsByToken.set(job.sourceCleanupToken, []);
|
||||||
|
groupsByToken.get(job.sourceCleanupToken).push(job);
|
||||||
|
}
|
||||||
|
const snapshots = [];
|
||||||
|
for (const jobs of groupsByToken.values()) {
|
||||||
|
const requiredHosters = storedRequiredHosters(jobs);
|
||||||
|
const confirmed = confirmedHosters(jobs, requiredHosters);
|
||||||
|
const provisional = provisionalHosters(jobs, requiredHosters);
|
||||||
|
const promoted = historyPersisted
|
||||||
|
? uniqueHosters([...confirmed, ...provisional])
|
||||||
|
: confirmed;
|
||||||
|
const orderedPromoted = requiredHosters.filter((hoster) => promoted.includes(hoster));
|
||||||
|
for (const job of jobs) {
|
||||||
|
snapshots.push({
|
||||||
|
job,
|
||||||
|
confirmedHosters: job.sourceCleanupMetadataVersion === metadataVersion
|
||||||
|
? uniqueHosters(job.sourceCleanupConfirmedHosters)
|
||||||
|
: []
|
||||||
|
});
|
||||||
|
job.sourceCleanupMetadataVersion = metadataVersion;
|
||||||
|
job.sourceCleanupConfirmedHosters = [...orderedPromoted];
|
||||||
|
job.sourceCleanupProvisionalHosters = [];
|
||||||
|
delete job.sourceCleanupCompletedHosters;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let persisted = false;
|
||||||
|
try {
|
||||||
|
persisted = (await options.persist()) === true;
|
||||||
|
} catch {}
|
||||||
|
if (!persisted) {
|
||||||
|
for (const snapshot of snapshots) {
|
||||||
|
snapshot.job.sourceCleanupMetadataVersion = metadataVersion;
|
||||||
|
snapshot.job.sourceCleanupConfirmedHosters = [...snapshot.confirmedHosters];
|
||||||
|
snapshot.job.sourceCleanupProvisionalHosters = [];
|
||||||
|
delete snapshot.job.sourceCleanupCompletedHosters;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return persisted;
|
||||||
|
}
|
||||||
|
|
||||||
function removeRequirement(queueJobs, job, platform) {
|
function removeRequirement(queueJobs, job, platform) {
|
||||||
if (!job || protectedStatuses.has(job.status)) return [];
|
if (!job || protectedStatuses.has(job.status)) return [];
|
||||||
const siblings = relatedJobs(queueJobs, job, platform);
|
const siblings = relatedJobs(queueJobs, job, platform);
|
||||||
@@ -176,13 +254,20 @@
|
|||||||
const required = Array.isArray(sibling.sourceCleanupRequiredHosters)
|
const required = Array.isArray(sibling.sourceCleanupRequiredHosters)
|
||||||
? sibling.sourceCleanupRequiredHosters
|
? sibling.sourceCleanupRequiredHosters
|
||||||
: [];
|
: [];
|
||||||
const completed = Array.isArray(sibling.sourceCleanupCompletedHosters)
|
const confirmed = sibling.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(sibling.sourceCleanupConfirmedHosters)
|
||||||
? sibling.sourceCleanupCompletedHosters
|
? sibling.sourceCleanupConfirmedHosters
|
||||||
: [];
|
: [];
|
||||||
|
const provisional = sibling.sourceCleanupMetadataVersion === metadataVersion && Array.isArray(sibling.sourceCleanupProvisionalHosters)
|
||||||
|
? sibling.sourceCleanupProvisionalHosters
|
||||||
|
: [];
|
||||||
|
sibling.sourceCleanupMetadataVersion = metadataVersion;
|
||||||
sibling.sourceCleanupRequiredHosters = uniqueHosters(required)
|
sibling.sourceCleanupRequiredHosters = uniqueHosters(required)
|
||||||
.filter((hoster) => hoster !== removedHoster);
|
.filter((hoster) => hoster !== removedHoster);
|
||||||
sibling.sourceCleanupCompletedHosters = uniqueHosters(completed)
|
sibling.sourceCleanupConfirmedHosters = uniqueHosters(confirmed)
|
||||||
.filter((hoster) => hoster !== removedHoster);
|
.filter((hoster) => hoster !== removedHoster);
|
||||||
|
sibling.sourceCleanupProvisionalHosters = uniqueHosters(provisional)
|
||||||
|
.filter((hoster) => hoster !== removedHoster);
|
||||||
|
delete sibling.sourceCleanupCompletedHosters;
|
||||||
}
|
}
|
||||||
return siblings;
|
return siblings;
|
||||||
}
|
}
|
||||||
@@ -211,6 +296,7 @@
|
|||||||
const api = {
|
const api = {
|
||||||
prepareGroups,
|
prepareGroups,
|
||||||
markCompleted,
|
markCompleted,
|
||||||
|
persistRoundCompletions,
|
||||||
removeRequirement,
|
removeRequirement,
|
||||||
applyFingerprints
|
applyFingerprints
|
||||||
};
|
};
|
||||||
|
|||||||
+43
-23
@@ -83,30 +83,29 @@ function createSourceFileCleanup(options) {
|
|||||||
|
|
||||||
function createManifest(input, canonicalFile) {
|
function createManifest(input, canonicalFile) {
|
||||||
const requiredHosters = uniqueStrings(input.requiredHosters);
|
const requiredHosters = uniqueStrings(input.requiredHosters);
|
||||||
const completedHosters = new Set(uniqueStrings(input.completedHosters));
|
const confirmedHosters = new Set(
|
||||||
|
uniqueStrings(input.confirmedHosters).filter((hoster) => requiredHosters.includes(hoster))
|
||||||
|
);
|
||||||
const jobs = new Map();
|
const jobs = new Map();
|
||||||
|
|
||||||
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
||||||
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
||||||
const status = completedHosters.has(job.hoster) ? 'done' : normalizeStatus(job.status);
|
const currentRound = job.currentRound !== false;
|
||||||
|
const status = currentRound ? 'pending' : normalizeStatus(job.status);
|
||||||
jobs.set(job.jobId, Object.freeze({
|
jobs.set(job.jobId, Object.freeze({
|
||||||
jobId: job.jobId,
|
jobId: job.jobId,
|
||||||
hoster: job.hoster,
|
hoster: job.hoster,
|
||||||
status
|
status,
|
||||||
|
currentRound
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const hoster of completedHosters) {
|
|
||||||
if (requiredHosters.includes(hoster)) continue;
|
|
||||||
completedHosters.delete(hoster);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
token: input.token || input.sourceCleanupToken,
|
token: input.token || input.sourceCleanupToken,
|
||||||
file: path.resolve(input.file),
|
file: path.resolve(input.file),
|
||||||
canonicalFile,
|
canonicalFile,
|
||||||
requiredHosters: Object.freeze(requiredHosters),
|
requiredHosters: Object.freeze(requiredHosters),
|
||||||
completedHosters,
|
confirmedHosters,
|
||||||
jobs,
|
jobs,
|
||||||
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
|
suppliedFingerprint: isFingerprint(input.fingerprint) ? cloneFingerprint(input.fingerprint) : null,
|
||||||
fingerprint: null,
|
fingerprint: null,
|
||||||
@@ -152,16 +151,32 @@ function createSourceFileCleanup(options) {
|
|||||||
...existing.requiredHosters,
|
...existing.requiredHosters,
|
||||||
...input.requiredHosters
|
...input.requiredHosters
|
||||||
]));
|
]));
|
||||||
for (const hoster of uniqueStrings(input.completedHosters)) {
|
const incomingJobs = Array.isArray(input.jobs) ? input.jobs : [];
|
||||||
if (existing.requiredHosters.includes(hoster)) existing.completedHosters.add(hoster);
|
const currentRoundHosters = new Set(
|
||||||
|
incomingJobs
|
||||||
|
.filter((job) => job && typeof job.hoster === 'string' && job.currentRound !== false)
|
||||||
|
.map((job) => job.hoster)
|
||||||
|
);
|
||||||
|
for (const hoster of currentRoundHosters) existing.confirmedHosters.delete(hoster);
|
||||||
|
for (const hoster of uniqueStrings(input.confirmedHosters)) {
|
||||||
|
if (existing.requiredHosters.includes(hoster) && !currentRoundHosters.has(hoster)) {
|
||||||
|
existing.confirmedHosters.add(hoster);
|
||||||
}
|
}
|
||||||
for (const job of Array.isArray(input.jobs) ? input.jobs : []) {
|
}
|
||||||
|
for (const job of incomingJobs) {
|
||||||
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
if (!job || typeof job.jobId !== 'string' || typeof job.hoster !== 'string') continue;
|
||||||
const previous = existing.jobs.get(job.jobId);
|
const previous = existing.jobs.get(job.jobId);
|
||||||
const status = existing.completedHosters.has(job.hoster)
|
const incomingCurrentRound = job.currentRound !== false;
|
||||||
? 'done'
|
const currentRound = Boolean((previous && previous.currentRound) || incomingCurrentRound);
|
||||||
|
const status = incomingCurrentRound
|
||||||
|
? 'pending'
|
||||||
: normalizeStatus(previous ? previous.status : job.status);
|
: normalizeStatus(previous ? previous.status : job.status);
|
||||||
existing.jobs.set(job.jobId, Object.freeze({ jobId: job.jobId, hoster: job.hoster, status }));
|
existing.jobs.set(job.jobId, Object.freeze({
|
||||||
|
jobId: job.jobId,
|
||||||
|
hoster: job.hoster,
|
||||||
|
status,
|
||||||
|
currentRound
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
fingerprints[token] = cloneFingerprint(existing.fingerprint);
|
fingerprints[token] = cloneFingerprint(existing.fingerprint);
|
||||||
continue;
|
continue;
|
||||||
@@ -194,10 +209,9 @@ function createSourceFileCleanup(options) {
|
|||||||
const manifest = groups.get(token);
|
const manifest = groups.get(token);
|
||||||
if (!manifest || manifest.finalizationPromise) return false;
|
if (!manifest || manifest.finalizationPromise) return false;
|
||||||
const job = manifest.jobs.get(event.jobId);
|
const job = manifest.jobs.get(event.jobId);
|
||||||
if (!job || job.hoster !== event.hoster) return false;
|
if (!job || !job.currentRound || job.hoster !== event.hoster) return false;
|
||||||
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
|
if (typeof event.file === 'string' && canonicalize(event.file) !== manifest.canonicalFile) return false;
|
||||||
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
|
manifest.jobs.set(event.jobId, Object.freeze({ ...job, status: event.status }));
|
||||||
if (event.status === 'done') manifest.completedHosters.add(event.hoster);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,7 +220,7 @@ function createSourceFileCleanup(options) {
|
|||||||
for (const manifest of groups.values()) {
|
for (const manifest of groups.values()) {
|
||||||
if (manifest.finalizationPromise) continue;
|
if (manifest.finalizationPromise) continue;
|
||||||
const job = manifest.jobs.get(jobId);
|
const job = manifest.jobs.get(jobId);
|
||||||
if (!job) continue;
|
if (!job || !job.currentRound) continue;
|
||||||
manifest.jobs.set(jobId, Object.freeze({ ...job, status: 'skipped' }));
|
manifest.jobs.set(jobId, Object.freeze({ ...job, status: 'skipped' }));
|
||||||
changed = true;
|
changed = true;
|
||||||
}
|
}
|
||||||
@@ -216,11 +230,17 @@ function createSourceFileCleanup(options) {
|
|||||||
function blockingStatuses(manifest) {
|
function blockingStatuses(manifest) {
|
||||||
const blocking = [];
|
const blocking = [];
|
||||||
for (const hoster of manifest.requiredHosters) {
|
for (const hoster of manifest.requiredHosters) {
|
||||||
if (manifest.completedHosters.has(hoster)) continue;
|
const jobs = [...manifest.jobs.values()].filter((job) => job.hoster === hoster);
|
||||||
const statuses = [...manifest.jobs.values()]
|
const currentJobs = jobs.filter((job) => job.currentRound);
|
||||||
.filter((job) => job.hoster === hoster)
|
if (currentJobs.length > 0) {
|
||||||
.map((job) => job.status);
|
const currentBlocker = currentJobs.find((job) => job.status !== 'done');
|
||||||
if (statuses.includes('done')) continue;
|
if (!currentBlocker) continue;
|
||||||
|
const status = currentJobs.map((job) => job.status).find((value) => value !== 'pending') || 'pending';
|
||||||
|
blocking.push({ hoster, status });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (manifest.confirmedHosters.has(hoster)) continue;
|
||||||
|
const statuses = jobs.map((job) => job.status).filter((status) => status !== 'done');
|
||||||
const status = statuses.find((value) => value !== 'pending') || 'pending';
|
const status = statuses.find((value) => value !== 'pending') || 'pending';
|
||||||
blocking.push({ hoster, status });
|
blocking.push({ hoster, status });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ let lastSessionSummary = null;
|
|||||||
let sourceDeleteJournal = null;
|
let sourceDeleteJournal = null;
|
||||||
const pendingUploadFinalizations = new Map();
|
const pendingUploadFinalizations = new Map();
|
||||||
|
|
||||||
function requestUploadFinalization(summary) {
|
function requestUploadFinalization(summary, historyPersisted) {
|
||||||
const finalizationId = `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
const finalizationId = `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
@@ -136,7 +136,7 @@ function requestUploadFinalization(summary) {
|
|||||||
resolve(value);
|
resolve(value);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
safeSend('upload-batch-done', { summary, finalizationId });
|
safeSend('upload-batch-done', { summary, finalizationId, historyPersisted: historyPersisted === true });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const activeUploadProducerTrackers = new Set();
|
const activeUploadProducerTrackers = new Set();
|
||||||
@@ -2276,7 +2276,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
for (const value of _progressByJob.values()) finalProgressBatch.push(value);
|
for (const value of _progressByJob.values()) finalProgressBatch.push(value);
|
||||||
_progressByJob.clear();
|
_progressByJob.clear();
|
||||||
if (finalProgressBatch.length) safeSend('upload-progress-batch', finalProgressBatch);
|
if (finalProgressBatch.length) safeSend('upload-progress-batch', finalProgressBatch);
|
||||||
const queuePersisted = await requestUploadFinalization(summary);
|
const queuePersisted = await requestUploadFinalization(summary, historyPersisted);
|
||||||
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
|
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
|
||||||
if (!queuePersisted) debugLog('upload finalization blocked: renderer queue acknowledgement missing');
|
if (!queuePersisted) debugLog('upload finalization blocked: renderer queue acknowledgement missing');
|
||||||
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted });
|
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted });
|
||||||
@@ -2978,6 +2978,10 @@ ipcMain.handle('complete-upload-finalization', async (_event, payload) => {
|
|||||||
const finalizationId = payload && payload.finalizationId;
|
const finalizationId = payload && payload.finalizationId;
|
||||||
const pending = finalizationId && pendingUploadFinalizations.get(finalizationId);
|
const pending = finalizationId && pendingUploadFinalizations.get(finalizationId);
|
||||||
if (!pending) return false;
|
if (!pending) return false;
|
||||||
|
if (payload.ready === false) {
|
||||||
|
pending.resolve(false);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
await configStore.savePendingQueue(payload.pendingQueue ?? null);
|
await configStore.savePendingQueue(payload.pendingQueue ?? null);
|
||||||
pending.resolve(true);
|
pending.resolve(true);
|
||||||
|
|||||||
+85
-19
@@ -60,6 +60,8 @@ let selectedUploadHosters = [];
|
|||||||
let config = { hosters: {}, hosterSettings: {}, globalSettings: {} };
|
let config = { hosters: {}, hosterSettings: {}, globalSettings: {} };
|
||||||
let hosterSettings = {};
|
let hosterSettings = {};
|
||||||
let uploading = false;
|
let uploading = false;
|
||||||
|
let sourceCleanupFinalizationPending = false;
|
||||||
|
let sourceCleanupRevocationPending = false;
|
||||||
let healthCheckRunning = false;
|
let healthCheckRunning = false;
|
||||||
|
|
||||||
let _rLongTasks = 0, _rLongTaskMax = 0, _rFrameLast = 0, _rFrameWorst = 0, _rFrameCount = 0, _rFrameJank = 0, _rPerfLastLog = 0, _rPerfWindowStart = 0;
|
let _rLongTasks = 0, _rLongTaskMax = 0, _rFrameLast = 0, _rFrameWorst = 0, _rFrameCount = 0, _rFrameJank = 0, _rPerfLastLog = 0, _rPerfWindowStart = 0;
|
||||||
@@ -445,16 +447,18 @@ async function init() {
|
|||||||
}
|
}
|
||||||
window.api.onUploadBatchDone(async (data) => {
|
window.api.onUploadBatchDone(async (data) => {
|
||||||
const summary = data && data.summary ? data.summary : data;
|
const summary = data && data.summary ? data.summary : data;
|
||||||
|
const requiresFinalization = Boolean(data && data.finalizationId && window.api.completeUploadFinalization);
|
||||||
|
if (!requiresFinalization) {
|
||||||
handleBatchDone(summary);
|
handleBatchDone(summary);
|
||||||
if (data && data.finalizationId && window.api.completeUploadFinalization) {
|
return;
|
||||||
if (_doneRemovalCoalescer) _doneRemovalCoalescer.drainSync();
|
}
|
||||||
queuePersistThrottle.cancel();
|
sourceCleanupFinalizationPending = true;
|
||||||
await window.api.completeUploadFinalization({
|
try {
|
||||||
finalizationId: data.finalizationId,
|
handleBatchDone(summary, { deferPersistence: true });
|
||||||
pendingQueue: queueJobs.some((job) => !['done', 'skipped'].includes(job.status))
|
await completeSourceCleanupFinalization(data);
|
||||||
? buildPersistedQueueState()
|
} finally {
|
||||||
: null
|
sourceCleanupFinalizationPending = false;
|
||||||
});
|
updateQueueActionButtons();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
window.api.onUploadStats((data) => {
|
window.api.onUploadStats((data) => {
|
||||||
@@ -1234,9 +1238,13 @@ function restoreQueueStateFromConfig() {
|
|||||||
failureDetails: job.failureDetails || null,
|
failureDetails: job.failureDetails || null,
|
||||||
interrupted: interruptedJobIds.size > 0 && !['done', 'error', 'skipped'].includes(job.status),
|
interrupted: interruptedJobIds.size > 0 && !['done', 'error', 'skipped'].includes(job.status),
|
||||||
result: job.result || null,
|
result: job.result || null,
|
||||||
|
sourceCleanupMetadataVersion: job.sourceCleanupMetadataVersion === 2 ? 2 : null,
|
||||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||||
sourceCleanupRequiredHosters: Array.isArray(job.sourceCleanupRequiredHosters) ? [...job.sourceCleanupRequiredHosters] : [],
|
sourceCleanupRequiredHosters: Array.isArray(job.sourceCleanupRequiredHosters) ? [...job.sourceCleanupRequiredHosters] : [],
|
||||||
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
sourceCleanupConfirmedHosters: job.sourceCleanupMetadataVersion === 2 && Array.isArray(job.sourceCleanupConfirmedHosters)
|
||||||
|
? [...job.sourceCleanupConfirmedHosters]
|
||||||
|
: [],
|
||||||
|
sourceCleanupProvisionalHosters: [],
|
||||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||||
attempt: 0,
|
attempt: 0,
|
||||||
maxAttempts: job.maxAttempts || 0,
|
maxAttempts: job.maxAttempts || 0,
|
||||||
@@ -1311,9 +1319,12 @@ function buildPersistedQueueState() {
|
|||||||
error: isTerminal ? (job.error || null) : null,
|
error: isTerminal ? (job.error || null) : null,
|
||||||
failureDetails: isTerminal ? (job.failureDetails || null) : null,
|
failureDetails: isTerminal ? (job.failureDetails || null) : null,
|
||||||
result: isTerminal ? (job.result || null) : null,
|
result: isTerminal ? (job.result || null) : null,
|
||||||
|
sourceCleanupMetadataVersion: job.sourceCleanupToken ? 2 : null,
|
||||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||||
sourceCleanupRequiredHosters: Array.isArray(job.sourceCleanupRequiredHosters) ? [...job.sourceCleanupRequiredHosters] : [],
|
sourceCleanupRequiredHosters: Array.isArray(job.sourceCleanupRequiredHosters) ? [...job.sourceCleanupRequiredHosters] : [],
|
||||||
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
sourceCleanupConfirmedHosters: job.sourceCleanupMetadataVersion === 2 && Array.isArray(job.sourceCleanupConfirmedHosters)
|
||||||
|
? [...job.sourceCleanupConfirmedHosters]
|
||||||
|
: [],
|
||||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||||
maxAttempts: job.maxAttempts || 0
|
maxAttempts: job.maxAttempts || 0
|
||||||
};
|
};
|
||||||
@@ -1527,7 +1538,7 @@ function updateStartButton() {
|
|||||||
const hosters = getSelectedHosters();
|
const hosters = getSelectedHosters();
|
||||||
const hasQueuedJobs = queueJobs.some(isStartableQueueJob);
|
const hasQueuedJobs = queueJobs.some(isStartableQueueJob);
|
||||||
const canBuildQueueFromSelection = selectedFiles.length > 0 && hosters.length > 0;
|
const canBuildQueueFromSelection = selectedFiles.length > 0 && hosters.length > 0;
|
||||||
btn.disabled = uploading || !(hasQueuedJobs || canBuildQueueFromSelection);
|
btn.disabled = uploading || sourceCleanupFinalizationPending || !(hasQueuedJobs || canBuildQueueFromSelection);
|
||||||
}
|
}
|
||||||
|
|
||||||
const _UPLOAD_SELECTION_STATUSES = new Set(['done', 'error', 'aborted', 'skipped']);
|
const _UPLOAD_SELECTION_STATUSES = new Set(['done', 'error', 'aborted', 'skipped']);
|
||||||
@@ -1563,8 +1574,8 @@ function updateQueueActionButtons() {
|
|||||||
const moveDownBtn = document.getElementById('moveDownBtn');
|
const moveDownBtn = document.getElementById('moveDownBtn');
|
||||||
const moveBottomBtn = document.getElementById('moveBottomBtn');
|
const moveBottomBtn = document.getElementById('moveBottomBtn');
|
||||||
|
|
||||||
if (startSelectedBtn) startSelectedBtn.disabled = uploading || !hasStartableSelection;
|
if (startSelectedBtn) startSelectedBtn.disabled = uploading || sourceCleanupFinalizationPending || !hasStartableSelection;
|
||||||
if (reuploadBtn) reuploadBtn.disabled = !hasUploadSelection;
|
if (reuploadBtn) reuploadBtn.disabled = sourceCleanupFinalizationPending || !hasUploadSelection;
|
||||||
if (abortSelectedBtn) abortSelectedBtn.disabled = !hasAbortSelection;
|
if (abortSelectedBtn) abortSelectedBtn.disabled = !hasAbortSelection;
|
||||||
if (finishStopBtn) finishStopBtn.disabled = !uploading;
|
if (finishStopBtn) finishStopBtn.disabled = !uploading;
|
||||||
if (abortAllBtn) abortAllBtn.disabled = !uploading;
|
if (abortAllBtn) abortAllBtn.disabled = !uploading;
|
||||||
@@ -2894,8 +2905,54 @@ function getSelectedJobLinks() {
|
|||||||
|
|
||||||
// --- Upload ---
|
// --- Upload ---
|
||||||
function prepareSourceCleanup(jobs) {
|
function prepareSourceCleanup(jobs) {
|
||||||
if (!config.globalSettings?.deleteSourceAfterSuccessfulUpload || !window.SourceCleanupPolicy) return { groups: [] };
|
if (!window.SourceCleanupPolicy) return { groups: [] };
|
||||||
return window.SourceCleanupPolicy.prepareGroups(queueJobs, jobs, () => window.crypto.randomUUID(), 'win32');
|
const enabled = config.globalSettings?.deleteSourceAfterSuccessfulUpload === true;
|
||||||
|
const hasMetadata = Array.isArray(jobs) && jobs.some((job) => job?.sourceCleanupToken);
|
||||||
|
if (!enabled && !hasMetadata) return { groups: [] };
|
||||||
|
const preparation = window.SourceCleanupPolicy.prepareGroups(queueJobs, jobs, () => window.crypto.randomUUID(), 'win32');
|
||||||
|
return enabled ? preparation : { ...preparation, groups: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function persistSourceCleanupRevocations(preparation) {
|
||||||
|
if (Array.isArray(preparation?.revokedHosters) && preparation.revokedHosters.length > 0) {
|
||||||
|
sourceCleanupRevocationPending = true;
|
||||||
|
}
|
||||||
|
if (!sourceCleanupRevocationPending) return;
|
||||||
|
queuePersistThrottle.cancel();
|
||||||
|
await persistQueueStateNow();
|
||||||
|
await flushConfigWrites();
|
||||||
|
sourceCleanupRevocationPending = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function completeSourceCleanupFinalization(data) {
|
||||||
|
if (!data?.finalizationId || !window.api.completeUploadFinalization) return false;
|
||||||
|
if (_doneRemovalCoalescer) _doneRemovalCoalescer.drainSync();
|
||||||
|
queuePersistThrottle.cancel();
|
||||||
|
let writesReady = true;
|
||||||
|
try {
|
||||||
|
await flushConfigWrites();
|
||||||
|
} catch {
|
||||||
|
writesReady = false;
|
||||||
|
}
|
||||||
|
const persist = async () => {
|
||||||
|
if (!writesReady) {
|
||||||
|
try {
|
||||||
|
await window.api.completeUploadFinalization({ finalizationId: data.finalizationId, ready: false });
|
||||||
|
} catch {}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return window.api.completeUploadFinalization({
|
||||||
|
finalizationId: data.finalizationId,
|
||||||
|
pendingQueue: queueJobs.some((job) => !['done', 'skipped'].includes(job.status))
|
||||||
|
? buildPersistedQueueState()
|
||||||
|
: null
|
||||||
|
});
|
||||||
|
};
|
||||||
|
if (!window.SourceCleanupPolicy?.persistRoundCompletions) return Boolean(await persist());
|
||||||
|
return window.SourceCleanupPolicy.persistRoundCompletions(queueJobs, {
|
||||||
|
historyPersisted: data.historyPersisted === true,
|
||||||
|
persist
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function serializeUploadJob(job) {
|
function serializeUploadJob(job) {
|
||||||
@@ -2904,15 +2961,16 @@ function serializeUploadJob(job) {
|
|||||||
file: job.file,
|
file: job.file,
|
||||||
fileName: job.fileName,
|
fileName: job.fileName,
|
||||||
hoster: job.hoster,
|
hoster: job.hoster,
|
||||||
|
sourceCleanupMetadataVersion: job.sourceCleanupToken ? 2 : null,
|
||||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||||
sourceCleanupRequiredHosters: job.sourceCleanupRequiredHosters || [],
|
sourceCleanupRequiredHosters: job.sourceCleanupRequiredHosters || [],
|
||||||
sourceCleanupCompletedHosters: job.sourceCleanupCompletedHosters || [],
|
sourceCleanupConfirmedHosters: job.sourceCleanupConfirmedHosters || [],
|
||||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null
|
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function startUpload(opts) {
|
async function startUpload(opts) {
|
||||||
if (uploading) return;
|
if (uploading || sourceCleanupFinalizationPending) return;
|
||||||
if (!(opts && opts._restoredAutoStart)) cancelStartupQueueAutoStart();
|
if (!(opts && opts._restoredAutoStart)) cancelStartupQueueAutoStart();
|
||||||
if (!(opts && opts._autoRetry)) _cancelAutoRetry(true);
|
if (!(opts && opts._autoRetry)) _cancelAutoRetry(true);
|
||||||
else _cancelAutoRetry(false);
|
else _cancelAutoRetry(false);
|
||||||
@@ -2950,6 +3008,7 @@ async function startUpload(opts) {
|
|||||||
updateQueueActionButtons();
|
updateQueueActionButtons();
|
||||||
renderQueueTable();
|
renderQueueTable();
|
||||||
updateStatusBar();
|
updateStatusBar();
|
||||||
|
await persistSourceCleanupRevocations(cleanupPreparation);
|
||||||
|
|
||||||
const uploadPayload = {
|
const uploadPayload = {
|
||||||
hosters,
|
hosters,
|
||||||
@@ -2991,6 +3050,7 @@ function _markSkippedJobs(result) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function startSelectedUpload(explicitJobs) {
|
async function startSelectedUpload(explicitJobs) {
|
||||||
|
if (sourceCleanupFinalizationPending) return;
|
||||||
const scopedJobs = Array.isArray(explicitJobs) ? explicitJobs : _getVisibleSelectedQueueJobs();
|
const scopedJobs = Array.isArray(explicitJobs) ? explicitJobs : _getVisibleSelectedQueueJobs();
|
||||||
if (uploading) {
|
if (uploading) {
|
||||||
_hydrateMissingJobSizes();
|
_hydrateMissingJobSizes();
|
||||||
@@ -3008,6 +3068,7 @@ async function startSelectedUpload(explicitJobs) {
|
|||||||
renderQueueTable();
|
renderQueueTable();
|
||||||
let result = null;
|
let result = null;
|
||||||
try {
|
try {
|
||||||
|
await persistSourceCleanupRevocations(cleanupPreparation);
|
||||||
result = await window.api.addJobsToBatch({
|
result = await window.api.addJobsToBatch({
|
||||||
jobs: addable.map(serializeUploadJob),
|
jobs: addable.map(serializeUploadJob),
|
||||||
sourceCleanupGroups: cleanupPreparation.groups
|
sourceCleanupGroups: cleanupPreparation.groups
|
||||||
@@ -3071,6 +3132,7 @@ async function startSelectedUpload(explicitJobs) {
|
|||||||
updateQueueActionButtons();
|
updateQueueActionButtons();
|
||||||
renderQueueTable();
|
renderQueueTable();
|
||||||
updateStatusBar();
|
updateStatusBar();
|
||||||
|
await persistSourceCleanupRevocations(cleanupPreparation);
|
||||||
|
|
||||||
const uploadPayload = {
|
const uploadPayload = {
|
||||||
hosters,
|
hosters,
|
||||||
@@ -3237,7 +3299,7 @@ function _handleProgressImpl(data) {
|
|||||||
persistQueueStateSoon();
|
persistQueueStateSoon();
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleBatchDone(summary) {
|
function handleBatchDone(summary, options = {}) {
|
||||||
uploading = false;
|
uploading = false;
|
||||||
applySummaryResults(summary);
|
applySummaryResults(summary);
|
||||||
_deletedJobIds.clear(); // Free memory — stale IDs no longer needed after batch completes
|
_deletedJobIds.clear(); // Free memory — stale IDs no longer needed after batch completes
|
||||||
@@ -3312,8 +3374,10 @@ function handleBatchDone(summary) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!options.deferPersistence) {
|
||||||
if (queueJobs.some((job) => !['done', 'skipped'].includes(job.status))) persistQueueStateSoon(true);
|
if (queueJobs.some((job) => !['done', 'skipped'].includes(job.status))) persistQueueStateSoon(true);
|
||||||
else clearPersistedQueueStateSoon();
|
else clearPersistedQueueStateSoon();
|
||||||
|
}
|
||||||
|
|
||||||
lastUploadStats = { state: 'idle', globalSpeedKbs: 0, totalBytes: lastUploadStats.totalBytes, elapsed: lastUploadStats.elapsed, activeJobs: 0 };
|
lastUploadStats = { state: 'idle', globalSpeedKbs: 0, totalBytes: lastUploadStats.totalBytes, elapsed: lastUploadStats.elapsed, activeJobs: 0 };
|
||||||
updateStatusBar();
|
updateStatusBar();
|
||||||
@@ -3408,6 +3472,7 @@ function _maybeShowBatchSummary(summary) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function _retryFailedFromBuckets(buckets, transientOnly) {
|
function _retryFailedFromBuckets(buckets, transientOnly) {
|
||||||
|
if (sourceCleanupFinalizationPending) return;
|
||||||
const cats = transientOnly ? ['hoster-transient', 'network', 'unknown'] : ['hoster-transient', 'network', 'unknown', 'file-rejected', 'account-error'];
|
const cats = transientOnly ? ['hoster-transient', 'network', 'unknown'] : ['hoster-transient', 'network', 'unknown', 'file-rejected', 'account-error'];
|
||||||
const toRetry = [];
|
const toRetry = [];
|
||||||
for (const cat of cats) {
|
for (const cat of cats) {
|
||||||
@@ -3540,6 +3605,7 @@ async function copyJobLogToClipboard() {
|
|||||||
|
|
||||||
// --- Retry ---
|
// --- Retry ---
|
||||||
async function retrySelectedJobs() {
|
async function retrySelectedJobs() {
|
||||||
|
if (sourceCleanupFinalizationPending) return;
|
||||||
_normalizeQueueSelectionToVisible();
|
_normalizeQueueSelectionToVisible();
|
||||||
const retryJobs = [];
|
const retryJobs = [];
|
||||||
// Build a Set for O(1) selectedFiles dedup below.
|
// Build a Set for O(1) selectedFiles dedup below.
|
||||||
|
|||||||
@@ -48,13 +48,13 @@ test('prepareGroups creates one Windows manifest with a stable token and immutab
|
|||||||
token: 'cleanup-1',
|
token: 'cleanup-1',
|
||||||
file: 'C:\\Uploads\\Movie.MKV',
|
file: 'C:\\Uploads\\Movie.MKV',
|
||||||
requiredHosters: ['doodstream.com', 'voe.sx', 'vidmoly.me', 'byse.sx'],
|
requiredHosters: ['doodstream.com', 'voe.sx', 'vidmoly.me', 'byse.sx'],
|
||||||
completedHosters: ['doodstream.com', 'voe.sx'],
|
confirmedHosters: [],
|
||||||
fingerprint: null,
|
fingerprint: null,
|
||||||
jobs: [
|
jobs: [
|
||||||
{ jobId: 'job-doodstream', hoster: 'doodstream.com', status: 'done' },
|
{ jobId: 'job-doodstream', hoster: 'doodstream.com', status: 'done', currentRound: false },
|
||||||
{ jobId: 'job-voe', hoster: 'voe.sx', status: 'done' },
|
{ jobId: 'job-voe', hoster: 'voe.sx', status: 'done', currentRound: false },
|
||||||
{ jobId: 'job-vidmoly', hoster: 'vidmoly.me', status: 'error' },
|
{ jobId: 'job-vidmoly', hoster: 'vidmoly.me', status: 'error', currentRound: true },
|
||||||
{ jobId: 'job-byse', hoster: 'byse.sx', status: 'preview' }
|
{ jobId: 'job-byse', hoster: 'byse.sx', status: 'preview', currentRound: false }
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
assert.deepEqual(queueJobs.map((job) => job.sourceCleanupToken), [
|
assert.deepEqual(queueJobs.map((job) => job.sourceCleanupToken), [
|
||||||
@@ -74,11 +74,13 @@ test('prepareGroups creates one Windows manifest with a stable token and immutab
|
|||||||
assert.doesNotThrow(() => JSON.stringify(prepared.groups));
|
assert.doesNotThrow(() => JSON.stringify(prepared.groups));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('prepareGroups reuses persisted metadata across a partial retry', () => {
|
test('prepareGroups reuses confirmed metadata across a partial retry', () => {
|
||||||
const queueJobs = queueFixture();
|
const queueJobs = queueFixture();
|
||||||
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-1', 'win32');
|
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-1', 'win32');
|
||||||
queueJobs[0].sourceCleanupCompletedHosters = ['doodstream.com', 'voe.sx'];
|
for (const job of queueJobs) {
|
||||||
queueJobs[1].sourceCleanupCompletedHosters = ['doodstream.com', 'voe.sx'];
|
job.sourceCleanupMetadataVersion = 2;
|
||||||
|
job.sourceCleanupConfirmedHosters = ['doodstream.com', 'voe.sx'];
|
||||||
|
}
|
||||||
queueJobs[0].status = 'preview';
|
queueJobs[0].status = 'preview';
|
||||||
queueJobs[1].status = 'preview';
|
queueJobs[1].status = 'preview';
|
||||||
|
|
||||||
@@ -93,7 +95,7 @@ test('prepareGroups reuses persisted metadata across a partial retry', () => {
|
|||||||
'vidmoly.me',
|
'vidmoly.me',
|
||||||
'byse.sx'
|
'byse.sx'
|
||||||
]);
|
]);
|
||||||
assert.deepEqual(prepared.groups[0].completedHosters, ['doodstream.com', 'voe.sx']);
|
assert.deepEqual(prepared.groups[0].confirmedHosters, ['doodstream.com', 'voe.sx']);
|
||||||
assert.equal(prepared.groups[0].jobs.find((job) => job.hoster === 'vidmoly.me').status, 'error');
|
assert.equal(prepared.groups[0].jobs.find((job) => job.hoster === 'vidmoly.me').status, 'error');
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -163,7 +165,7 @@ test('removeRequirement drops only an explicitly discarded unstarted hoster', ()
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('markCompleted preserves successful hosters for later retries', () => {
|
test('markCompleted keeps successful hosters provisional for the current round', () => {
|
||||||
const queueJobs = queueFixture();
|
const queueJobs = queueFixture();
|
||||||
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-1', 'win32');
|
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-1', 'win32');
|
||||||
|
|
||||||
@@ -171,11 +173,8 @@ test('markCompleted preserves successful hosters for later retries', () => {
|
|||||||
policy.markCompleted(queueJobs, queueJobs[2], 'win32');
|
policy.markCompleted(queueJobs, queueJobs[2], 'win32');
|
||||||
|
|
||||||
for (const job of queueJobs) {
|
for (const job of queueJobs) {
|
||||||
assert.deepEqual(job.sourceCleanupCompletedHosters, [
|
assert.deepEqual(job.sourceCleanupConfirmedHosters, []);
|
||||||
'doodstream.com',
|
assert.deepEqual(job.sourceCleanupProvisionalHosters, ['vidmoly.me']);
|
||||||
'voe.sx',
|
|
||||||
'vidmoly.me'
|
|
||||||
]);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -203,3 +202,127 @@ test('applyFingerprints attaches Main fingerprints by token and includes them in
|
|||||||
assert.notEqual(job.sourceCleanupFingerprint, fingerprint);
|
assert.notEqual(job.sourceCleanupFingerprint, fingerprint);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('keeps round successes provisional until history and queue persistence succeed', async () => {
|
||||||
|
const queueJobs = [
|
||||||
|
{ id: 'job-voe', file: 'C:\\Uploads\\Round.bin', hoster: 'voe.sx', status: 'preview' },
|
||||||
|
{ id: 'job-byse', file: 'C:\\Uploads\\Round.bin', hoster: 'byse.sx', status: 'error' }
|
||||||
|
];
|
||||||
|
policy.prepareGroups(queueJobs, queueJobs, () => 'cleanup-round', 'win32');
|
||||||
|
queueJobs[0].status = 'done';
|
||||||
|
|
||||||
|
policy.markCompleted(queueJobs, queueJobs[0], 'win32');
|
||||||
|
|
||||||
|
for (const job of queueJobs) {
|
||||||
|
assert.deepEqual(job.sourceCleanupConfirmedHosters, []);
|
||||||
|
assert.deepEqual(job.sourceCleanupProvisionalHosters, ['voe.sx']);
|
||||||
|
}
|
||||||
|
|
||||||
|
let persistedHosters = null;
|
||||||
|
const queuePersisted = await policy.persistRoundCompletions(queueJobs, {
|
||||||
|
historyPersisted: true,
|
||||||
|
persist: async () => {
|
||||||
|
persistedHosters = queueJobs.map((job) => [...job.sourceCleanupConfirmedHosters]);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(queuePersisted, true);
|
||||||
|
assert.deepEqual(persistedHosters, [['voe.sx'], ['voe.sx']]);
|
||||||
|
for (const job of queueJobs) {
|
||||||
|
assert.deepEqual(job.sourceCleanupConfirmedHosters, ['voe.sx']);
|
||||||
|
assert.deepEqual(job.sourceCleanupProvisionalHosters, []);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rolls back provisional promotion when the final queue save fails', async () => {
|
||||||
|
const queueJobs = [
|
||||||
|
{
|
||||||
|
id: 'job-voe',
|
||||||
|
file: 'C:\\Uploads\\Partial.bin',
|
||||||
|
hoster: 'voe.sx',
|
||||||
|
status: 'done',
|
||||||
|
sourceCleanupMetadataVersion: 2,
|
||||||
|
sourceCleanupToken: 'cleanup-partial',
|
||||||
|
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||||
|
sourceCleanupConfirmedHosters: ['voe.sx']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'job-byse',
|
||||||
|
file: 'C:\\Uploads\\Partial.bin',
|
||||||
|
hoster: 'byse.sx',
|
||||||
|
status: 'preview',
|
||||||
|
sourceCleanupMetadataVersion: 2,
|
||||||
|
sourceCleanupToken: 'cleanup-partial',
|
||||||
|
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||||
|
sourceCleanupConfirmedHosters: ['voe.sx']
|
||||||
|
}
|
||||||
|
];
|
||||||
|
policy.prepareGroups(queueJobs, [queueJobs[1]], () => 'unused', 'win32');
|
||||||
|
queueJobs[1].status = 'done';
|
||||||
|
policy.markCompleted(queueJobs, queueJobs[1], 'win32');
|
||||||
|
|
||||||
|
const queuePersisted = await policy.persistRoundCompletions(queueJobs, {
|
||||||
|
historyPersisted: true,
|
||||||
|
persist: async () => false
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(queuePersisted, false);
|
||||||
|
for (const job of queueJobs) {
|
||||||
|
assert.deepEqual(job.sourceCleanupConfirmedHosters, ['voe.sx']);
|
||||||
|
assert.deepEqual(job.sourceCleanupProvisionalHosters, []);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ignores legacy v2.1.19 completed markers when preparing a retry', () => {
|
||||||
|
const queueJobs = [{
|
||||||
|
id: 'job-voe',
|
||||||
|
file: 'C:\\Uploads\\Legacy.bin',
|
||||||
|
hoster: 'voe.sx',
|
||||||
|
status: 'error',
|
||||||
|
sourceCleanupToken: 'cleanup-legacy',
|
||||||
|
sourceCleanupRequiredHosters: ['voe.sx'],
|
||||||
|
sourceCleanupCompletedHosters: ['voe.sx']
|
||||||
|
}];
|
||||||
|
|
||||||
|
const prepared = policy.prepareGroups(queueJobs, queueJobs, () => 'unused', 'win32');
|
||||||
|
|
||||||
|
assert.deepEqual(prepared.groups[0].confirmedHosters, []);
|
||||||
|
assert.equal(queueJobs[0].sourceCleanupMetadataVersion, 2);
|
||||||
|
assert.deepEqual(queueJobs[0].sourceCleanupConfirmedHosters, []);
|
||||||
|
assert.equal(Object.prototype.hasOwnProperty.call(queueJobs[0], 'sourceCleanupCompletedHosters'), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('selecting a confirmed hoster for retry revokes its durable completion', () => {
|
||||||
|
const queueJobs = [
|
||||||
|
{
|
||||||
|
id: 'job-voe',
|
||||||
|
file: 'C:\\Uploads\\Retry.bin',
|
||||||
|
hoster: 'voe.sx',
|
||||||
|
status: 'preview',
|
||||||
|
sourceCleanupMetadataVersion: 2,
|
||||||
|
sourceCleanupToken: 'cleanup-retry',
|
||||||
|
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||||
|
sourceCleanupConfirmedHosters: ['voe.sx', 'byse.sx']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'job-byse',
|
||||||
|
file: 'C:\\Uploads\\Retry.bin',
|
||||||
|
hoster: 'byse.sx',
|
||||||
|
status: 'done',
|
||||||
|
sourceCleanupMetadataVersion: 2,
|
||||||
|
sourceCleanupToken: 'cleanup-retry',
|
||||||
|
sourceCleanupRequiredHosters: ['voe.sx', 'byse.sx'],
|
||||||
|
sourceCleanupConfirmedHosters: ['voe.sx', 'byse.sx']
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
const prepared = policy.prepareGroups(queueJobs, [queueJobs[0]], () => 'unused', 'win32');
|
||||||
|
|
||||||
|
assert.deepEqual(prepared.groups[0].confirmedHosters, ['byse.sx']);
|
||||||
|
assert.equal(prepared.groups[0].jobs[0].currentRound, true);
|
||||||
|
assert.equal(prepared.groups[0].jobs[1].currentRound, false);
|
||||||
|
for (const job of queueJobs) {
|
||||||
|
assert.deepEqual(job.sourceCleanupConfirmedHosters, ['byse.sx']);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ function group(file, overrides = {}) {
|
|||||||
token: 'cleanup-1',
|
token: 'cleanup-1',
|
||||||
file,
|
file,
|
||||||
requiredHosters: ['voe.sx', 'byse.sx'],
|
requiredHosters: ['voe.sx', 'byse.sx'],
|
||||||
completedHosters: [],
|
confirmedHosters: [],
|
||||||
jobs: [
|
jobs: [
|
||||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending' },
|
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending' },
|
||||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending' }
|
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending' }
|
||||||
@@ -81,7 +81,7 @@ test('fingerprints a regular file and keeps the registered manifest immutable',
|
|||||||
assert.equal(typeof fingerprints['cleanup-1'].ino, 'number');
|
assert.equal(typeof fingerprints['cleanup-1'].ino, 'number');
|
||||||
|
|
||||||
manifest.requiredHosters.splice(1, 1);
|
manifest.requiredHosters.splice(1, 1);
|
||||||
manifest.completedHosters.push('byse.sx');
|
manifest.confirmedHosters.push('byse.sx');
|
||||||
manifest.jobs[1].hoster = 'voe.sx';
|
manifest.jobs[1].hoster = 'voe.sx';
|
||||||
await cleanup.settle({
|
await cleanup.settle({
|
||||||
token: 'cleanup-1',
|
token: 'cleanup-1',
|
||||||
@@ -182,7 +182,13 @@ test('combines previous successes with a successful retry without relaxing other
|
|||||||
await t.test('last retry completes the immutable group', async (subtest) => {
|
await t.test('last retry completes the immutable group', async (subtest) => {
|
||||||
const { file } = await makeSource(subtest, 'retry-completes.bin');
|
const { file } = await makeSource(subtest, 'retry-completes.bin');
|
||||||
const { cleanup } = makeCleanup();
|
const { cleanup } = makeCleanup();
|
||||||
const manifest = group(file, { completedHosters: ['voe.sx'] });
|
const manifest = group(file, {
|
||||||
|
confirmedHosters: ['voe.sx'],
|
||||||
|
jobs: [
|
||||||
|
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: false },
|
||||||
|
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true }
|
||||||
|
]
|
||||||
|
});
|
||||||
await cleanup.registerGroups([manifest]);
|
await cleanup.registerGroups([manifest]);
|
||||||
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'done' });
|
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'done' });
|
||||||
|
|
||||||
@@ -196,11 +202,11 @@ test('combines previous successes with a successful retry without relaxing other
|
|||||||
const { cleanup, audits } = makeCleanup();
|
const { cleanup, audits } = makeCleanup();
|
||||||
const manifest = group(file, {
|
const manifest = group(file, {
|
||||||
requiredHosters: ['voe.sx', 'byse.sx', 'vidmoly.me'],
|
requiredHosters: ['voe.sx', 'byse.sx', 'vidmoly.me'],
|
||||||
completedHosters: ['voe.sx'],
|
confirmedHosters: ['voe.sx'],
|
||||||
jobs: [
|
jobs: [
|
||||||
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' },
|
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: false },
|
||||||
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending' },
|
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true },
|
||||||
{ jobId: 'job-vidmoly', file, hoster: 'vidmoly.me', status: 'error' }
|
{ jobId: 'job-vidmoly', file, hoster: 'vidmoly.me', status: 'error', currentRound: false }
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
await cleanup.registerGroups([manifest]);
|
await cleanup.registerGroups([manifest]);
|
||||||
@@ -467,3 +473,118 @@ test('rechecks the staged file and restores a replacement without deleting it',
|
|||||||
assert.equal((await fs.promises.readFile(file, 'utf-8')), 'replacement data');
|
assert.equal((await fs.promises.readFile(file, 'utf-8')), 'replacement data');
|
||||||
assert.equal(audits.at(-1).outcome, 'source-changed');
|
assert.equal(audits.at(-1).outcome, 'source-changed');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('keeps the source after done, failed history persistence, and an aborted retry of the same hoster', async (t) => {
|
||||||
|
const { file } = await makeSource(t, 'history-barrier-retry.bin');
|
||||||
|
const firstRound = makeCleanup();
|
||||||
|
const firstManifest = group(file, {
|
||||||
|
requiredHosters: ['voe.sx'],
|
||||||
|
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||||
|
});
|
||||||
|
await firstRound.cleanup.registerGroups([firstManifest]);
|
||||||
|
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||||
|
|
||||||
|
assert.deepEqual(await firstRound.cleanup.finishBatch({ historyPersisted: false, queuePersisted: true }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
|
||||||
|
const retryRound = makeCleanup();
|
||||||
|
const retryManifest = group(file, {
|
||||||
|
requiredHosters: ['voe.sx'],
|
||||||
|
completedHosters: ['voe.sx'],
|
||||||
|
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||||
|
});
|
||||||
|
await retryRound.cleanup.registerGroups([retryManifest]);
|
||||||
|
await retryRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'aborted' });
|
||||||
|
|
||||||
|
assert.deepEqual(await retryRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('deletes after a persisted partial round and a successful retry of the remaining hoster', async (t) => {
|
||||||
|
const { file } = await makeSource(t, 'partial-round-retry.bin');
|
||||||
|
const firstRound = makeCleanup();
|
||||||
|
const firstManifest = group(file, {
|
||||||
|
jobs: [
|
||||||
|
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true },
|
||||||
|
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
await firstRound.cleanup.registerGroups([firstManifest]);
|
||||||
|
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||||
|
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'error' });
|
||||||
|
|
||||||
|
assert.deepEqual(await firstRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
|
||||||
|
const retryRound = makeCleanup();
|
||||||
|
const retryManifest = group(file, {
|
||||||
|
confirmedHosters: ['voe.sx'],
|
||||||
|
jobs: [
|
||||||
|
{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: false },
|
||||||
|
{ jobId: 'job-byse', file, hoster: 'byse.sx', status: 'pending', currentRound: true }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
await retryRound.cleanup.registerGroups([retryManifest]);
|
||||||
|
await retryRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-byse', file, hoster: 'byse.sx', status: 'done' });
|
||||||
|
|
||||||
|
assert.deepEqual(await retryRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['deleted']);
|
||||||
|
assert.equal(await exists(file), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('does not trust a legacy completion after a failed queue barrier and restart', async (t) => {
|
||||||
|
const { file } = await makeSource(t, 'queue-barrier-restart.bin');
|
||||||
|
const firstRound = makeCleanup();
|
||||||
|
const firstManifest = group(file, {
|
||||||
|
requiredHosters: ['voe.sx'],
|
||||||
|
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||||
|
});
|
||||||
|
await firstRound.cleanup.registerGroups([firstManifest]);
|
||||||
|
await firstRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||||
|
|
||||||
|
assert.deepEqual(await firstRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: false }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
|
||||||
|
const restartedRound = makeCleanup();
|
||||||
|
const restartedManifest = group(file, {
|
||||||
|
requiredHosters: ['voe.sx'],
|
||||||
|
completedHosters: ['voe.sx'],
|
||||||
|
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||||
|
});
|
||||||
|
await restartedRound.cleanup.registerGroups([restartedManifest]);
|
||||||
|
await restartedRound.cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'error' });
|
||||||
|
|
||||||
|
assert.deepEqual(await restartedRound.cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lets a current non-done retry override an earlier confirmed completion', async (t) => {
|
||||||
|
const { file } = await makeSource(t, 'confirmed-hoster-retry.bin');
|
||||||
|
const { cleanup, audits } = makeCleanup();
|
||||||
|
const manifest = group(file, {
|
||||||
|
requiredHosters: ['voe.sx'],
|
||||||
|
confirmedHosters: ['voe.sx'],
|
||||||
|
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done', currentRound: true }]
|
||||||
|
});
|
||||||
|
await cleanup.registerGroups([manifest]);
|
||||||
|
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'aborted' });
|
||||||
|
|
||||||
|
assert.deepEqual(await cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
assert.deepEqual(audits[0].blockingStatuses, [{ hoster: 'voe.sx', status: 'aborted' }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('re-registering a hoster for the current round invalidates its earlier current success', async (t) => {
|
||||||
|
const { file } = await makeSource(t, 'same-batch-retry.bin');
|
||||||
|
const { cleanup, audits } = makeCleanup();
|
||||||
|
const manifest = group(file, {
|
||||||
|
requiredHosters: ['voe.sx'],
|
||||||
|
jobs: [{ jobId: 'job-voe', file, hoster: 'voe.sx', status: 'pending', currentRound: true }]
|
||||||
|
});
|
||||||
|
await cleanup.registerGroups([manifest]);
|
||||||
|
await cleanup.settle({ token: 'cleanup-1', jobId: 'job-voe', file, hoster: 'voe.sx', status: 'done' });
|
||||||
|
await cleanup.registerGroups([manifest]);
|
||||||
|
|
||||||
|
assert.deepEqual(await cleanup.finishBatch({ historyPersisted: true, queuePersisted: true }), ['blocked']);
|
||||||
|
assert.equal(await exists(file), true);
|
||||||
|
assert.deepEqual(audits[0].blockingStatuses, [{ hoster: 'voe.sx', status: 'pending' }]);
|
||||||
|
});
|
||||||
|
|||||||
@@ -1711,6 +1711,35 @@ setTimeout(async () => {
|
|||||||
|
|
||||||
const importedQueueMerge = await wc.executeJavaScript('(() => { queueJobs = [{ id: "ui-live-queue-after-import", file: "C:/ui/live-after-import.bin", fileName: "live-after-import.bin", hoster: "byse.sx", status: "queued", bytesTotal: 12 }]; rebuildJobIndex(); const imported = structuredClone(config); imported.globalSettings = { ...(imported.globalSettings || {}), webhookUrl: "https://queue-merge.invalid/imported", pendingQueue: { savedAt: 1, queueJobs: [{ id: "ui-stale-import-queue" }] } }; applyImportedConfig(imported, "Importiert"); return { webhookUrl: config.globalSettings.webhookUrl, ids: config.globalSettings.pendingQueue?.queueJobs?.map(job => job.id) || [] }; })()');
|
const importedQueueMerge = await wc.executeJavaScript('(() => { queueJobs = [{ id: "ui-live-queue-after-import", file: "C:/ui/live-after-import.bin", fileName: "live-after-import.bin", hoster: "byse.sx", status: "queued", bytesTotal: 12 }]; rebuildJobIndex(); const imported = structuredClone(config); imported.globalSettings = { ...(imported.globalSettings || {}), webhookUrl: "https://queue-merge.invalid/imported", pendingQueue: { savedAt: 1, queueJobs: [{ id: "ui-stale-import-queue" }] } }; applyImportedConfig(imported, "Importiert"); return { webhookUrl: config.globalSettings.webhookUrl, ids: config.globalSettings.pendingQueue?.queueJobs?.map(job => job.id) || [] }; })()');
|
||||||
check('Imported settings keep the live local queue in renderer memory', importedQueueMerge.webhookUrl === 'https://queue-merge.invalid/imported' && importedQueueMerge.ids.join('|') === 'ui-live-queue-after-import');
|
check('Imported settings keep the live local queue in renderer memory', importedQueueMerge.webhookUrl === 'https://queue-merge.invalid/imported' && importedQueueMerge.ids.join('|') === 'ui-live-queue-after-import');
|
||||||
|
const sourceCleanupDisabledRetry = await wc.executeJavaScript('(() => { const previousSetting = config.globalSettings?.deleteSourceAfterSuccessfulUpload; config.globalSettings = { ...(config.globalSettings || {}), deleteSourceAfterSuccessfulUpload: false }; queueJobs = [{ id: "ui-cleanup-disabled-retry", file: "C:/ui/cleanup-disabled.bin", fileName: "cleanup-disabled.bin", hoster: "voe.sx", status: "preview", bytesTotal: 10, sourceCleanupMetadataVersion: 2, sourceCleanupToken: "ui-cleanup-disabled-token", sourceCleanupRequiredHosters: ["voe.sx"], sourceCleanupConfirmedHosters: ["voe.sx"] }]; rebuildJobIndex(); const preparation = prepareSourceCleanup(queueJobs); config.globalSettings.deleteSourceAfterSuccessfulUpload = previousSetting; return { groups: preparation.groups.length, revokedHosters: preparation.revokedHosters || [], confirmedHosters: queueJobs[0].sourceCleanupConfirmedHosters || [] }; })()');
|
||||||
|
check('Retry revokes a confirmed cleanup hoster even while source deletion is disabled', sourceCleanupDisabledRetry.groups === 0 && sourceCleanupDisabledRetry.revokedHosters.join('|') === 'voe.sx' && sourceCleanupDisabledRetry.confirmedHosters.length === 0);
|
||||||
|
let sourceCleanupRevocationSaveCalls = 0;
|
||||||
|
ipcMain.removeHandler('save-pending-queue');
|
||||||
|
ipcMain.handle('save-pending-queue', async () => {
|
||||||
|
sourceCleanupRevocationSaveCalls++;
|
||||||
|
if (sourceCleanupRevocationSaveCalls === 1) throw new Error('injected cleanup revocation save failure');
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
const sourceCleanupRevocationRetry = await wc.executeJavaScript('(async () => { queueJobs = [{ id: "ui-cleanup-revocation-retry", file: "C:/ui/cleanup-revocation-retry.bin", fileName: "cleanup-revocation-retry.bin", hoster: "voe.sx", status: "preview", bytesTotal: 10, sourceCleanupMetadataVersion: 2, sourceCleanupToken: "ui-cleanup-revocation-retry-token", sourceCleanupRequiredHosters: ["voe.sx"], sourceCleanupConfirmedHosters: ["voe.sx"] }]; rebuildJobIndex(); const firstPreparation = prepareSourceCleanup(queueJobs); let firstFailed = false; try { await persistSourceCleanupRevocations(firstPreparation); } catch { firstFailed = true; } const secondPreparation = prepareSourceCleanup(queueJobs); let secondSucceeded = true; try { await persistSourceCleanupRevocations(secondPreparation); } catch { secondSucceeded = false; } return { firstFailed, secondSucceeded, secondRevocations: secondPreparation.revokedHosters || [] }; })()');
|
||||||
|
const sourceCleanupRevocationCallsBeforeRecovery = sourceCleanupRevocationSaveCalls;
|
||||||
|
restoreInitialIpcHandler('save-pending-queue');
|
||||||
|
await wc.executeJavaScript('flushConfigWrites()');
|
||||||
|
check('A failed cleanup revocation save stays mandatory for the next start attempt', sourceCleanupRevocationRetry.firstFailed === true && sourceCleanupRevocationRetry.secondSucceeded === true && sourceCleanupRevocationRetry.secondRevocations.length === 0 && sourceCleanupRevocationCallsBeforeRecovery >= 2);
|
||||||
|
const sourceCleanupFinalizationGate = await wc.executeJavaScript('(async () => { if (typeof sourceCleanupFinalizationPending === "undefined") return { available: false }; queueJobs = [{ id: "ui-cleanup-finalizing", file: "C:/ui/cleanup-finalizing.bin", fileName: "cleanup-finalizing.bin", hoster: "voe.sx", status: "done", bytesTotal: 10, result: { download_url: "https://example.invalid/finalizing" } }]; rebuildJobIndex(); selectedJobIds.clear(); selectedJobIds.add(queueJobs[0].id); sourceCleanupFinalizationPending = true; updateQueueActionButtons(); const disabled = document.getElementById("reuploadSelectedBtn")?.disabled === true; await retrySelectedJobs(); const status = queueJobs[0].status; sourceCleanupFinalizationPending = false; selectedJobIds.clear(); updateQueueActionButtons(); return { available: true, disabled, status }; })()');
|
||||||
|
check('Final cleanup persistence blocks a new retry until the handshake settles', sourceCleanupFinalizationGate.available === true && sourceCleanupFinalizationGate.disabled === true && sourceCleanupFinalizationGate.status === 'done');
|
||||||
|
let sourceCleanupFinalizationPayload = null;
|
||||||
|
ipcMain.removeHandler('complete-upload-finalization');
|
||||||
|
ipcMain.handle('complete-upload-finalization', (_event, payload) => {
|
||||||
|
sourceCleanupFinalizationPayload = payload;
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
const sourceCleanupRollback = await wc.executeJavaScript('(async () => { queuePersistThrottle.cancel(); await flushConfigWrites(); queueJobs = [{ id: "ui-cleanup-voe", file: "C:/ui/cleanup-round.bin", fileName: "cleanup-round.bin", hoster: "voe.sx", status: "preview", bytesTotal: 10 }, { id: "ui-cleanup-byse", file: "C:/ui/cleanup-round.bin", fileName: "cleanup-round.bin", hoster: "byse.sx", status: "error", bytesTotal: 10 }]; rebuildJobIndex(); window.SourceCleanupPolicy.prepareGroups(queueJobs, queueJobs, () => "ui-cleanup-token", "win32"); queueJobs[0].status = "done"; window.SourceCleanupPolicy.markCompleted(queueJobs, queueJobs[0], "win32"); const before = buildPersistedQueueState(); if (typeof completeSourceCleanupFinalization !== "function") return { available: false, before }; const result = await completeSourceCleanupFinalization({ finalizationId: "ui-cleanup-finalization", historyPersisted: true }); const after = buildPersistedQueueState(); return { available: true, result, before, after }; })()');
|
||||||
|
const sourceCleanupBeforeJobs = sourceCleanupRollback.before?.queueJobs || [];
|
||||||
|
const sourceCleanupAfterJobs = sourceCleanupRollback.after?.queueJobs || [];
|
||||||
|
const sourceCleanupPromotedJobs = sourceCleanupFinalizationPayload?.pendingQueue?.queueJobs || [];
|
||||||
|
const sourceCleanupRollbackOk = sourceCleanupRollback.available === true && sourceCleanupRollback.result === false && sourceCleanupBeforeJobs.length === 2 && sourceCleanupBeforeJobs.every(job => !Object.prototype.hasOwnProperty.call(job, 'sourceCleanupProvisionalHosters') && !Object.prototype.hasOwnProperty.call(job, 'sourceCleanupCompletedHosters') && (job.sourceCleanupConfirmedHosters || []).length === 0) && sourceCleanupPromotedJobs.length === 2 && sourceCleanupPromotedJobs.every(job => (job.sourceCleanupConfirmedHosters || []).join('|') === 'voe.sx') && sourceCleanupAfterJobs.length === 2 && sourceCleanupAfterJobs.every(job => (job.sourceCleanupConfirmedHosters || []).length === 0);
|
||||||
|
check('Final queue persistence promotes only inside the handshake and rolls back failed saves', sourceCleanupRollbackOk);
|
||||||
|
restoreInitialIpcHandler('complete-upload-finalization');
|
||||||
await wc.executeJavaScript('document.getElementById("copyToast")?.classList.remove("show")');
|
await wc.executeJavaScript('document.getElementById("copyToast")?.classList.remove("show")');
|
||||||
|
|
||||||
console.log('\\n=== History View ===');
|
console.log('\\n=== History View ===');
|
||||||
|
|||||||
Reference in New Issue
Block a user