Files
Multi-Hoster-Upload/lib/source-cleanup-policy.js
SucukdeluxeandClaude Opus 5.5 23fc22fd7d
CI / verify (push) Canceled after 0s
Ignore staged source deletions in folder automation
Deleting a source after a successful upload first renames it to
.<name>.mhu-delete-<uuid> and then unlinks it. On a slow disk the unlink
can take several seconds. With an empty extension filter the folder
watcher and the reconcile scan reported the staged file as a new file,
so the automation queued and uploaded it again until the unlink removed
it. The jobs for the vanished staged file were then restarted by every
automatic start and skipped with "Datei nicht gefunden".

- source-cleanup-policy: shared stagedSourceName/isStagedSourceFile.
- folder-monitor and file-discovery skip staged delete files.
- Automatic starts no longer restart queue jobs that point at a staged
  delete file; manual starts are unchanged.
- An ENOENT while unlinking the staged file counts as deleted instead of
  a failed unlink with a restore attempt. EBUSY/EPERM keep their retries
  and stay visible as failures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 17:25:55 +02:00

236 lines
7.8 KiB
JavaScript

(function (root) {
'use strict';
const pathApi = typeof require === 'function' ? require('path') : null;
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
const stagedSourcePattern = /^\..+\.mhu-delete-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
function stagedSourceName(fileName, id) {
return `.${fileName}.mhu-delete-${id}`;
}
function isStagedSourceFile(file) {
if (typeof file !== 'string' || !file) return false;
return stagedSourcePattern.test(file.split(/[\\/]/).pop());
}
function normalizeFile(file, platform) {
const value = typeof file === 'string' ? file.trim() : '';
if (!value) return '';
if (platform === 'win32') {
if (pathApi) return pathApi.win32.resolve(value);
return value.replace(/\//g, '\\');
}
if (pathApi) return pathApi.resolve(value);
return value;
}
function normalizeHoster(hoster) {
return typeof hoster === 'string' ? hoster.trim().toLowerCase() : '';
}
function uniqueHosters(values) {
const hosters = [];
const seen = new Set();
for (const value of values) {
const hoster = normalizeHoster(value);
if (!hoster || seen.has(hoster)) continue;
seen.add(hoster);
hosters.push(hoster);
}
return hosters;
}
function cloneFingerprint(fingerprint) {
if (!fingerprint || typeof fingerprint !== 'object' || Array.isArray(fingerprint)) return null;
return { ...fingerprint };
}
function relatedJobs(queueJobs, job, platform) {
if (!Array.isArray(queueJobs) || !job) return [];
const file = normalizeFile(job.file, platform);
const token = typeof job.sourceCleanupToken === 'string' && job.sourceCleanupToken
? job.sourceCleanupToken
: null;
return queueJobs.filter((candidate) => {
if (!candidate) return false;
if (file && normalizeFile(candidate.file, platform) === file) return true;
return token !== null && candidate.sourceCleanupToken === token;
});
}
function storedRequiredHosters(jobs) {
const values = [];
for (const job of jobs) {
if (Array.isArray(job.sourceCleanupRequiredHosters)) {
values.push(...job.sourceCleanupRequiredHosters);
}
}
return uniqueHosters(values);
}
function completedHosters(jobs, requiredHosters) {
const values = [];
for (const job of jobs) {
if (Array.isArray(job.sourceCleanupCompletedHosters)) {
values.push(...job.sourceCleanupCompletedHosters);
}
}
for (const job of jobs) {
if (job.status === 'done') values.push(job.hoster);
}
const completed = new Set(uniqueHosters(values));
return requiredHosters.filter((hoster) => completed.has(hoster));
}
function storedToken(jobs) {
for (const job of jobs) {
if (typeof job.sourceCleanupToken === 'string' && job.sourceCleanupToken) {
return job.sourceCleanupToken;
}
}
return null;
}
function storedFingerprint(jobs) {
for (const job of jobs) {
const fingerprint = cloneFingerprint(job.sourceCleanupFingerprint);
if (fingerprint) return fingerprint;
}
return null;
}
function assignMetadata(jobs, token, requiredHosters, completed, fingerprint, touchedJobs, touchedSet) {
for (const job of jobs) {
job.sourceCleanupToken = token;
job.sourceCleanupRequiredHosters = [...requiredHosters];
job.sourceCleanupCompletedHosters = [...completed];
job.sourceCleanupFingerprint = cloneFingerprint(fingerprint);
if (!touchedSet.has(job)) {
touchedSet.add(job);
touchedJobs.push(job);
}
}
}
function prepareGroups(queueJobs, jobsToStart, createToken, platform) {
const groups = [];
const touchedJobs = [];
const touchedSet = new Set();
const preparedFiles = new Set();
if (!Array.isArray(queueJobs) || !Array.isArray(jobsToStart)) return { groups, touchedJobs };
for (const selectedJob of jobsToStart) {
const file = normalizeFile(selectedJob && selectedJob.file, platform);
if (!file || preparedFiles.has(file)) continue;
const siblings = relatedJobs(queueJobs, selectedJob, platform);
if (siblings.length === 0) continue;
preparedFiles.add(file);
const token = storedToken(siblings) || (typeof createToken === 'function' ? createToken(file) : null);
if (typeof token !== 'string' || !token) continue;
const persistedRequired = storedRequiredHosters(siblings);
const requiredHosters = uniqueHosters([
...persistedRequired,
...siblings.map((job) => job.hoster)
]);
const completed = completedHosters(siblings, requiredHosters);
const fingerprint = storedFingerprint(siblings);
assignMetadata(
siblings,
token,
requiredHosters,
completed,
fingerprint,
touchedJobs,
touchedSet
);
groups.push({
token,
file: selectedJob.file,
requiredHosters: [...requiredHosters],
completedHosters: [...completed],
fingerprint: cloneFingerprint(fingerprint),
jobs: siblings.map((job) => ({
jobId: job.id,
hoster: normalizeHoster(job.hoster),
status: job.status
}))
});
}
return { groups, touchedJobs };
}
function markCompleted(queueJobs, job, platform) {
const siblings = relatedJobs(queueJobs, job, platform);
if (siblings.length === 0) return [];
const requiredHosters = storedRequiredHosters(siblings);
const completed = new Set(completedHosters(siblings, requiredHosters));
const hoster = normalizeHoster(job.hoster);
if (requiredHosters.includes(hoster)) completed.add(hoster);
const orderedCompleted = requiredHosters.filter((required) => completed.has(required));
for (const sibling of siblings) {
sibling.sourceCleanupCompletedHosters = [...orderedCompleted];
}
return siblings;
}
function removeRequirement(queueJobs, job, platform) {
if (!job || protectedStatuses.has(job.status)) return [];
const siblings = relatedJobs(queueJobs, job, platform);
const removedHoster = normalizeHoster(job.hoster);
for (const sibling of siblings) {
const required = Array.isArray(sibling.sourceCleanupRequiredHosters)
? sibling.sourceCleanupRequiredHosters
: [];
const completed = Array.isArray(sibling.sourceCleanupCompletedHosters)
? sibling.sourceCleanupCompletedHosters
: [];
sibling.sourceCleanupRequiredHosters = uniqueHosters(required)
.filter((hoster) => hoster !== removedHoster);
sibling.sourceCleanupCompletedHosters = uniqueHosters(completed)
.filter((hoster) => hoster !== removedHoster);
}
return siblings;
}
function fingerprintFor(fingerprints, token) {
if (fingerprints instanceof Map) return fingerprints.get(token);
if (!fingerprints || typeof fingerprints !== 'object') return undefined;
return Object.prototype.hasOwnProperty.call(fingerprints, token)
? fingerprints[token]
: undefined;
}
function applyFingerprints(queueJobs, fingerprints) {
if (!Array.isArray(queueJobs)) return [];
const touchedJobs = [];
for (const job of queueJobs) {
if (!job || typeof job.sourceCleanupToken !== 'string' || !job.sourceCleanupToken) continue;
const fingerprint = fingerprintFor(fingerprints, job.sourceCleanupToken);
if (fingerprint === undefined) continue;
job.sourceCleanupFingerprint = cloneFingerprint(fingerprint);
touchedJobs.push(job);
}
return touchedJobs;
}
const api = {
prepareGroups,
markCompleted,
removeRequirement,
applyFingerprints,
stagedSourceName,
isStagedSourceFile
};
if (typeof module !== 'undefined' && module.exports) {
module.exports = api;
} else if (root) {
root.SourceCleanupPolicy = api;
}
})(typeof window !== 'undefined' ? window : this);