Ignore staged source deletions in folder automation
CI / verify (push) Canceled after 0s

Deleting a source after a successful upload first renames it to
.<name>.mhu-delete-<uuid> and then unlinks it. On a slow disk the unlink
can take several seconds. With an empty extension filter the folder
watcher and the reconcile scan reported the staged file as a new file,
so the automation queued and uploaded it again until the unlink removed
it. The jobs for the vanished staged file were then restarted by every
automatic start and skipped with "Datei nicht gefunden".

- source-cleanup-policy: shared stagedSourceName/isStagedSourceFile.
- folder-monitor and file-discovery skip staged delete files.
- Automatic starts no longer restart queue jobs that point at a staged
  delete file; manual starts are unchanged.
- An ENOENT while unlinking the staged file counts as deleted instead of
  a failed unlink with a restore attempt. EBUSY/EPERM keep their retries
  and stay visible as failures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
SucukdeluxeandClaude Opus 5.5 committed 2026-10-22 17:25:55 +02:00
1 parent 3c7ce614a1
commit 23fc22fd7d
5 files changed
+23 -3

No files matched your search

+2 -1
View File
@@ -1,5 +1,6 @@
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const { isStagedSourceFile } = require('./source-cleanup-policy');
async function walkFolderAsync(rootDir, options = {}) { async function walkFolderAsync(rootDir, options = {}) {
const fsPromises = options.fsPromises || fs.promises; const fsPromises = options.fsPromises || fs.promises;
@@ -22,7 +23,7 @@ async function walkFolderAsync(rootDir, options = {}) {
const fullPath = pathImpl.join(dir, entry.name); const fullPath = pathImpl.join(dir, entry.name);
if (entry.isDirectory()) { if (entry.isDirectory()) {
if (recursive) stack.push(fullPath); if (recursive) stack.push(fullPath);
} else if (entry.isFile()) { } else if (entry.isFile() && !isStagedSourceFile(entry.name)) {
let size = 0; let size = 0;
try { try {
size = (await fsPromises.stat(fullPath)).size; size = (await fsPromises.stat(fullPath)).size;
+3
View File
@@ -4,6 +4,7 @@ const path = require('path');
const chokidar = require('chokidar'); const chokidar = require('chokidar');
const { walkFolderAsync } = require('./file-discovery'); const { walkFolderAsync } = require('./file-discovery');
const { normalizeAutomationSettings } = require('./automation-control'); const { normalizeAutomationSettings } = require('./automation-control');
const { isStagedSourceFile } = require('./source-cleanup-policy');
class FolderMonitor extends EventEmitter { class FolderMonitor extends EventEmitter {
constructor({ constructor({
@@ -267,6 +268,7 @@ class FolderMonitor extends EventEmitter {
_onNewFile(filePath, generation) { _onNewFile(filePath, generation) {
if (!this._acceptCallback(generation)) return; if (!this._acceptCallback(generation)) return;
if (isStagedSourceFile(filePath)) return;
if (!this._classifyPath(filePath, this._settings).allowed) return; if (!this._classifyPath(filePath, this._settings).allowed) return;
const normalized = this._normalizePath(filePath); const normalized = this._normalizePath(filePath);
const ownsSeenReservation = !!this._settings?.skipDuplicates && !this._seenFiles.has(normalized); const ownsSeenReservation = !!this._settings?.skipDuplicates && !this._seenFiles.has(normalized);
@@ -381,6 +383,7 @@ class FolderMonitor extends EventEmitter {
for (const descriptor of discovered) { for (const descriptor of discovered) {
if (!this._isCurrent(generation)) return []; if (!this._isCurrent(generation)) return [];
if (!settings.recursive && this._isNestedPath(descriptor.path, settings.folderPath)) continue; if (!settings.recursive && this._isNestedPath(descriptor.path, settings.folderPath)) continue;
if (isStagedSourceFile(descriptor.path)) continue;
const classification = this._classifyPath(descriptor.path, settings); const classification = this._classifyPath(descriptor.path, settings);
let mtimeMs = 0; let mtimeMs = 0;
try { try {
+13 -1
View File
@@ -3,6 +3,16 @@
const pathApi = typeof require === 'function' ? require('path') : null; const pathApi = typeof require === 'function' ? require('path') : null;
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']); const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
const stagedSourcePattern = /^\..+\.mhu-delete-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
function stagedSourceName(fileName, id) {
return `.${fileName}.mhu-delete-${id}`;
}
function isStagedSourceFile(file) {
if (typeof file !== 'string' || !file) return false;
return stagedSourcePattern.test(file.split(/[\\/]/).pop());
}
function normalizeFile(file, platform) { function normalizeFile(file, platform) {
const value = typeof file === 'string' ? file.trim() : ''; const value = typeof file === 'string' ? file.trim() : '';
@@ -212,7 +222,9 @@
prepareGroups, prepareGroups,
markCompleted, markCompleted,
removeRequirement, removeRequirement,
applyFingerprints applyFingerprints,
stagedSourceName,
isStagedSourceFile
}; };
if (typeof module !== 'undefined' && module.exports) { if (typeof module !== 'undefined' && module.exports) {
+4 -1
View File
@@ -1,6 +1,7 @@
const RETRY_DELAYS = [100, 250, 500, 1000, 2000]; const RETRY_DELAYS = [100, 250, 500, 1000, 2000];
const TERMINAL_STATUSES = new Set(['done', 'error', 'aborted', 'skipped', 'failed', 'missing-account']); const TERMINAL_STATUSES = new Set(['done', 'error', 'aborted', 'skipped', 'failed', 'missing-account']);
const crypto = require('node:crypto'); const crypto = require('node:crypto');
const { stagedSourceName } = require('./source-cleanup-policy');
function createSourceFileCleanup(options) { function createSourceFileCleanup(options) {
if (!options || !options.fs || !options.path) { if (!options || !options.fs || !options.path) {
@@ -322,6 +323,7 @@ function createSourceFileCleanup(options) {
await fs.promises.unlink(file); await fs.promises.unlink(file);
return { attempts }; return { attempts };
} catch (error) { } catch (error) {
if (error && error.code === 'ENOENT') return { attempts, alreadyRemoved: true };
const retryable = platform === 'win32' && error && (error.code === 'EBUSY' || error.code === 'EPERM'); const retryable = platform === 'win32' && error && (error.code === 'EBUSY' || error.code === 'EPERM');
if (!retryable || attempts > RETRY_DELAYS.length) throw Object.assign(error, { cleanupAttempts: attempts }); if (!retryable || attempts > RETRY_DELAYS.length) throw Object.assign(error, { cleanupAttempts: attempts });
await wait(RETRY_DELAYS[attempts - 1]); await wait(RETRY_DELAYS[attempts - 1]);
@@ -346,7 +348,7 @@ function createSourceFileCleanup(options) {
async function stageSource(manifest) { async function stageSource(manifest) {
const stagedFile = path.join( const stagedFile = path.join(
path.dirname(manifest.file), path.dirname(manifest.file),
`.${path.basename(manifest.file)}.mhu-delete-${crypto.randomUUID()}` stagedSourceName(path.basename(manifest.file), crypto.randomUUID())
); );
if (!journal || typeof journal.plan !== 'function') { if (!journal || typeof journal.plan !== 'function') {
return { outcome: 'blocked', trigger: 'delete-journal-unavailable' }; return { outcome: 'blocked', trigger: 'delete-journal-unavailable' };
@@ -464,6 +466,7 @@ function createSourceFileCleanup(options) {
await emitAudit(manifest, 'deleted', { await emitAudit(manifest, 'deleted', {
trigger: 'all-selected-hosters-succeeded', trigger: 'all-selected-hosters-succeeded',
attempts: result.attempts, attempts: result.attempts,
...(result.alreadyRemoved ? { alreadyRemoved: true } : {}),
...scope ...scope
}); });
return 'deleted'; return 'deleted';
+1
View File
@@ -446,6 +446,7 @@ function isStartableQueueJob(job) {
function isAutomaticallyStartableQueueJob(job) { function isAutomaticallyStartableQueueJob(job) {
if (!isStartableQueueJob(job)) return false; if (!isStartableQueueJob(job)) return false;
if (window.SourceCleanupPolicy?.isStagedSourceFile(job.file || job.fileName)) return false;
if (job.status !== 'error' || !window.Stats) return true; if (job.status !== 'error' || !window.Stats) return true;
return window.Stats.classifyErrorCategory(job.error) !== 'file-rejected'; return window.Stats.classifyErrorCategory(job.error) !== 'file-rejected';
} }