Deleting a source after a successful upload first renames it to .<name>.mhu-delete-<uuid> and then unlinks it. On a slow disk the unlink can take several seconds. With an empty extension filter the folder watcher and the reconcile scan reported the staged file as a new file, so the automation queued and uploaded it again until the unlink removed it. The jobs for the vanished staged file were then restarted by every automatic start and skipped with "Datei nicht gefunden". - source-cleanup-policy: shared stagedSourceName/isStagedSourceFile. - folder-monitor and file-discovery skip staged delete files. - Automatic starts no longer restart queue jobs that point at a staged delete file; manual starts are unchanged. - An ENOENT while unlinking the staged file counts as deleted instead of a failed unlink with a restore attempt. EBUSY/EPERM keep their retries and stay visible as failures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3c7ce614a1
commit
23fc22fd7d
5 files changed
+23
-3
No files matched your search
@@ -1,5 +1,6 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
|
const { isStagedSourceFile } = require('./source-cleanup-policy');
|
||||||
|
|
||||||
async function walkFolderAsync(rootDir, options = {}) {
|
async function walkFolderAsync(rootDir, options = {}) {
|
||||||
const fsPromises = options.fsPromises || fs.promises;
|
const fsPromises = options.fsPromises || fs.promises;
|
||||||
@@ -22,7 +23,7 @@ async function walkFolderAsync(rootDir, options = {}) {
|
|||||||
const fullPath = pathImpl.join(dir, entry.name);
|
const fullPath = pathImpl.join(dir, entry.name);
|
||||||
if (entry.isDirectory()) {
|
if (entry.isDirectory()) {
|
||||||
if (recursive) stack.push(fullPath);
|
if (recursive) stack.push(fullPath);
|
||||||
} else if (entry.isFile()) {
|
} else if (entry.isFile() && !isStagedSourceFile(entry.name)) {
|
||||||
let size = 0;
|
let size = 0;
|
||||||
try {
|
try {
|
||||||
size = (await fsPromises.stat(fullPath)).size;
|
size = (await fsPromises.stat(fullPath)).size;
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ const path = require('path');
|
|||||||
const chokidar = require('chokidar');
|
const chokidar = require('chokidar');
|
||||||
const { walkFolderAsync } = require('./file-discovery');
|
const { walkFolderAsync } = require('./file-discovery');
|
||||||
const { normalizeAutomationSettings } = require('./automation-control');
|
const { normalizeAutomationSettings } = require('./automation-control');
|
||||||
|
const { isStagedSourceFile } = require('./source-cleanup-policy');
|
||||||
|
|
||||||
class FolderMonitor extends EventEmitter {
|
class FolderMonitor extends EventEmitter {
|
||||||
constructor({
|
constructor({
|
||||||
@@ -267,6 +268,7 @@ class FolderMonitor extends EventEmitter {
|
|||||||
|
|
||||||
_onNewFile(filePath, generation) {
|
_onNewFile(filePath, generation) {
|
||||||
if (!this._acceptCallback(generation)) return;
|
if (!this._acceptCallback(generation)) return;
|
||||||
|
if (isStagedSourceFile(filePath)) return;
|
||||||
if (!this._classifyPath(filePath, this._settings).allowed) return;
|
if (!this._classifyPath(filePath, this._settings).allowed) return;
|
||||||
const normalized = this._normalizePath(filePath);
|
const normalized = this._normalizePath(filePath);
|
||||||
const ownsSeenReservation = !!this._settings?.skipDuplicates && !this._seenFiles.has(normalized);
|
const ownsSeenReservation = !!this._settings?.skipDuplicates && !this._seenFiles.has(normalized);
|
||||||
@@ -381,6 +383,7 @@ class FolderMonitor extends EventEmitter {
|
|||||||
for (const descriptor of discovered) {
|
for (const descriptor of discovered) {
|
||||||
if (!this._isCurrent(generation)) return [];
|
if (!this._isCurrent(generation)) return [];
|
||||||
if (!settings.recursive && this._isNestedPath(descriptor.path, settings.folderPath)) continue;
|
if (!settings.recursive && this._isNestedPath(descriptor.path, settings.folderPath)) continue;
|
||||||
|
if (isStagedSourceFile(descriptor.path)) continue;
|
||||||
const classification = this._classifyPath(descriptor.path, settings);
|
const classification = this._classifyPath(descriptor.path, settings);
|
||||||
let mtimeMs = 0;
|
let mtimeMs = 0;
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -3,6 +3,16 @@
|
|||||||
|
|
||||||
const pathApi = typeof require === 'function' ? require('path') : null;
|
const pathApi = typeof require === 'function' ? require('path') : null;
|
||||||
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
|
const protectedStatuses = new Set(['done', 'error', 'aborted', 'skipped']);
|
||||||
|
const stagedSourcePattern = /^\..+\.mhu-delete-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||||
|
|
||||||
|
function stagedSourceName(fileName, id) {
|
||||||
|
return `.${fileName}.mhu-delete-${id}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isStagedSourceFile(file) {
|
||||||
|
if (typeof file !== 'string' || !file) return false;
|
||||||
|
return stagedSourcePattern.test(file.split(/[\\/]/).pop());
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeFile(file, platform) {
|
function normalizeFile(file, platform) {
|
||||||
const value = typeof file === 'string' ? file.trim() : '';
|
const value = typeof file === 'string' ? file.trim() : '';
|
||||||
@@ -212,7 +222,9 @@
|
|||||||
prepareGroups,
|
prepareGroups,
|
||||||
markCompleted,
|
markCompleted,
|
||||||
removeRequirement,
|
removeRequirement,
|
||||||
applyFingerprints
|
applyFingerprints,
|
||||||
|
stagedSourceName,
|
||||||
|
isStagedSourceFile
|
||||||
};
|
};
|
||||||
|
|
||||||
if (typeof module !== 'undefined' && module.exports) {
|
if (typeof module !== 'undefined' && module.exports) {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
const RETRY_DELAYS = [100, 250, 500, 1000, 2000];
|
const RETRY_DELAYS = [100, 250, 500, 1000, 2000];
|
||||||
const TERMINAL_STATUSES = new Set(['done', 'error', 'aborted', 'skipped', 'failed', 'missing-account']);
|
const TERMINAL_STATUSES = new Set(['done', 'error', 'aborted', 'skipped', 'failed', 'missing-account']);
|
||||||
const crypto = require('node:crypto');
|
const crypto = require('node:crypto');
|
||||||
|
const { stagedSourceName } = require('./source-cleanup-policy');
|
||||||
|
|
||||||
function createSourceFileCleanup(options) {
|
function createSourceFileCleanup(options) {
|
||||||
if (!options || !options.fs || !options.path) {
|
if (!options || !options.fs || !options.path) {
|
||||||
@@ -322,6 +323,7 @@ function createSourceFileCleanup(options) {
|
|||||||
await fs.promises.unlink(file);
|
await fs.promises.unlink(file);
|
||||||
return { attempts };
|
return { attempts };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error && error.code === 'ENOENT') return { attempts, alreadyRemoved: true };
|
||||||
const retryable = platform === 'win32' && error && (error.code === 'EBUSY' || error.code === 'EPERM');
|
const retryable = platform === 'win32' && error && (error.code === 'EBUSY' || error.code === 'EPERM');
|
||||||
if (!retryable || attempts > RETRY_DELAYS.length) throw Object.assign(error, { cleanupAttempts: attempts });
|
if (!retryable || attempts > RETRY_DELAYS.length) throw Object.assign(error, { cleanupAttempts: attempts });
|
||||||
await wait(RETRY_DELAYS[attempts - 1]);
|
await wait(RETRY_DELAYS[attempts - 1]);
|
||||||
@@ -346,7 +348,7 @@ function createSourceFileCleanup(options) {
|
|||||||
async function stageSource(manifest) {
|
async function stageSource(manifest) {
|
||||||
const stagedFile = path.join(
|
const stagedFile = path.join(
|
||||||
path.dirname(manifest.file),
|
path.dirname(manifest.file),
|
||||||
`.${path.basename(manifest.file)}.mhu-delete-${crypto.randomUUID()}`
|
stagedSourceName(path.basename(manifest.file), crypto.randomUUID())
|
||||||
);
|
);
|
||||||
if (!journal || typeof journal.plan !== 'function') {
|
if (!journal || typeof journal.plan !== 'function') {
|
||||||
return { outcome: 'blocked', trigger: 'delete-journal-unavailable' };
|
return { outcome: 'blocked', trigger: 'delete-journal-unavailable' };
|
||||||
@@ -464,6 +466,7 @@ function createSourceFileCleanup(options) {
|
|||||||
await emitAudit(manifest, 'deleted', {
|
await emitAudit(manifest, 'deleted', {
|
||||||
trigger: 'all-selected-hosters-succeeded',
|
trigger: 'all-selected-hosters-succeeded',
|
||||||
attempts: result.attempts,
|
attempts: result.attempts,
|
||||||
|
...(result.alreadyRemoved ? { alreadyRemoved: true } : {}),
|
||||||
...scope
|
...scope
|
||||||
});
|
});
|
||||||
return 'deleted';
|
return 'deleted';
|
||||||
|
|||||||
@@ -446,6 +446,7 @@ function isStartableQueueJob(job) {
|
|||||||
|
|
||||||
function isAutomaticallyStartableQueueJob(job) {
|
function isAutomaticallyStartableQueueJob(job) {
|
||||||
if (!isStartableQueueJob(job)) return false;
|
if (!isStartableQueueJob(job)) return false;
|
||||||
|
if (window.SourceCleanupPolicy?.isStagedSourceFile(job.file || job.fileName)) return false;
|
||||||
if (job.status !== 'error' || !window.Stats) return true;
|
if (job.status !== 'error' || !window.Stats) return true;
|
||||||
return window.Stats.classifyErrorCategory(job.error) !== 'file-rejected';
|
return window.Stats.classifyErrorCategory(job.error) !== 'file-rejected';
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user