Persist successful watched-file uploads in a dedicated fsync-backed ledger before exposing completion to the renderer. Match entries by normalized full path, hoster, size, and modification time so restart reconciliation skips unchanged completed files while changed files and explicit manual retries remain available. Remove restored queue ghosts from the ledger even when history and user upload logging are unavailable. Preserve per-hoster partial completion, capture missing file metadata asynchronously, fail closed on corrupted or unwritable evidence, and keep local persistence failures outside automatic upload retries. Stream managed upload logs with bounded lines, bytes, files, directories, and result counts. Include numbered rotations, reject unconfirmed rows, share concurrent scans through a generation-safe cache, invalidate after successful appends, avoid synchronous configuration and directory reads, and close streams on every path.
This commit is contained in:
+141
-1
@@ -147,10 +147,143 @@
|
||||
return String(value || '').replace(/\\/g, '/').toLowerCase();
|
||||
}
|
||||
|
||||
function isPathWithinAutomationFolder(filePath, folderPath, recursive = true) {
|
||||
const file = normalizePath(filePath).replace(/\/+$/, '');
|
||||
const folder = normalizePath(folderPath).replace(/\/+$/, '');
|
||||
if (!file || !folder || !file.startsWith(`${folder}/`)) return false;
|
||||
const relative = file.slice(folder.length + 1);
|
||||
return Boolean(relative) && (recursive === true || !relative.includes('/'));
|
||||
}
|
||||
|
||||
function baseName(value) {
|
||||
return String(value || '').split(/[\\/]/).pop().toLowerCase();
|
||||
}
|
||||
|
||||
function normalizeAutomationCompletion(value) {
|
||||
const row = asObject(value);
|
||||
const path = String(row.path || '').trim();
|
||||
const hoster = String(row.hoster || '').trim().toLowerCase();
|
||||
const size = Number(row.size);
|
||||
const mtimeMs = Number(row.mtimeMs);
|
||||
const completedAt = Number(row.completedAt);
|
||||
if (!path || !hoster || !Number.isFinite(size) || size < 0 || !Number.isFinite(mtimeMs) || mtimeMs < 0) return null;
|
||||
return {
|
||||
path,
|
||||
size,
|
||||
mtimeMs: Math.trunc(mtimeMs),
|
||||
hoster,
|
||||
completedAt: Number.isFinite(completedAt) && completedAt >= 0 ? Math.trunc(completedAt) : 0
|
||||
};
|
||||
}
|
||||
|
||||
function automationCompletionKey(value) {
|
||||
const row = normalizeAutomationCompletion(value);
|
||||
return row ? `${normalizePath(row.path)}\u0000${row.hoster}` : '';
|
||||
}
|
||||
|
||||
function mergeAutomationCompletions(existing, incoming, maxEntries = 250000) {
|
||||
const merged = new Map();
|
||||
for (const source of [asArray(existing), asArray(incoming)]) {
|
||||
for (const value of source) {
|
||||
const row = normalizeAutomationCompletion(value);
|
||||
const key = automationCompletionKey(row);
|
||||
if (key) {
|
||||
merged.delete(key);
|
||||
merged.set(key, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
const limit = Number.isFinite(Number(maxEntries)) ? Math.max(1, Math.floor(Number(maxEntries))) : 250000;
|
||||
if (merged.size > limit) throw new Error('Automatik-Abschlussdatei enthält zu viele Einträge');
|
||||
return [...merged.values()];
|
||||
}
|
||||
|
||||
function removeAutomationCompletions(existing, removals) {
|
||||
const rules = asArray(removals).map(value => ({
|
||||
path: normalizePath(value?.path),
|
||||
hoster: String(value?.hoster || '').trim().toLowerCase()
|
||||
})).filter(value => value.path);
|
||||
if (rules.length === 0) return mergeAutomationCompletions(existing, []);
|
||||
return mergeAutomationCompletions(existing, []).filter(row => {
|
||||
const path = normalizePath(row.path);
|
||||
return !rules.some(rule => rule.path === path && (!rule.hoster || rule.hoster === row.hoster));
|
||||
});
|
||||
}
|
||||
|
||||
function classifyAutomationCompletionLedger(input = {}) {
|
||||
const value = asObject(input);
|
||||
const rows = new Map();
|
||||
for (const entry of asArray(value.completionRows)) {
|
||||
const row = normalizeAutomationCompletion(entry);
|
||||
const key = automationCompletionKey(row);
|
||||
if (key) rows.set(key, row);
|
||||
}
|
||||
const processedPaths = [];
|
||||
const completedByPath = [];
|
||||
const remainingByPath = [];
|
||||
for (const candidate of asArray(value.candidates)) {
|
||||
const path = String(candidate?.path || '');
|
||||
const size = Number(candidate?.size);
|
||||
const mtimeMs = Number(candidate?.mtimeMs);
|
||||
const hosters = [...new Set(asArray(candidate?.eligibleHosters).map(hoster => String(hoster || '').trim().toLowerCase()).filter(Boolean))];
|
||||
const completed = [];
|
||||
const remaining = [];
|
||||
for (const hoster of hosters) {
|
||||
const row = rows.get(`${normalizePath(path)}\u0000${hoster}`);
|
||||
if (row && Number.isFinite(size) && size === row.size && Number.isFinite(mtimeMs) && Math.trunc(mtimeMs) === row.mtimeMs) completed.push(hoster);
|
||||
else remaining.push(hoster);
|
||||
}
|
||||
if (hosters.length > 0 && remaining.length === 0) processedPaths.push(path);
|
||||
completedByPath.push({ path, hosters: completed });
|
||||
remainingByPath.push({ path, hosters: remaining });
|
||||
}
|
||||
return { processedPaths, completedByPath, remainingByPath };
|
||||
}
|
||||
|
||||
function createAutomationCompletionWriter(options = {}) {
|
||||
const settings = asObject(options);
|
||||
if (typeof settings.save !== 'function') throw new TypeError('save is required');
|
||||
const schedule = typeof settings.schedule === 'function' ? settings.schedule : queueMicrotask;
|
||||
const pending = new Map();
|
||||
let scheduled = false;
|
||||
let tail = Promise.resolve();
|
||||
const flush = () => {
|
||||
scheduled = false;
|
||||
if (pending.size === 0) return tail;
|
||||
const snapshot = [...pending.entries()];
|
||||
const rows = snapshot.map(([, row]) => row);
|
||||
const operation = tail.then(async () => {
|
||||
try {
|
||||
await settings.save(rows);
|
||||
for (const [key, row] of snapshot) {
|
||||
if (pending.get(key) === row) pending.delete(key);
|
||||
}
|
||||
try { settings.onPersisted?.(rows); } catch {}
|
||||
} catch (error) {
|
||||
try { settings.onError?.(error, rows); } catch {}
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
tail = operation.catch(() => {});
|
||||
return operation;
|
||||
};
|
||||
return {
|
||||
add(value) {
|
||||
const row = normalizeAutomationCompletion(value);
|
||||
const key = automationCompletionKey(row);
|
||||
if (!key) return false;
|
||||
pending.set(key, row);
|
||||
if (!scheduled) {
|
||||
scheduled = true;
|
||||
schedule(() => { flush().catch(() => {}); });
|
||||
}
|
||||
return true;
|
||||
},
|
||||
flush,
|
||||
pendingCount: () => pending.size
|
||||
};
|
||||
}
|
||||
|
||||
function classifyProcessedCandidates(input = {}) {
|
||||
const value = asObject(input);
|
||||
const candidates = asArray(value.candidates);
|
||||
@@ -191,6 +324,13 @@
|
||||
rollDailyTelemetry,
|
||||
applyTelemetryDelta,
|
||||
deriveAutomationState,
|
||||
classifyProcessedCandidates
|
||||
isPathWithinAutomationFolder,
|
||||
classifyProcessedCandidates,
|
||||
classifyAutomationCompletionLedger,
|
||||
automationCompletionKey,
|
||||
createAutomationCompletionWriter,
|
||||
normalizeAutomationCompletion,
|
||||
mergeAutomationCompletions,
|
||||
removeAutomationCompletions
|
||||
};
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user