Persist successful watched-file uploads in a dedicated fsync-backed ledger before exposing completion to the renderer. Match entries by normalized full path, hoster, size, and modification time so restart reconciliation skips unchanged completed files while changed files and explicit manual retries remain available. Remove restored queue ghosts from the ledger even when history and user upload logging are unavailable. Preserve per-hoster partial completion, capture missing file metadata asynchronously, fail closed on corrupted or unwritable evidence, and keep local persistence failures outside automatic upload retries. Stream managed upload logs with bounded lines, bytes, files, directories, and result counts. Include numbered rotations, reject unconfirmed rows, share concurrent scans through a generation-safe cache, invalidate after successful appends, avoid synchronous configuration and directory reads, and close streams on every path.
This commit is contained in:
@@ -13,6 +13,8 @@ electron-config.json.pre-history-split.bak
|
|||||||
electron-config.pre-import-*.json
|
electron-config.pre-import-*.json
|
||||||
electron-history.json
|
electron-history.json
|
||||||
electron-history.json.tmp
|
electron-history.json.tmp
|
||||||
|
automation-completions.json
|
||||||
|
automation-completions.json.tmp
|
||||||
*.log
|
*.log
|
||||||
debug.log
|
debug.log
|
||||||
fileuploader.log
|
fileuploader.log
|
||||||
|
|||||||
+141
-1
@@ -147,10 +147,143 @@
|
|||||||
return String(value || '').replace(/\\/g, '/').toLowerCase();
|
return String(value || '').replace(/\\/g, '/').toLowerCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isPathWithinAutomationFolder(filePath, folderPath, recursive = true) {
|
||||||
|
const file = normalizePath(filePath).replace(/\/+$/, '');
|
||||||
|
const folder = normalizePath(folderPath).replace(/\/+$/, '');
|
||||||
|
if (!file || !folder || !file.startsWith(`${folder}/`)) return false;
|
||||||
|
const relative = file.slice(folder.length + 1);
|
||||||
|
return Boolean(relative) && (recursive === true || !relative.includes('/'));
|
||||||
|
}
|
||||||
|
|
||||||
function baseName(value) {
|
function baseName(value) {
|
||||||
return String(value || '').split(/[\\/]/).pop().toLowerCase();
|
return String(value || '').split(/[\\/]/).pop().toLowerCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeAutomationCompletion(value) {
|
||||||
|
const row = asObject(value);
|
||||||
|
const path = String(row.path || '').trim();
|
||||||
|
const hoster = String(row.hoster || '').trim().toLowerCase();
|
||||||
|
const size = Number(row.size);
|
||||||
|
const mtimeMs = Number(row.mtimeMs);
|
||||||
|
const completedAt = Number(row.completedAt);
|
||||||
|
if (!path || !hoster || !Number.isFinite(size) || size < 0 || !Number.isFinite(mtimeMs) || mtimeMs < 0) return null;
|
||||||
|
return {
|
||||||
|
path,
|
||||||
|
size,
|
||||||
|
mtimeMs: Math.trunc(mtimeMs),
|
||||||
|
hoster,
|
||||||
|
completedAt: Number.isFinite(completedAt) && completedAt >= 0 ? Math.trunc(completedAt) : 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function automationCompletionKey(value) {
|
||||||
|
const row = normalizeAutomationCompletion(value);
|
||||||
|
return row ? `${normalizePath(row.path)}\u0000${row.hoster}` : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeAutomationCompletions(existing, incoming, maxEntries = 250000) {
|
||||||
|
const merged = new Map();
|
||||||
|
for (const source of [asArray(existing), asArray(incoming)]) {
|
||||||
|
for (const value of source) {
|
||||||
|
const row = normalizeAutomationCompletion(value);
|
||||||
|
const key = automationCompletionKey(row);
|
||||||
|
if (key) {
|
||||||
|
merged.delete(key);
|
||||||
|
merged.set(key, row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const limit = Number.isFinite(Number(maxEntries)) ? Math.max(1, Math.floor(Number(maxEntries))) : 250000;
|
||||||
|
if (merged.size > limit) throw new Error('Automatik-Abschlussdatei enthält zu viele Einträge');
|
||||||
|
return [...merged.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
function removeAutomationCompletions(existing, removals) {
|
||||||
|
const rules = asArray(removals).map(value => ({
|
||||||
|
path: normalizePath(value?.path),
|
||||||
|
hoster: String(value?.hoster || '').trim().toLowerCase()
|
||||||
|
})).filter(value => value.path);
|
||||||
|
if (rules.length === 0) return mergeAutomationCompletions(existing, []);
|
||||||
|
return mergeAutomationCompletions(existing, []).filter(row => {
|
||||||
|
const path = normalizePath(row.path);
|
||||||
|
return !rules.some(rule => rule.path === path && (!rule.hoster || rule.hoster === row.hoster));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function classifyAutomationCompletionLedger(input = {}) {
|
||||||
|
const value = asObject(input);
|
||||||
|
const rows = new Map();
|
||||||
|
for (const entry of asArray(value.completionRows)) {
|
||||||
|
const row = normalizeAutomationCompletion(entry);
|
||||||
|
const key = automationCompletionKey(row);
|
||||||
|
if (key) rows.set(key, row);
|
||||||
|
}
|
||||||
|
const processedPaths = [];
|
||||||
|
const completedByPath = [];
|
||||||
|
const remainingByPath = [];
|
||||||
|
for (const candidate of asArray(value.candidates)) {
|
||||||
|
const path = String(candidate?.path || '');
|
||||||
|
const size = Number(candidate?.size);
|
||||||
|
const mtimeMs = Number(candidate?.mtimeMs);
|
||||||
|
const hosters = [...new Set(asArray(candidate?.eligibleHosters).map(hoster => String(hoster || '').trim().toLowerCase()).filter(Boolean))];
|
||||||
|
const completed = [];
|
||||||
|
const remaining = [];
|
||||||
|
for (const hoster of hosters) {
|
||||||
|
const row = rows.get(`${normalizePath(path)}\u0000${hoster}`);
|
||||||
|
if (row && Number.isFinite(size) && size === row.size && Number.isFinite(mtimeMs) && Math.trunc(mtimeMs) === row.mtimeMs) completed.push(hoster);
|
||||||
|
else remaining.push(hoster);
|
||||||
|
}
|
||||||
|
if (hosters.length > 0 && remaining.length === 0) processedPaths.push(path);
|
||||||
|
completedByPath.push({ path, hosters: completed });
|
||||||
|
remainingByPath.push({ path, hosters: remaining });
|
||||||
|
}
|
||||||
|
return { processedPaths, completedByPath, remainingByPath };
|
||||||
|
}
|
||||||
|
|
||||||
|
function createAutomationCompletionWriter(options = {}) {
|
||||||
|
const settings = asObject(options);
|
||||||
|
if (typeof settings.save !== 'function') throw new TypeError('save is required');
|
||||||
|
const schedule = typeof settings.schedule === 'function' ? settings.schedule : queueMicrotask;
|
||||||
|
const pending = new Map();
|
||||||
|
let scheduled = false;
|
||||||
|
let tail = Promise.resolve();
|
||||||
|
const flush = () => {
|
||||||
|
scheduled = false;
|
||||||
|
if (pending.size === 0) return tail;
|
||||||
|
const snapshot = [...pending.entries()];
|
||||||
|
const rows = snapshot.map(([, row]) => row);
|
||||||
|
const operation = tail.then(async () => {
|
||||||
|
try {
|
||||||
|
await settings.save(rows);
|
||||||
|
for (const [key, row] of snapshot) {
|
||||||
|
if (pending.get(key) === row) pending.delete(key);
|
||||||
|
}
|
||||||
|
try { settings.onPersisted?.(rows); } catch {}
|
||||||
|
} catch (error) {
|
||||||
|
try { settings.onError?.(error, rows); } catch {}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
tail = operation.catch(() => {});
|
||||||
|
return operation;
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
add(value) {
|
||||||
|
const row = normalizeAutomationCompletion(value);
|
||||||
|
const key = automationCompletionKey(row);
|
||||||
|
if (!key) return false;
|
||||||
|
pending.set(key, row);
|
||||||
|
if (!scheduled) {
|
||||||
|
scheduled = true;
|
||||||
|
schedule(() => { flush().catch(() => {}); });
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
flush,
|
||||||
|
pendingCount: () => pending.size
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function classifyProcessedCandidates(input = {}) {
|
function classifyProcessedCandidates(input = {}) {
|
||||||
const value = asObject(input);
|
const value = asObject(input);
|
||||||
const candidates = asArray(value.candidates);
|
const candidates = asArray(value.candidates);
|
||||||
@@ -191,6 +324,13 @@
|
|||||||
rollDailyTelemetry,
|
rollDailyTelemetry,
|
||||||
applyTelemetryDelta,
|
applyTelemetryDelta,
|
||||||
deriveAutomationState,
|
deriveAutomationState,
|
||||||
classifyProcessedCandidates
|
isPathWithinAutomationFolder,
|
||||||
|
classifyProcessedCandidates,
|
||||||
|
classifyAutomationCompletionLedger,
|
||||||
|
automationCompletionKey,
|
||||||
|
createAutomationCompletionWriter,
|
||||||
|
normalizeAutomationCompletion,
|
||||||
|
mergeAutomationCompletions,
|
||||||
|
removeAutomationCompletions
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ const fs = require('fs');
|
|||||||
const path = require('path');
|
const path = require('path');
|
||||||
const secretStore = require('./secret-store');
|
const secretStore = require('./secret-store');
|
||||||
const { normalizeLogMode } = require('./log-mode');
|
const { normalizeLogMode } = require('./log-mode');
|
||||||
|
const { mergeAutomationCompletions, normalizeAutomationCompletion, removeAutomationCompletions } = require('./automation-control');
|
||||||
|
|
||||||
const HOSTER_SETTINGS_DEFAULTS = {
|
const HOSTER_SETTINGS_DEFAULTS = {
|
||||||
retries: 3,
|
retries: 3,
|
||||||
@@ -196,8 +197,11 @@ class ConfigStore {
|
|||||||
: path.join(__dirname, '..');
|
: path.join(__dirname, '..');
|
||||||
this.filePath = path.join(dir, 'electron-config.json');
|
this.filePath = path.join(dir, 'electron-config.json');
|
||||||
this.historyPath = path.join(dir, 'electron-history.json');
|
this.historyPath = path.join(dir, 'electron-history.json');
|
||||||
|
this.automationCompletionPath = path.join(dir, 'automation-completions.json');
|
||||||
this._writeQueue = Promise.resolve(); // Serializes all writes to prevent race conditions
|
this._writeQueue = Promise.resolve(); // Serializes all writes to prevent race conditions
|
||||||
this._historyWriteQueue = Promise.resolve();
|
this._historyWriteQueue = Promise.resolve();
|
||||||
|
this._automationCompletionWriteQueue = Promise.resolve();
|
||||||
|
this._automationCompletionCache = null;
|
||||||
this._pendingWriteOperations = new Set();
|
this._pendingWriteOperations = new Set();
|
||||||
this._writesQuiesced = false;
|
this._writesQuiesced = false;
|
||||||
this._historyMigrated = false;
|
this._historyMigrated = false;
|
||||||
@@ -622,6 +626,71 @@ class ConfigStore {
|
|||||||
}, options);
|
}, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async _readAutomationCompletionFile() {
|
||||||
|
try {
|
||||||
|
const raw = await fs.promises.readFile(this.automationCompletionPath, 'utf8');
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.entries)) throw new Error('Automatik-Abschlussdatei ist ungültig');
|
||||||
|
if (parsed.entries.some(entry => !normalizeAutomationCompletion(entry))) throw new Error('Automatik-Abschlussdatei ist ungültig');
|
||||||
|
return mergeAutomationCompletions([], parsed.entries);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.code === 'ENOENT') return [];
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async _writeAutomationCompletionFile(entries) {
|
||||||
|
const target = this.automationCompletionPath;
|
||||||
|
const temporary = `${target}.tmp`;
|
||||||
|
await fs.promises.mkdir(path.dirname(target), { recursive: true });
|
||||||
|
const handle = await fs.promises.open(temporary, 'w');
|
||||||
|
try {
|
||||||
|
await handle.writeFile(JSON.stringify({ version: 1, entries }), 'utf8');
|
||||||
|
await handle.sync();
|
||||||
|
} finally {
|
||||||
|
await handle.close();
|
||||||
|
}
|
||||||
|
await fs.promises.rename(temporary, target);
|
||||||
|
}
|
||||||
|
|
||||||
|
_enqueueAutomationCompletionWrite(operation) {
|
||||||
|
const result = this._automationCompletionWriteQueue.then(operation);
|
||||||
|
this._automationCompletionWriteQueue = result.catch(() => {});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async loadAutomationCompletions() {
|
||||||
|
await this._automationCompletionWriteQueue;
|
||||||
|
if (!this._automationCompletionCache) this._automationCompletionCache = await this._readAutomationCompletionFile();
|
||||||
|
return this._clone(this._automationCompletionCache);
|
||||||
|
}
|
||||||
|
|
||||||
|
saveAutomationCompletions(entries) {
|
||||||
|
const incoming = this._clone(Array.isArray(entries) ? entries : []);
|
||||||
|
return this._enqueueAutomationCompletionWrite(async () => {
|
||||||
|
const current = this._automationCompletionCache || await this._readAutomationCompletionFile();
|
||||||
|
const next = mergeAutomationCompletions(current, incoming);
|
||||||
|
await this._writeAutomationCompletionFile(next);
|
||||||
|
this._automationCompletionCache = next;
|
||||||
|
return this._clone(next);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
clearAutomationCompletions(removals) {
|
||||||
|
const requested = this._clone(Array.isArray(removals) ? removals : []);
|
||||||
|
return this._enqueueAutomationCompletionWrite(async () => {
|
||||||
|
const current = this._automationCompletionCache || await this._readAutomationCompletionFile();
|
||||||
|
const next = removeAutomationCompletions(current, requested);
|
||||||
|
await this._writeAutomationCompletionFile(next);
|
||||||
|
this._automationCompletionCache = next;
|
||||||
|
return this._clone(next);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async drainAutomationCompletionWrites() {
|
||||||
|
await this._automationCompletionWriteQueue;
|
||||||
|
}
|
||||||
|
|
||||||
saveLastBrowseDirectory(directory) {
|
saveLastBrowseDirectory(directory) {
|
||||||
const snapshot = String(directory || '').trim();
|
const snapshot = String(directory || '').trim();
|
||||||
return this._enqueueWrite(() => {
|
return this._enqueueWrite(() => {
|
||||||
|
|||||||
@@ -27,7 +27,8 @@
|
|||||||
return {
|
return {
|
||||||
path: filePath,
|
path: filePath,
|
||||||
name: sourceName || path.basename(filePath),
|
name: sourceName || path.basename(filePath),
|
||||||
size: Number.isFinite(Number(source.size)) ? Number(source.size) : null
|
size: Number.isFinite(Number(source.size)) ? Number(source.size) : null,
|
||||||
|
mtimeMs: Number.isFinite(Number(source.mtimeMs)) ? Number(source.mtimeMs) : null
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,8 +63,8 @@
|
|||||||
try {
|
try {
|
||||||
fileHandle = await openPath(filePath, 'r');
|
fileHandle = await openPath(filePath, 'r');
|
||||||
const fileStat = await fileHandle.stat();
|
const fileStat = await fileHandle.stat();
|
||||||
if (!fileStat.isFile()) return { exists: true, readable: false, size: fileStat.size };
|
if (!fileStat.isFile()) return { exists: true, readable: false, size: fileStat.size, mtimeMs: fileStat.mtimeMs };
|
||||||
return { exists: true, readable: true, size: fileStat.size };
|
return { exists: true, readable: true, size: fileStat.size, mtimeMs: fileStat.mtimeMs };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error && (error.code === 'ENOENT' || error.code === 'ENOTDIR')) return { exists: false };
|
if (error && (error.code === 'ENOENT' || error.code === 'ENOTDIR')) return { exists: false };
|
||||||
return { exists: true, readable: false };
|
return { exists: true, readable: false };
|
||||||
@@ -109,8 +110,9 @@
|
|||||||
try {
|
try {
|
||||||
const result = await inspectPath(entry.path, entry);
|
const result = await inspectPath(entry.path, entry);
|
||||||
const reason = unavailableReason(result);
|
const reason = unavailableReason(result);
|
||||||
if (reason) return { entry: { ...entry, size: Number(result?.size) || 0 }, reason };
|
const mtimeMs = Number.isFinite(Number(result?.mtimeMs)) ? Number(result.mtimeMs) : entry.mtimeMs;
|
||||||
return { entry: { ...entry, size: Number(result.size) }, reason: '' };
|
if (reason) return { entry: { ...entry, size: Number(result?.size) || 0, mtimeMs }, reason };
|
||||||
|
return { entry: { ...entry, size: Number(result.size), mtimeMs }, reason: '' };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return { entry, reason: error && error.code === 'ENOENT' ? 'missing' : 'unreadable' };
|
return { entry, reason: error && error.code === 'ENOENT' ? 'missing' : 'unreadable' };
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-2
@@ -106,9 +106,11 @@
|
|||||||
const normalizedBaseName = baseName.toLowerCase();
|
const normalizedBaseName = baseName.toLowerCase();
|
||||||
const normalizedExt = ext.toLowerCase();
|
const normalizedExt = ext.toLowerCase();
|
||||||
if (!normalizedExt || !normalizedValue.endsWith(normalizedExt)) return false;
|
if (!normalizedExt || !normalizedValue.endsWith(normalizedExt)) return false;
|
||||||
if (stripModeStampFromFileName(normalizedValue) === `${normalizedBaseName}${normalizedExt}`) return true;
|
|
||||||
const stem = normalizedValue.slice(0, -normalizedExt.length);
|
const stem = normalizedValue.slice(0, -normalizedExt.length);
|
||||||
return /^\d{2}-\d{2}-\d{4}-mdu-session-\d{2}-\d{2}(?:-\d+)?$/.test(stem);
|
const managedStem = stem.replace(/\.\d+$/, '');
|
||||||
|
const managedValue = `${managedStem}${normalizedExt}`;
|
||||||
|
if (stripModeStampFromFileName(managedValue) === `${normalizedBaseName}${normalizedExt}`) return true;
|
||||||
|
return /^\d{2}-\d{2}-\d{4}-mdu-session-\d{2}-\d{2}(?:-\d+)?$/.test(managedStem);
|
||||||
}
|
}
|
||||||
|
|
||||||
const api = { normalizeLogMode, resolveLogFileName, formatDateStamp, formatSessionStamp, stripModeStampFromFileName, isManagedUploadLogFileName, VALID_MODES };
|
const api = { normalizeLogMode, resolveLogFileName, formatDateStamp, formatSessionStamp, stripModeStampFromFileName, isManagedUploadLogFileName, VALID_MODES };
|
||||||
|
|||||||
@@ -42,6 +42,7 @@
|
|||||||
function classifyErrorCategory(err) {
|
function classifyErrorCategory(err) {
|
||||||
if (!err || typeof err !== 'string') return 'unknown';
|
if (!err || typeof err !== 'string') return 'unknown';
|
||||||
const s = err.toLowerCase();
|
const s = err.toLowerCase();
|
||||||
|
if (/automatik-abschlussnachweis.*nicht gespeichert|automation completion evidence.*not saved/.test(s)) return 'local-persistence';
|
||||||
if (/abgebrochen|aborted|cancel/.test(s)) return 'aborted';
|
if (/abgebrochen|aborted|cancel/.test(s)) return 'aborted';
|
||||||
if (/not video file format|kein videoformat|invalid file|wrong format|duplicate|already exists|file too (small|big|large)|datei zu (gro|klein)/.test(s)) return 'file-rejected';
|
if (/not video file format|kein videoformat|invalid file|wrong format|duplicate|already exists|file too (small|big|large)|datei zu (gro|klein)/.test(s)) return 'file-rejected';
|
||||||
if (/quota|storage (full|exhausted|voll)|account (full|banned|suspended)|disk (space )?full|insufficient (disk )?space|not enough (disk )?(space|storage)/.test(s)) return 'account-error';
|
if (/quota|storage (full|exhausted|voll)|account (full|banned|suspended)|disk (space )?full|insufficient (disk )?space|not enough (disk )?(space|storage)/.test(s)) return 'account-error';
|
||||||
@@ -57,6 +58,7 @@
|
|||||||
'hoster-transient': [],
|
'hoster-transient': [],
|
||||||
'network': [],
|
'network': [],
|
||||||
'unknown': [],
|
'unknown': [],
|
||||||
|
'local-persistence': [],
|
||||||
'aborted': []
|
'aborted': []
|
||||||
};
|
};
|
||||||
if (!batchSummary || !Array.isArray(batchSummary.files)) return buckets;
|
if (!batchSummary || !Array.isArray(batchSummary.files)) return buckets;
|
||||||
@@ -129,6 +131,7 @@
|
|||||||
'hoster-transient': 'Hoster-Flake',
|
'hoster-transient': 'Hoster-Flake',
|
||||||
'network': 'Netzwerk',
|
'network': 'Netzwerk',
|
||||||
'unknown': 'Unbekannt',
|
'unknown': 'Unbekannt',
|
||||||
|
'local-persistence': 'Lokale Speicherung',
|
||||||
'aborted': 'Abgebrochen'
|
'aborted': 'Abgebrochen'
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
+63
-3
@@ -18,14 +18,74 @@
|
|||||||
if (parts.length < 5) return null;
|
if (parts.length < 5) return null;
|
||||||
const hoster = (parts[1] || '').trim();
|
const hoster = (parts[1] || '').trim();
|
||||||
let fileName = '';
|
let fileName = '';
|
||||||
|
let fileNameIndex = -1;
|
||||||
for (let i = parts.length - 1; i >= 4; i--) {
|
for (let i = parts.length - 1; i >= 4; i--) {
|
||||||
if (parts[i].trim() !== '') { fileName = parts[i]; break; }
|
if (parts[i].trim() !== '') { fileName = parts[i]; fileNameIndex = i; break; }
|
||||||
}
|
}
|
||||||
if (!hoster || !fileName) return null;
|
if (!hoster || !fileName) return null;
|
||||||
|
const confirmed = parts.slice(2, fileNameIndex).some(value => value.trim() !== '');
|
||||||
const tsStr = (parts[0] || '').trim();
|
const tsStr = (parts[0] || '').trim();
|
||||||
const tsParsed = tsStr ? Date.parse(tsStr.replace(' ', 'T')) : NaN;
|
const tsParsed = tsStr ? Date.parse(tsStr.replace(' ', 'T')) : NaN;
|
||||||
const ts = isNaN(tsParsed) ? undefined : tsParsed;
|
const ts = isNaN(tsParsed) ? undefined : tsParsed;
|
||||||
return { hoster, fileName, ts };
|
return { hoster, fileName, ts, confirmed };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function* iterateBoundedUploadLogLines(chunks, maxLineLength) {
|
||||||
|
let buffer = '';
|
||||||
|
for await (const chunk of chunks) {
|
||||||
|
buffer += String(chunk);
|
||||||
|
for (;;) {
|
||||||
|
const separator = buffer.indexOf('\n');
|
||||||
|
if (separator < 0) break;
|
||||||
|
const line = buffer.slice(0, separator).replace(/\r$/, '');
|
||||||
|
if (line.length > maxLineLength) throw new Error('Upload-Log-Zeile ist zu lang');
|
||||||
|
yield line;
|
||||||
|
buffer = buffer.slice(separator + 1);
|
||||||
|
}
|
||||||
|
if (buffer.length > maxLineLength) throw new Error('Upload-Log-Zeile ist zu lang');
|
||||||
|
}
|
||||||
|
if (buffer) yield buffer.replace(/\r$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function* iterateBoundedUploadLogChunks(chunks, maxBytes, onBytes) {
|
||||||
|
const BufferImpl = typeof require === 'function' ? require('node:buffer').Buffer : null;
|
||||||
|
let total = 0;
|
||||||
|
for await (const chunk of chunks) {
|
||||||
|
const bytes = BufferImpl ? BufferImpl.byteLength(String(chunk), 'utf8') : String(chunk).length;
|
||||||
|
total += bytes;
|
||||||
|
if (total > maxBytes) throw new Error('Upload-Log überschreitet das Leselimit');
|
||||||
|
if (typeof onBytes === 'function') onBytes(bytes);
|
||||||
|
yield chunk;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function* iterateUploadLogEntries(filePath, options = {}) {
|
||||||
|
const fsImpl = options.fs || (typeof require === 'function' ? require('node:fs') : null);
|
||||||
|
if (!options.lines && !fsImpl?.createReadStream) throw new Error('Upload-Log-Stream ist nicht verfügbar');
|
||||||
|
const yieldEvery = Number.isFinite(Number(options.yieldEvery)) ? Math.max(1, Math.floor(Number(options.yieldEvery))) : 1000;
|
||||||
|
const maxLineLength = Number.isFinite(Number(options.maxLineLength)) ? Math.max(1, Math.floor(Number(options.maxLineLength))) : 65536;
|
||||||
|
const maxBytes = Number.isFinite(Number(options.maxBytes)) ? Math.max(1, Math.floor(Number(options.maxBytes))) : 256 * 1024 * 1024;
|
||||||
|
const yieldFn = typeof options.yieldFn === 'function' ? options.yieldFn : (() => new Promise(resolve => setImmediate(resolve)));
|
||||||
|
const input = options.lines ? null : fsImpl.createReadStream(filePath, { encoding: 'utf8', highWaterMark: 32768 });
|
||||||
|
const lines = options.lines || iterateBoundedUploadLogLines(iterateBoundedUploadLogChunks(input, maxBytes, options.onBytes), maxLineLength);
|
||||||
|
let count = 0;
|
||||||
|
try {
|
||||||
|
for await (const line of lines) {
|
||||||
|
if (String(line).length > maxLineLength) throw new Error('Upload-Log-Zeile ist zu lang');
|
||||||
|
const parsed = parseUploadLogLine(line);
|
||||||
|
if (parsed) yield parsed;
|
||||||
|
count++;
|
||||||
|
if (count % yieldEvery === 0) await yieldFn();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (input && !input.destroyed && typeof input.destroy === 'function') input.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readUploadLogEntries(filePath, options = {}) {
|
||||||
|
const entries = [];
|
||||||
|
for await (const entry of iterateUploadLogEntries(filePath, options)) entries.push(entry);
|
||||||
|
return entries;
|
||||||
}
|
}
|
||||||
|
|
||||||
function summarizeBatchPlan(payload) {
|
function summarizeBatchPlan(payload) {
|
||||||
@@ -73,7 +133,7 @@
|
|||||||
})}\r\n`;
|
})}\r\n`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const api = { formatUploadLogLine, parseUploadLogLine, summarizeBatchPlan, formatUploadPlanLogLine };
|
const api = { formatUploadLogLine, parseUploadLogLine, iterateUploadLogEntries, readUploadLogEntries, summarizeBatchPlan, formatUploadPlanLogLine };
|
||||||
if (typeof module !== 'undefined' && module.exports) module.exports = api;
|
if (typeof module !== 'undefined' && module.exports) module.exports = api;
|
||||||
else if (root) root.UploadLog = api;
|
else if (root) root.UploadLog = api;
|
||||||
})(typeof window !== 'undefined' ? window : this);
|
})(typeof window !== 'undefined' ? window : this);
|
||||||
|
|||||||
@@ -489,6 +489,7 @@ class UploadManager extends EventEmitter {
|
|||||||
finalStatus = status;
|
finalStatus = status;
|
||||||
|
|
||||||
const result = {
|
const result = {
|
||||||
|
jobId,
|
||||||
hoster: task.hoster,
|
hoster: task.hoster,
|
||||||
status,
|
status,
|
||||||
error: payload.error || null,
|
error: payload.error || null,
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ const { walkFolderAsync } = require('./lib/file-discovery');
|
|||||||
const RemoteServer = require('./lib/remote-server');
|
const RemoteServer = require('./lib/remote-server');
|
||||||
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
||||||
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
||||||
const { formatUploadLogLine, parseUploadLogLine, summarizeBatchPlan, formatUploadPlanLogLine } = require('./lib/upload-log');
|
const { formatUploadLogLine, iterateUploadLogEntries, summarizeBatchPlan, formatUploadPlanLogLine } = require('./lib/upload-log');
|
||||||
const { createInternalLogPathResolver, createInternalLogWriter, createUploadAuditWriter, createBufferedInternalLogFlusher, getLogOpenDirectory } = require('./lib/upload-audit');
|
const { createInternalLogPathResolver, createInternalLogWriter, createUploadAuditWriter, createBufferedInternalLogFlusher, getLogOpenDirectory } = require('./lib/upload-audit');
|
||||||
const { selectOrphanTmps } = require('./lib/orphan-tmp');
|
const { selectOrphanTmps } = require('./lib/orphan-tmp');
|
||||||
const { sanitizeConfig, buildSupportBundleText, collectSecretValues, redactLogText, valueScrub, collectFile, REDACTED } = require('./lib/support-bundle');
|
const { sanitizeConfig, buildSupportBundleText, collectSecretValues, redactLogText, valueScrub, collectFile, REDACTED } = require('./lib/support-bundle');
|
||||||
@@ -39,7 +39,7 @@ const { createCollectors } = require('./lib/diagnostics-collectors');
|
|||||||
const { createAgent } = require('./lib/diagnostics-agent');
|
const { createAgent } = require('./lib/diagnostics-agent');
|
||||||
const { buildSessionReport, buildSessionReportCsv } = require('./lib/session-report');
|
const { buildSessionReport, buildSessionReportCsv } = require('./lib/session-report');
|
||||||
const { inspectImportEntries, inspectReadableImportPath } = require('./lib/import-preflight');
|
const { inspectImportEntries, inspectReadableImportPath } = require('./lib/import-preflight');
|
||||||
const { normalizeAutomationSettings } = require('./lib/automation-control');
|
const { normalizeAutomationSettings, automationCompletionKey, createAutomationCompletionWriter, isPathWithinAutomationFolder, normalizeAutomationCompletion } = require('./lib/automation-control');
|
||||||
|
|
||||||
const _eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
const _eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
||||||
_eventLoopDelay.enable();
|
_eventLoopDelay.enable();
|
||||||
@@ -125,8 +125,10 @@ const updateAnnouncementState = createUpdateAnnouncementState();
|
|||||||
let _lastImportPath = null;
|
let _lastImportPath = null;
|
||||||
let dropTargetWindow = null;
|
let dropTargetWindow = null;
|
||||||
let tray = null;
|
let tray = null;
|
||||||
|
let _cachedLogSettings = null;
|
||||||
const configStore = new ConfigStore(app);
|
const configStore = new ConfigStore(app);
|
||||||
configStore.setPerfLog((m) => { try { logInfo(m); } catch {} });
|
configStore.setPerfLog((m) => { try { logInfo(m); } catch {} });
|
||||||
|
_setLogSettingsSnapshot((configStore.load() || {}).globalSettings);
|
||||||
const onlineBackupKeyring = createOnlineBackupKeyring({
|
const onlineBackupKeyring = createOnlineBackupKeyring({
|
||||||
filePath: path.join(app.getPath('userData'), 'online-backup-keys.json')
|
filePath: path.join(app.getPath('userData'), 'online-backup-keys.json')
|
||||||
});
|
});
|
||||||
@@ -153,7 +155,7 @@ async function waitForUploadManagerRelease(manager, timeoutMs = 300000) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function requestUploadFinalization(summary) {
|
function requestUploadFinalization(summary, preserveQueue = false) {
|
||||||
const finalizationId = `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
const finalizationId = `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
@@ -167,7 +169,7 @@ function requestUploadFinalization(summary) {
|
|||||||
resolve(value);
|
resolve(value);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
safeSend('upload-batch-done', { summary, finalizationId });
|
safeSend('upload-batch-done', { summary, finalizationId, preserveQueue });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const activeUploadProducerTrackers = new Set();
|
const activeUploadProducerTrackers = new Set();
|
||||||
@@ -181,6 +183,7 @@ function assertConfigWriteAllowed() {
|
|||||||
|
|
||||||
async function waitForConfigStoreWrites() {
|
async function waitForConfigStoreWrites() {
|
||||||
await configStore.drainWrites();
|
await configStore.drainWrites();
|
||||||
|
await configStore.drainAutomationCompletionWrites();
|
||||||
}
|
}
|
||||||
|
|
||||||
const ONLINE_BACKUP_RENDERER_URL = pathToFileURL(path.join(__dirname, 'renderer', 'index.html'));
|
const ONLINE_BACKUP_RENDERER_URL = pathToFileURL(path.join(__dirname, 'renderer', 'index.html'));
|
||||||
@@ -790,18 +793,20 @@ function getDefaultLogFilePath() {
|
|||||||
// (incl. an 8 MB+ history) on every flush — a major long-running main-thread
|
// (incl. an 8 MB+ history) on every flush — a major long-running main-thread
|
||||||
// drag. logFilePath/logMode change only when the user saves settings, so cache
|
// drag. logFilePath/logMode change only when the user saves settings, so cache
|
||||||
// the two strings and invalidate on those saves (see _invalidateLogSettings).
|
// the two strings and invalidate on those saves (see _invalidateLogSettings).
|
||||||
let _cachedLogSettings = null;
|
function _setLogSettingsSnapshot(globalSettings) {
|
||||||
function _getLogSettings() {
|
const settings = globalSettings || {};
|
||||||
if (!_cachedLogSettings) {
|
|
||||||
const gs = (configStore.load() || {}).globalSettings || {};
|
|
||||||
_cachedLogSettings = {
|
_cachedLogSettings = {
|
||||||
logFilePath: String(gs.logFilePath || '').trim(),
|
logFilePath: String(settings.logFilePath || '').trim(),
|
||||||
logMode: gs.logMode || 'single'
|
logMode: settings.logMode || 'single'
|
||||||
};
|
};
|
||||||
}
|
|
||||||
return _cachedLogSettings;
|
|
||||||
}
|
}
|
||||||
function _invalidateLogSettings() { _cachedLogSettings = null; }
|
function _getLogSettings() {
|
||||||
|
return _cachedLogSettings || { logFilePath: '', logMode: 'single' };
|
||||||
|
}
|
||||||
|
function _invalidateLogSettings(globalSettings) {
|
||||||
|
_setLogSettingsSnapshot(globalSettings);
|
||||||
|
_invalidateUploadLogEvidenceCache();
|
||||||
|
}
|
||||||
|
|
||||||
function getBaseLogFilePath() {
|
function getBaseLogFilePath() {
|
||||||
const customPath = _getLogSettings().logFilePath;
|
const customPath = _getLogSettings().logFilePath;
|
||||||
@@ -958,7 +963,9 @@ function _flushUploadLog() {
|
|||||||
_flushUploadLog();
|
_flushUploadLog();
|
||||||
}, 1000);
|
}, 1000);
|
||||||
}
|
}
|
||||||
} else if (target.isFallback && !_uploadLogFallbackWarned) {
|
} else {
|
||||||
|
_invalidateUploadLogEvidenceCache();
|
||||||
|
if (target.isFallback && !_uploadLogFallbackWarned) {
|
||||||
_uploadLogFallbackWarned = true;
|
_uploadLogFallbackWarned = true;
|
||||||
// Auto-persist the working fallback into the user's config so the
|
// Auto-persist the working fallback into the user's config so the
|
||||||
// next session writes here directly (no more fallback ladder) and
|
// next session writes here directly (no more fallback ladder) and
|
||||||
@@ -966,6 +973,7 @@ function _flushUploadLog() {
|
|||||||
_persistFallbackLogPath(target.path);
|
_persistFallbackLogPath(target.path);
|
||||||
safeSend('upload-log-fallback', { fallbackPath: target.path });
|
safeSend('upload-log-fallback', { fallbackPath: target.path });
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if (_uploadLogBuffer.length && !_uploadLogFlushTimer) setImmediate(_flushUploadLog);
|
if (_uploadLogBuffer.length && !_uploadLogFlushTimer) setImmediate(_flushUploadLog);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -993,7 +1001,7 @@ async function _persistFallbackLogPath(workingPath) {
|
|||||||
cfg.globalSettings = gs;
|
cfg.globalSettings = gs;
|
||||||
await configStore.save({ globalSettings: gs });
|
await configStore.save({ globalSettings: gs });
|
||||||
_invalidateUploadLogTargetCache();
|
_invalidateUploadLogTargetCache();
|
||||||
_invalidateLogSettings();
|
_invalidateLogSettings(gs);
|
||||||
safeSend('log-path-auto-updated', { logFilePath: toSave });
|
safeSend('log-path-auto-updated', { logFilePath: toSave });
|
||||||
return true;
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -1225,6 +1233,42 @@ function buildUploadTasksFromJobs(config, jobs, pick) {
|
|||||||
return tasks;
|
return tasks;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function registerAutomationCompletionJobs(manager, jobs) {
|
||||||
|
if (!manager || !Array.isArray(jobs)) return;
|
||||||
|
if (!manager._automationCompletionMetadata) manager._automationCompletionMetadata = new Map();
|
||||||
|
const folderSettings = configStore.load().globalSettings?.folderMonitor || {};
|
||||||
|
const candidates = jobs.filter(job => {
|
||||||
|
const monitoredManualJob = folderSettings.enabled === true && isPathWithinAutomationFolder(job?.file, folderSettings.folderPath, folderSettings.recursive === true);
|
||||||
|
return (job?.automationAdmission === true || monitoredManualJob) && job.id && job.file && job.hoster;
|
||||||
|
});
|
||||||
|
let cursor = 0;
|
||||||
|
const hasFiniteMetadata = value => value !== null && value !== undefined && value !== '' && Number.isFinite(Number(value));
|
||||||
|
async function worker() {
|
||||||
|
while (cursor < candidates.length) {
|
||||||
|
const job = candidates[cursor++];
|
||||||
|
const sourceSize = job.sourceSize ?? job.automationSize ?? job.bytesTotal;
|
||||||
|
const sourceMtimeMs = job.sourceMtimeMs ?? job.automationMtimeMs;
|
||||||
|
let size = hasFiniteMetadata(sourceSize) ? Number(sourceSize) : Number.NaN;
|
||||||
|
let mtimeMs = hasFiniteMetadata(sourceMtimeMs) ? Number(sourceMtimeMs) : Number.NaN;
|
||||||
|
if (!Number.isFinite(size) || !Number.isFinite(mtimeMs)) {
|
||||||
|
try {
|
||||||
|
const stat = await fs.promises.stat(job.file);
|
||||||
|
size = Number(stat.size);
|
||||||
|
mtimeMs = Number(stat.mtimeMs);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
if (!Number.isFinite(size) || !Number.isFinite(mtimeMs)) continue;
|
||||||
|
manager._automationCompletionMetadata.set(job.id, {
|
||||||
|
path: job.file,
|
||||||
|
size,
|
||||||
|
mtimeMs,
|
||||||
|
hoster: job.hoster
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await Promise.all(Array.from({ length: Math.min(16, candidates.length) }, worker));
|
||||||
|
}
|
||||||
|
|
||||||
async function checkDoodstreamHealth(hosterConfig, otp) {
|
async function checkDoodstreamHealth(hosterConfig, otp) {
|
||||||
const username = hosterConfig && hosterConfig.username
|
const username = hosterConfig && hosterConfig.username
|
||||||
? String(hosterConfig.username).trim()
|
? String(hosterConfig.username).trim()
|
||||||
@@ -1828,7 +1872,7 @@ ipcMain.handle('get-config', () => {
|
|||||||
ipcMain.handle('save-config', async (_event, config) => {
|
ipcMain.handle('save-config', async (_event, config) => {
|
||||||
assertConfigWriteAllowed();
|
assertConfigWriteAllowed();
|
||||||
await configStore.save(config);
|
await configStore.save(config);
|
||||||
if (config && config.globalSettings) _invalidateLogSettings();
|
if (config && config.globalSettings) _invalidateLogSettings(config.globalSettings);
|
||||||
try {
|
try {
|
||||||
if (config && config.globalSettings && Object.prototype.hasOwnProperty.call(config.globalSettings, 'logVerbose')) {
|
if (config && config.globalSettings && Object.prototype.hasOwnProperty.call(config.globalSettings, 'logVerbose')) {
|
||||||
setLogVerbose(!!config.globalSettings.logVerbose);
|
setLogVerbose(!!config.globalSettings.logVerbose);
|
||||||
@@ -2294,6 +2338,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
uploadManager = new UploadManager(config.hosterSettings || {}, config.globalSettings || {}, buildAccountPools(config));
|
uploadManager = new UploadManager(config.hosterSettings || {}, config.globalSettings || {}, buildAccountPools(config));
|
||||||
globalThis._mhuUploadManagerRef = uploadManager;
|
globalThis._mhuUploadManagerRef = uploadManager;
|
||||||
const _thisManager = uploadManager;
|
const _thisManager = uploadManager;
|
||||||
|
await registerAutomationCompletionJobs(_thisManager, jobs);
|
||||||
|
|
||||||
await appendUploadPlanAudit(batchPlan, 'start');
|
await appendUploadPlanAudit(batchPlan, 'start');
|
||||||
if (configStore.load().globalSettings?.folderMonitor?.paused === true) {
|
if (configStore.load().globalSettings?.folderMonitor?.paused === true) {
|
||||||
@@ -2379,6 +2424,35 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
}, PROGRESS_BATCH_INTERVAL_MS);
|
}, PROGRESS_BATCH_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function _queueProgressForRenderer(data) {
|
||||||
|
const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped';
|
||||||
|
if (isTerminal) {
|
||||||
|
if (data.jobId) _progressByJob.delete(data.jobId);
|
||||||
|
_progressTerminalQueue.push(data);
|
||||||
|
} else if (data.jobId) {
|
||||||
|
_progressByJob.set(data.jobId, data);
|
||||||
|
} else {
|
||||||
|
_progressTerminalQueue.push(data);
|
||||||
|
}
|
||||||
|
_scheduleProgressFlush();
|
||||||
|
}
|
||||||
|
|
||||||
|
_thisManager._automationCompletionProgress = new Map();
|
||||||
|
_thisManager._automationCompletionWriter = createAutomationCompletionWriter({
|
||||||
|
schedule: callback => setTimeout(callback, 100),
|
||||||
|
save: entries => configStore.saveAutomationCompletions(entries),
|
||||||
|
onPersisted: entries => {
|
||||||
|
for (const entry of entries) {
|
||||||
|
const key = automationCompletionKey(entry);
|
||||||
|
const progress = _thisManager._automationCompletionProgress.get(key);
|
||||||
|
if (!progress) continue;
|
||||||
|
_thisManager._automationCompletionProgress.delete(key);
|
||||||
|
_queueProgressForRenderer(progress);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onError: error => debugLog(`automation completion ledger failed: ${error.message}`)
|
||||||
|
});
|
||||||
|
|
||||||
uploadManager.on('progress', (data) => {
|
uploadManager.on('progress', (data) => {
|
||||||
if (data.status !== 'uploading') {
|
if (data.status !== 'uploading') {
|
||||||
debugLog(`progress: ${data.fileName} ${data.hoster} ${data.status} ${data.error || ''}`);
|
debugLog(`progress: ${data.fileName} ${data.hoster} ${data.status} ${data.error || ''}`);
|
||||||
@@ -2389,6 +2463,7 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (data.status === 'done' && data.result) {
|
if (data.status === 'done' && data.result) {
|
||||||
|
_invalidateUploadLogEvidenceCache();
|
||||||
const link = data.result.download_url || data.result.embed_url || data.result.file_code || '';
|
const link = data.result.download_url || data.result.embed_url || data.result.file_code || '';
|
||||||
if (link) {
|
if (link) {
|
||||||
if (shouldLogHosterToFile(data.hoster)) {
|
if (shouldLogHosterToFile(data.hoster)) {
|
||||||
@@ -2400,16 +2475,16 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
debugLog(`WARNING: done but no link for ${data.fileName} @ ${data.hoster}: ${JSON.stringify(data.result)}`);
|
debugLog(`WARNING: done but no link for ${data.fileName} @ ${data.hoster}: ${JSON.stringify(data.result)}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped';
|
if (data.status === 'done' && data.jobId) {
|
||||||
if (isTerminal) {
|
const completion = _thisManager._automationCompletionMetadata?.get(data.jobId);
|
||||||
if (data.jobId) _progressByJob.delete(data.jobId);
|
if (completion) {
|
||||||
_progressTerminalQueue.push(data);
|
const entry = { ...completion, completedAt: Date.now() };
|
||||||
} else if (data.jobId) {
|
_thisManager._automationCompletionProgress.set(automationCompletionKey(entry), data);
|
||||||
_progressByJob.set(data.jobId, data);
|
_thisManager._automationCompletionWriter.add(entry);
|
||||||
} else {
|
return;
|
||||||
_progressTerminalQueue.push(data);
|
|
||||||
}
|
}
|
||||||
_scheduleProgressFlush();
|
}
|
||||||
|
_queueProgressForRenderer(data);
|
||||||
});
|
});
|
||||||
|
|
||||||
uploadManager.on('stats', (data) => {
|
uploadManager.on('stats', (data) => {
|
||||||
@@ -2493,6 +2568,34 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
// orphans (cancel/addJobs see null, the new batch keeps running invisibly).
|
// orphans (cancel/addJobs see null, the new batch keeps running invisibly).
|
||||||
uploadManager.on('batch-done', async (summary) => {
|
uploadManager.on('batch-done', async (summary) => {
|
||||||
summary = stats.mergeSkippedIntoSummary(summary, skippedJobs);
|
summary = stats.mergeSkippedIntoSummary(summary, skippedJobs);
|
||||||
|
let automationCompletionsPersisted = true;
|
||||||
|
try { await _thisManager._automationCompletionWriter?.flush(); } catch (error) {
|
||||||
|
automationCompletionsPersisted = false;
|
||||||
|
debugLog(`automation completion ledger failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
if (!automationCompletionsPersisted) {
|
||||||
|
const failedJobIds = new Set();
|
||||||
|
for (const progress of _thisManager._automationCompletionProgress.values()) {
|
||||||
|
if (progress.jobId) failedJobIds.add(progress.jobId);
|
||||||
|
_queueProgressForRenderer({
|
||||||
|
...progress,
|
||||||
|
status: 'error',
|
||||||
|
error: 'Automatik-Abschlussnachweis konnte nicht gespeichert werden'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
_thisManager._automationCompletionProgress.clear();
|
||||||
|
let changed = 0;
|
||||||
|
for (const file of summary.files || []) {
|
||||||
|
for (const result of file.results || []) {
|
||||||
|
if (!failedJobIds.has(result.jobId)) continue;
|
||||||
|
result.status = 'error';
|
||||||
|
result.error = 'Automatik-Abschlussnachweis konnte nicht gespeichert werden';
|
||||||
|
changed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
summary.succeeded = Math.max(0, Number(summary.succeeded) - changed);
|
||||||
|
summary.failed = Math.max(0, Number(summary.failed) + changed);
|
||||||
|
}
|
||||||
lastSessionSummary = summary;
|
lastSessionSummary = summary;
|
||||||
debugLog(`batch-done: total=${summary.total} ok=${summary.succeeded} fail=${summary.failed}`);
|
debugLog(`batch-done: total=${summary.total} ok=${summary.succeeded} fail=${summary.failed}`);
|
||||||
logMarker('BATCH END', { total: summary.total, ok: summary.succeeded, fail: summary.failed });
|
logMarker('BATCH END', { total: summary.total, ok: summary.succeeded, fail: summary.failed });
|
||||||
@@ -2513,10 +2616,13 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
for (const value of _progressByJob.values()) finalProgressBatch.push(value);
|
for (const value of _progressByJob.values()) finalProgressBatch.push(value);
|
||||||
_progressByJob.clear();
|
_progressByJob.clear();
|
||||||
if (finalProgressBatch.length) safeSend('upload-progress-batch', finalProgressBatch);
|
if (finalProgressBatch.length) safeSend('upload-progress-batch', finalProgressBatch);
|
||||||
const queuePersisted = await requestUploadFinalization(summary);
|
const queuePersisted = await requestUploadFinalization(summary, !automationCompletionsPersisted);
|
||||||
|
const finalizationPersisted = queuePersisted && automationCompletionsPersisted;
|
||||||
|
if (finalizationPersisted) {
|
||||||
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
|
try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); }
|
||||||
if (!queuePersisted) debugLog('upload finalization blocked: renderer queue acknowledgement missing');
|
}
|
||||||
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted });
|
if (!finalizationPersisted) debugLog('upload finalization blocked: queue or automation completion evidence was not persisted');
|
||||||
|
await sourceCleanup.finishBatch({ historyPersisted, queuePersisted: finalizationPersisted });
|
||||||
_producerTracker.finish();
|
_producerTracker.finish();
|
||||||
|
|
||||||
const fullyAborted = isAllAborted(summary);
|
const fullyAborted = isAllAborted(summary);
|
||||||
@@ -2615,12 +2721,12 @@ ipcMain.handle('add-jobs-to-batch', async (_event, payload) => {
|
|||||||
return { added: 0, skippedJobs, alreadyInBatchJobIds: [], sourceCleanupFingerprints };
|
return { added: 0, skippedJobs, alreadyInBatchJobIds: [], sourceCleanupFingerprints };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await registerAutomationCompletionJobs(batchManager, jobs);
|
||||||
const addResult = batchManager.addJobs(tasks);
|
const addResult = batchManager.addJobs(tasks);
|
||||||
const added = typeof addResult === 'number' ? addResult : (addResult && addResult.added) || 0;
|
const added = typeof addResult === 'number' ? addResult : (addResult && addResult.added) || 0;
|
||||||
const alreadyInBatchJobIds = (addResult && Array.isArray(addResult.alreadyInBatchJobIds))
|
const alreadyInBatchJobIds = (addResult && Array.isArray(addResult.alreadyInBatchJobIds))
|
||||||
? addResult.alreadyInBatchJobIds
|
? addResult.alreadyInBatchJobIds
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
debugLog(
|
debugLog(
|
||||||
`add-jobs-to-batch: ${added} of ${tasks.length} tasks added (${alreadyInBatchJobIds.length} already in batch, ${skippedJobs.length} skipped)`
|
`add-jobs-to-batch: ${added} of ${tasks.length} tasks added (${alreadyInBatchJobIds.length} already in batch, ${skippedJobs.length} skipped)`
|
||||||
);
|
);
|
||||||
@@ -2883,8 +2989,8 @@ async function applyImportedSettings(imported) {
|
|||||||
_rotationCursors = {};
|
_rotationCursors = {};
|
||||||
_accountCooldowns.clear();
|
_accountCooldowns.clear();
|
||||||
_sessionAccountOverrides.clear();
|
_sessionAccountOverrides.clear();
|
||||||
_invalidateLogSettings();
|
|
||||||
const config = configStore.load();
|
const config = configStore.load();
|
||||||
|
_invalidateLogSettings(config.globalSettings);
|
||||||
const warnings = await syncImportedRuntime(config);
|
const warnings = await syncImportedRuntime(config);
|
||||||
return { config, warnings };
|
return { config, warnings };
|
||||||
} finally {
|
} finally {
|
||||||
@@ -3015,23 +3121,37 @@ ipcMain.handle('online-backup:restore', async (_event, key) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
ipcMain.handle('read-own-upload-log', async () => {
|
let _uploadLogEvidenceCache = null;
|
||||||
|
let _uploadLogEvidenceInFlight = null;
|
||||||
|
let _uploadLogEvidenceGeneration = 0;
|
||||||
|
|
||||||
|
function _invalidateUploadLogEvidenceCache() {
|
||||||
|
_uploadLogEvidenceCache = null;
|
||||||
|
_uploadLogEvidenceGeneration++;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function _scanOwnUploadLog() {
|
||||||
const entries = new Map();
|
const entries = new Map();
|
||||||
const basePath = getBaseLogFilePath();
|
const basePath = getBaseLogFilePath();
|
||||||
const dir = path.dirname(basePath);
|
const dir = path.dirname(basePath);
|
||||||
const ext = path.extname(basePath);
|
const ext = path.extname(basePath);
|
||||||
const name = path.basename(basePath, ext);
|
const name = path.basename(basePath, ext);
|
||||||
|
|
||||||
const activeTarget = _resolveUploadLogTarget();
|
|
||||||
const directories = new Set([dir]);
|
const directories = new Set([dir]);
|
||||||
if (activeTarget?.path) directories.add(path.dirname(activeTarget.path));
|
if (_activeLogPath) directories.add(path.dirname(_activeLogPath));
|
||||||
const desktop = getSafeDesktopDir();
|
try {
|
||||||
|
const desktop = app.getPath('desktop');
|
||||||
if (desktop) directories.add(desktop);
|
if (desktop) directories.add(desktop);
|
||||||
|
} catch {}
|
||||||
try { directories.add(app.getPath('userData')); } catch {}
|
try { directories.add(app.getPath('userData')); } catch {}
|
||||||
const logFiles = new Set();
|
const logFiles = new Set();
|
||||||
for (const directory of directories) {
|
for (const directory of directories) {
|
||||||
try {
|
try {
|
||||||
for (const file of fs.readdirSync(directory)) {
|
const directoryHandle = await fs.promises.opendir(directory);
|
||||||
|
let directoryEntries = 0;
|
||||||
|
for await (const entry of directoryHandle) {
|
||||||
|
directoryEntries++;
|
||||||
|
if (directoryEntries > 50000) throw new Error('Upload-Log-Verzeichnis enthält zu viele Einträge');
|
||||||
|
const file = entry.name;
|
||||||
if (
|
if (
|
||||||
isManagedUploadLogFileName(file, { baseName: name, ext })
|
isManagedUploadLogFileName(file, { baseName: name, ext })
|
||||||
|| isManagedUploadLogFileName(file, { baseName: 'fileuploader', ext: '.log' })
|
|| isManagedUploadLogFileName(file, { baseName: 'fileuploader', ext: '.log' })
|
||||||
@@ -3039,26 +3159,63 @@ ipcMain.handle('read-own-upload-log', async () => {
|
|||||||
logFiles.add(path.join(directory, file));
|
logFiles.add(path.join(directory, file));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {}
|
} catch (error) {
|
||||||
|
if (error?.code !== 'ENOENT') throw error;
|
||||||
}
|
}
|
||||||
if (activeTarget?.path && fs.existsSync(activeTarget.path)) logFiles.add(activeTarget.path);
|
}
|
||||||
if (fs.existsSync(basePath)) logFiles.add(basePath);
|
if (_activeLogPath) logFiles.add(_activeLogPath);
|
||||||
|
logFiles.add(basePath);
|
||||||
|
if (logFiles.size > 256) throw new Error('Zu viele verwaltete Upload-Logs');
|
||||||
|
|
||||||
for (const logPath of logFiles) {
|
let expectedBytes = 0;
|
||||||
|
let actualBytes = 0;
|
||||||
|
for (const logPath of [...logFiles].sort()) {
|
||||||
try {
|
try {
|
||||||
const content = await fs.promises.readFile(logPath, 'utf-8');
|
if (typeof fs.promises.stat === 'function') {
|
||||||
for (const line of content.split('\n')) {
|
const stat = await fs.promises.stat(logPath);
|
||||||
const parsed = parseUploadLogLine(line);
|
expectedBytes += Number(stat.size) || 0;
|
||||||
if (!parsed) continue;
|
if (expectedBytes > 256 * 1024 * 1024) throw new Error('Upload-Logs überschreiten das Leselimit');
|
||||||
|
}
|
||||||
|
for await (const parsed of iterateUploadLogEntries(logPath, {
|
||||||
|
maxBytes: 256 * 1024 * 1024,
|
||||||
|
onBytes(bytes) {
|
||||||
|
actualBytes += bytes;
|
||||||
|
if (actualBytes > 256 * 1024 * 1024) throw new Error('Upload-Logs überschreiten das Leselimit');
|
||||||
|
}
|
||||||
|
})) {
|
||||||
|
if (parsed.confirmed !== true) continue;
|
||||||
const key = `${parsed.hoster.toLowerCase()}\u0000${parsed.fileName.toLowerCase()}`;
|
const key = `${parsed.hoster.toLowerCase()}\u0000${parsed.fileName.toLowerCase()}`;
|
||||||
const previous = entries.get(key);
|
const previous = entries.get(key);
|
||||||
const timestamp = Number.isFinite(parsed.ts) ? parsed.ts : -Infinity;
|
const timestamp = Number.isFinite(parsed.ts) ? parsed.ts : -Infinity;
|
||||||
const previousTimestamp = Number.isFinite(previous?.ts) ? previous.ts : -Infinity;
|
const previousTimestamp = Number.isFinite(previous?.ts) ? previous.ts : -Infinity;
|
||||||
if (!previous || timestamp >= previousTimestamp) entries.set(key, parsed);
|
if (!previous || timestamp >= previousTimestamp) entries.set(key, parsed);
|
||||||
|
if (entries.size > 250000) throw new Error('Upload-Log enthält zu viele eindeutige Einträge');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.code !== 'ENOENT') throw error;
|
||||||
}
|
}
|
||||||
} catch {}
|
|
||||||
}
|
}
|
||||||
return [...entries.values()];
|
return [...entries.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
ipcMain.handle('read-own-upload-log', async () => {
|
||||||
|
const now = Date.now();
|
||||||
|
if (_uploadLogEvidenceCache?.expiresAt > now) return _uploadLogEvidenceCache.entries;
|
||||||
|
const generation = _uploadLogEvidenceGeneration;
|
||||||
|
if (_uploadLogEvidenceInFlight?.generation === generation) return _uploadLogEvidenceInFlight.promise;
|
||||||
|
const pending = _scanOwnUploadLog().then(entries => {
|
||||||
|
if (generation === _uploadLogEvidenceGeneration) {
|
||||||
|
_uploadLogEvidenceCache = { entries, expiresAt: Date.now() + 5000 };
|
||||||
|
}
|
||||||
|
return entries;
|
||||||
|
});
|
||||||
|
const inFlight = { generation, promise: pending };
|
||||||
|
_uploadLogEvidenceInFlight = inFlight;
|
||||||
|
try {
|
||||||
|
return await pending;
|
||||||
|
} finally {
|
||||||
|
if (_uploadLogEvidenceInFlight === inFlight) _uploadLogEvidenceInFlight = null;
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
ipcMain.handle('import-upload-log', async () => {
|
ipcMain.handle('import-upload-log', async () => {
|
||||||
@@ -3252,7 +3409,7 @@ ipcMain.handle('save-global-settings', async (_event, globalSettings) => {
|
|||||||
assertConfigWriteAllowed();
|
assertConfigWriteAllowed();
|
||||||
await configStore.saveRendererGlobalSettings(globalSettings);
|
await configStore.saveRendererGlobalSettings(globalSettings);
|
||||||
globalSettings = configStore.load().globalSettings;
|
globalSettings = configStore.load().globalSettings;
|
||||||
_invalidateLogSettings();
|
_invalidateLogSettings(globalSettings);
|
||||||
if (uploadManager) {
|
if (uploadManager) {
|
||||||
try { uploadManager.updateSettings(null, globalSettings); } catch (error) { debugLog(`global settings runtime update failed: ${error.message}`); }
|
try { uploadManager.updateSettings(null, globalSettings); } catch (error) { debugLog(`global settings runtime update failed: ${error.message}`); }
|
||||||
}
|
}
|
||||||
@@ -3467,6 +3624,18 @@ ipcMain.handle('automation:get-status', () => {
|
|||||||
return automationStatusSnapshot();
|
return automationStatusSnapshot();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('automation:get-completions', () => {
|
||||||
|
return configStore.loadAutomationCompletions();
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('automation:record-completions', async (_event, entries) => {
|
||||||
|
const source = Array.isArray(entries) ? entries.slice(0, 10000) : [];
|
||||||
|
const normalized = source.map(normalizeAutomationCompletion);
|
||||||
|
if (source.length === 0 || normalized.some(entry => !entry)) throw new Error('Automatik-Abschlussnachweise sind ungültig');
|
||||||
|
await configStore.saveAutomationCompletions(normalized);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
ipcMain.handle('automation:pause-after-active', () => enqueueAutomationLifecycle(async generation => {
|
ipcMain.handle('automation:pause-after-active', () => enqueueAutomationLifecycle(async generation => {
|
||||||
let monitorError = '';
|
let monitorError = '';
|
||||||
await withAutomationStatusSuppressed(async () => {
|
await withAutomationStatusSuppressed(async () => {
|
||||||
|
|||||||
@@ -110,6 +110,8 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
folderMonitorTestScan: () => ipcRenderer.invoke('folder-monitor:test-scan'),
|
folderMonitorTestScan: () => ipcRenderer.invoke('folder-monitor:test-scan'),
|
||||||
folderMonitorReconcile: () => ipcRenderer.invoke('folder-monitor:reconcile'),
|
folderMonitorReconcile: () => ipcRenderer.invoke('folder-monitor:reconcile'),
|
||||||
automationGetStatus: () => ipcRenderer.invoke('automation:get-status'),
|
automationGetStatus: () => ipcRenderer.invoke('automation:get-status'),
|
||||||
|
getAutomationCompletions: () => ipcRenderer.invoke('automation:get-completions'),
|
||||||
|
recordAutomationCompletions: (entries) => ipcRenderer.invoke('automation:record-completions', entries),
|
||||||
automationPauseAfterActive: () => ipcRenderer.invoke('automation:pause-after-active'),
|
automationPauseAfterActive: () => ipcRenderer.invoke('automation:pause-after-active'),
|
||||||
automationResume: () => ipcRenderer.invoke('automation:resume'),
|
automationResume: () => ipcRenderer.invoke('automation:resume'),
|
||||||
onFolderMonitorNewFiles: (callback) => {
|
onFolderMonitorNewFiles: (callback) => {
|
||||||
|
|||||||
+178
-35
@@ -397,6 +397,7 @@ let _updateDialogReturnFocus = null;
|
|||||||
let _updateDialogInertState = [];
|
let _updateDialogInertState = [];
|
||||||
let _startupAutoResumeController = null;
|
let _startupAutoResumeController = null;
|
||||||
let _startupAutoResumeCanceled = false;
|
let _startupAutoResumeCanceled = false;
|
||||||
|
let _startupQueueEvidenceAvailable = true;
|
||||||
|
|
||||||
// Session-specific files for the "Files" panel (resets each session)
|
// Session-specific files for the "Files" panel (resets each session)
|
||||||
let sessionFilesData = [];
|
let sessionFilesData = [];
|
||||||
@@ -556,11 +557,12 @@ function createAutomationStatusSnapshot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function loadAutomationEvidenceSnapshot() {
|
async function loadAutomationEvidenceSnapshot() {
|
||||||
const [history, uploadLog] = await Promise.all([
|
const [history, uploadLog, automationCompletions] = await Promise.all([
|
||||||
window.api.getHistory(),
|
window.api.getHistory(),
|
||||||
window.api.readOwnUploadLog()
|
window.api.readOwnUploadLog(),
|
||||||
|
typeof window.api.getAutomationCompletions === 'function' ? window.api.getAutomationCompletions() : Promise.resolve([])
|
||||||
]);
|
]);
|
||||||
return { history, uploadLog };
|
return { history, uploadLog, automationCompletions };
|
||||||
}
|
}
|
||||||
|
|
||||||
function invalidateAutomationEvidenceSnapshot() {
|
function invalidateAutomationEvidenceSnapshot() {
|
||||||
@@ -598,14 +600,16 @@ async function evaluateAutomationCandidates(files, options = {}) {
|
|||||||
const selectedHosters = Array.from(new Set((Array.isArray(options.selectedHosters) ? options.selectedHosters : folderSettings.hosters || [])
|
const selectedHosters = Array.from(new Set((Array.isArray(options.selectedHosters) ? options.selectedHosters : folderSettings.hosters || [])
|
||||||
.map(value => String(value || '').trim())
|
.map(value => String(value || '').trim())
|
||||||
.filter(Boolean)));
|
.filter(Boolean)));
|
||||||
const { history, uploadLog } = options.evidenceSnapshot || await loadAutomationEvidenceSnapshot();
|
const { history, uploadLog, automationCompletions } = options.evidenceSnapshot || await loadAutomationEvidenceSnapshot();
|
||||||
const completedPaths = [..._completedUploadKeys].map(key => {
|
const currentPaths = new Set([...queueJobs.map(job => job.file), ...selectedFiles.map(file => file.path)].map(normalizeAutomationPath));
|
||||||
const separator = key.lastIndexOf('|');
|
const durableCompletionPaths = new Set((Array.isArray(automationCompletions) ? automationCompletions : []).map(row => normalizeAutomationPath(row?.path)).filter(Boolean));
|
||||||
return separator > 0 ? key.slice(0, separator) : '';
|
const legacyCandidates = matched.filter(candidate => {
|
||||||
}).filter(Boolean);
|
const key = normalizeAutomationPath(candidate.path);
|
||||||
|
return currentPaths.has(key) || !durableCompletionPaths.has(key);
|
||||||
|
});
|
||||||
const processed = window.AutomationControl.classifyProcessedCandidates({
|
const processed = window.AutomationControl.classifyProcessedCandidates({
|
||||||
candidates: matched,
|
candidates: legacyCandidates,
|
||||||
queuePaths: [...queueJobs.map(job => job.file), ...selectedFiles.map(file => file.path), ...completedPaths],
|
queuePaths: [...queueJobs.map(job => job.file), ...selectedFiles.map(file => file.path)],
|
||||||
historyRows: flattenAutomationHistoryRows(history),
|
historyRows: flattenAutomationHistoryRows(history),
|
||||||
uploadLogRows: uploadLog
|
uploadLogRows: uploadLog
|
||||||
});
|
});
|
||||||
@@ -625,7 +629,7 @@ async function evaluateAutomationCandidates(files, options = {}) {
|
|||||||
...(metadata.get(normalizeAutomationPath(file?.path)) || {}),
|
...(metadata.get(normalizeAutomationPath(file?.path)) || {}),
|
||||||
...file
|
...file
|
||||||
}));
|
}));
|
||||||
const plannedCandidates = accepted.map(file => {
|
const initialPlannedCandidates = accepted.map(file => {
|
||||||
const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings);
|
const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings);
|
||||||
return {
|
return {
|
||||||
path: file.path,
|
path: file.path,
|
||||||
@@ -636,6 +640,28 @@ async function evaluateAutomationCandidates(files, options = {}) {
|
|||||||
eligibleJobCount: eligibleHosters.length
|
eligibleJobCount: eligibleHosters.length
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
const completionState = window.AutomationControl.classifyAutomationCompletionLedger({
|
||||||
|
candidates: initialPlannedCandidates,
|
||||||
|
completionRows: automationCompletions
|
||||||
|
});
|
||||||
|
const ledgerProcessedPaths = new Set(completionState.processedPaths.map(normalizeAutomationPath));
|
||||||
|
const completedHostersByPath = new Map(completionState.completedByPath.map(entry => [normalizeAutomationPath(entry.path), new Set(entry.hosters)]));
|
||||||
|
for (const path of ledgerProcessedPaths) {
|
||||||
|
processedPaths.add(path);
|
||||||
|
reasons.set(path, 'processed');
|
||||||
|
}
|
||||||
|
const plannedCandidates = initialPlannedCandidates
|
||||||
|
.filter(candidate => !ledgerProcessedPaths.has(normalizeAutomationPath(candidate.path)))
|
||||||
|
.map(candidate => {
|
||||||
|
const completedHosters = completedHostersByPath.get(normalizeAutomationPath(candidate.path)) || new Set();
|
||||||
|
const eligibleHosters = candidate.eligibleHosters.filter(hoster => !completedHosters.has(String(hoster).toLowerCase()));
|
||||||
|
return {
|
||||||
|
...candidate,
|
||||||
|
eligibleHosters,
|
||||||
|
eligibleJobCount: eligibleHosters.length,
|
||||||
|
completedHosters: [...completedHosters]
|
||||||
|
};
|
||||||
|
});
|
||||||
const normalizedSettings = automationSettings();
|
const normalizedSettings = automationSettings();
|
||||||
const currentJobCount = window.AutomationControl.countAutomaticQueueJobs(queueJobs);
|
const currentJobCount = window.AutomationControl.countAutomaticQueueJobs(queueJobs);
|
||||||
const availableSlots = normalizedSettings.queueLimitJobs === 0 ? null : Math.max(0, normalizedSettings.queueLimitJobs - currentJobCount);
|
const availableSlots = normalizedSettings.queueLimitJobs === 0 ? null : Math.max(0, normalizedSettings.queueLimitJobs - currentJobCount);
|
||||||
@@ -657,6 +683,7 @@ async function evaluateAutomationCandidates(files, options = {}) {
|
|||||||
}
|
}
|
||||||
const actionableCandidates = plannedCandidates.filter(candidate => candidate.eligibleJobCount > 0);
|
const actionableCandidates = plannedCandidates.filter(candidate => candidate.eligibleJobCount > 0);
|
||||||
const resultingJobs = actionableCandidates.reduce((total, candidate) => total + candidate.eligibleJobCount, 0);
|
const resultingJobs = actionableCandidates.reduce((total, candidate) => total + candidate.eligibleJobCount, 0);
|
||||||
|
const sizeLimitedJobs = initialPlannedCandidates.reduce((total, candidate) => total + Math.max(0, selectedHosters.length - candidate.eligibleHosters.length), 0);
|
||||||
const classifications = candidates.map(candidate => ({
|
const classifications = candidates.map(candidate => ({
|
||||||
path: candidate.path,
|
path: candidate.path,
|
||||||
name: candidate.name,
|
name: candidate.name,
|
||||||
@@ -666,9 +693,9 @@ async function evaluateAutomationCandidates(files, options = {}) {
|
|||||||
const summary = {
|
const summary = {
|
||||||
found: candidates.length,
|
found: candidates.length,
|
||||||
filterMatched: matched.length,
|
filterMatched: matched.length,
|
||||||
alreadyProcessed: processed.processedPaths.length + inspectionDuplicatePaths.size,
|
alreadyProcessed: processed.processedPaths.length + inspectionDuplicatePaths.size + ledgerProcessedPaths.size,
|
||||||
unavailable: unavailablePaths.size,
|
unavailable: unavailablePaths.size,
|
||||||
sizeLimitedJobs: plannedCandidates.length * selectedHosters.length - resultingJobs,
|
sizeLimitedJobs,
|
||||||
acceptedFiles: selectedHosters.length === 0 ? plannedCandidates.length : actionableCandidates.length,
|
acceptedFiles: selectedHosters.length === 0 ? plannedCandidates.length : actionableCandidates.length,
|
||||||
selectedTargets: selectedHosters.length,
|
selectedTargets: selectedHosters.length,
|
||||||
resultingJobs,
|
resultingJobs,
|
||||||
@@ -714,6 +741,10 @@ function createAutomationPreviewJob(file, hoster) {
|
|||||||
attempt: 0,
|
attempt: 0,
|
||||||
maxAttempts: 0,
|
maxAttempts: 0,
|
||||||
link: '',
|
link: '',
|
||||||
|
automationMtimeMs: file.mtimeMs,
|
||||||
|
automationSize: file.size,
|
||||||
|
sourceMtimeMs: file.mtimeMs,
|
||||||
|
sourceSize: file.size,
|
||||||
automationAdmission: true
|
automationAdmission: true
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -750,14 +781,17 @@ async function applyAutomationEvaluation(evaluation) {
|
|||||||
.map(value => String(value || '').trim())
|
.map(value => String(value || '').trim())
|
||||||
.filter(Boolean)));
|
.filter(Boolean)));
|
||||||
const replannedCandidates = evaluation.candidates.map(file => {
|
const replannedCandidates = evaluation.candidates.map(file => {
|
||||||
const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings);
|
const completedHosters = new Set((Array.isArray(file.completedHosters) ? file.completedHosters : []).map(hoster => String(hoster || '').toLowerCase()));
|
||||||
|
const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings)
|
||||||
|
.filter(hoster => !completedHosters.has(String(hoster).toLowerCase()));
|
||||||
return {
|
return {
|
||||||
path: file.path,
|
path: file.path,
|
||||||
name: file.name,
|
name: file.name,
|
||||||
size: file.size,
|
size: file.size,
|
||||||
mtimeMs: file.mtimeMs,
|
mtimeMs: file.mtimeMs,
|
||||||
eligibleHosters,
|
eligibleHosters,
|
||||||
eligibleJobCount: eligibleHosters.length
|
eligibleJobCount: eligibleHosters.length,
|
||||||
|
completedHosters: [...completedHosters]
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
const ownedPendingPaths = new Set(Array.isArray(evaluation.ownedPendingPaths) ? evaluation.ownedPendingPaths : []);
|
const ownedPendingPaths = new Set(Array.isArray(evaluation.ownedPendingPaths) ? evaluation.ownedPendingPaths : []);
|
||||||
@@ -1319,6 +1353,7 @@ async function init() {
|
|||||||
renderHosterSummary();
|
renderHosterSummary();
|
||||||
renderHosterModal();
|
renderHosterModal();
|
||||||
renderSettings();
|
renderSettings();
|
||||||
|
if (!_startupQueueEvidenceAvailable) showCopyToast('Automatische Wiederaufnahme wurde wegen nicht verfügbarer Abschlussnachweise blockiert.', 9000);
|
||||||
renderAccounts();
|
renderAccounts();
|
||||||
setupListeners();
|
setupListeners();
|
||||||
importEntryCoordinator.ready();
|
importEntryCoordinator.ready();
|
||||||
@@ -1355,7 +1390,7 @@ async function init() {
|
|||||||
queuePersistThrottle.cancel();
|
queuePersistThrottle.cancel();
|
||||||
await window.api.completeUploadFinalization({
|
await window.api.completeUploadFinalization({
|
||||||
finalizationId: data.finalizationId,
|
finalizationId: data.finalizationId,
|
||||||
pendingQueue: queueJobs.some((job) => !['done', 'skipped'].includes(job.status))
|
pendingQueue: data.preserveQueue === true || queueJobs.some((job) => !['done', 'skipped'].includes(job.status))
|
||||||
? buildPersistedQueueState()
|
? buildPersistedQueueState()
|
||||||
: null
|
: null
|
||||||
});
|
});
|
||||||
@@ -2149,7 +2184,12 @@ function restoreQueueStateFromConfig() {
|
|||||||
selectedFiles = Array.isArray(pending.selectedFiles)
|
selectedFiles = Array.isArray(pending.selectedFiles)
|
||||||
? pending.selectedFiles
|
? pending.selectedFiles
|
||||||
.filter(file => file && file.path)
|
.filter(file => file && file.path)
|
||||||
.map(file => ({ path: file.path, name: file.name || file.path.split(/[\\/]/).pop(), size: file.size || 0 }))
|
.map(file => ({
|
||||||
|
path: file.path,
|
||||||
|
name: file.name || file.path.split(/[\\/]/).pop(),
|
||||||
|
size: file.size || 0,
|
||||||
|
mtimeMs: Number.isFinite(Number(file.mtimeMs)) ? Number(file.mtimeMs) : null
|
||||||
|
}))
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
const interruptedJobIds = new Set(Array.isArray(config?.globalSettings?.uploadRecovery?.jobIds) ? config.globalSettings.uploadRecovery.jobIds : []);
|
const interruptedJobIds = new Set(Array.isArray(config?.globalSettings?.uploadRecovery?.jobIds) ? config.globalSettings.uploadRecovery.jobIds : []);
|
||||||
@@ -2177,6 +2217,10 @@ function restoreQueueStateFromConfig() {
|
|||||||
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
||||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||||
automationAdmission: job.automationAdmission === true,
|
automationAdmission: job.automationAdmission === true,
|
||||||
|
automationMtimeMs: Number.isFinite(Number(job.automationMtimeMs)) ? Number(job.automationMtimeMs) : null,
|
||||||
|
automationSize: Number.isFinite(Number(job.automationSize)) ? Number(job.automationSize) : null,
|
||||||
|
sourceMtimeMs: Number.isFinite(Number(job.sourceMtimeMs)) ? Number(job.sourceMtimeMs) : null,
|
||||||
|
sourceSize: Number.isFinite(Number(job.sourceSize)) ? Number(job.sourceSize) : null,
|
||||||
...(job.automationPaused === true ? { automationPaused: true } : {}),
|
...(job.automationPaused === true ? { automationPaused: true } : {}),
|
||||||
attempt: 0,
|
attempt: 0,
|
||||||
maxAttempts: job.maxAttempts || 0,
|
maxAttempts: job.maxAttempts || 0,
|
||||||
@@ -2208,7 +2252,8 @@ function buildPersistedQueueState() {
|
|||||||
selectedFileMap.set(job.file, {
|
selectedFileMap.set(job.file, {
|
||||||
path: job.file,
|
path: job.file,
|
||||||
name: job.fileName,
|
name: job.fileName,
|
||||||
size: job.bytesTotal || 0
|
size: job.sourceSize ?? job.automationSize ?? job.bytesTotal ?? 0,
|
||||||
|
mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs ?? null
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2257,6 +2302,10 @@ function buildPersistedQueueState() {
|
|||||||
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [],
|
||||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||||
automationAdmission: job.automationAdmission === true,
|
automationAdmission: job.automationAdmission === true,
|
||||||
|
automationMtimeMs: Number.isFinite(Number(job.automationMtimeMs)) ? Number(job.automationMtimeMs) : null,
|
||||||
|
automationSize: Number.isFinite(Number(job.automationSize)) ? Number(job.automationSize) : null,
|
||||||
|
sourceMtimeMs: Number.isFinite(Number(job.sourceMtimeMs)) ? Number(job.sourceMtimeMs) : null,
|
||||||
|
sourceSize: Number.isFinite(Number(job.sourceSize)) ? Number(job.sourceSize) : null,
|
||||||
...(automationPaused ? { automationPaused: true } : {}),
|
...(automationPaused ? { automationPaused: true } : {}),
|
||||||
maxAttempts: job.maxAttempts || 0
|
maxAttempts: job.maxAttempts || 0
|
||||||
};
|
};
|
||||||
@@ -2657,6 +2706,8 @@ function buildQueuePreview() {
|
|||||||
id: `preview-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
id: `preview-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||||
file: file.path, fileName: file.name, hoster,
|
file: file.path, fileName: file.name, hoster,
|
||||||
status: 'preview', bytesUploaded: 0, bytesTotal: file.size || 0,
|
status: 'preview', bytesUploaded: 0, bytesTotal: file.size || 0,
|
||||||
|
sourceMtimeMs: file.mtimeMs,
|
||||||
|
sourceSize: file.size,
|
||||||
speedKbs: 0, elapsed: 0, remaining: 0,
|
speedKbs: 0, elapsed: 0, remaining: 0,
|
||||||
error: null, result: null, attempt: 0, maxAttempts: 0, link: ''
|
error: null, result: null, attempt: 0, maxAttempts: 0, link: ''
|
||||||
};
|
};
|
||||||
@@ -2702,7 +2753,7 @@ async function startRestoredQueueAfterChecks(jobIds) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function scheduleRestoredQueueAutoStart() {
|
function scheduleRestoredQueueAutoStart() {
|
||||||
if (config?.globalSettings?.autoStartRestoredQueue !== true || !window.AutoResume) return;
|
if (!_startupQueueEvidenceAvailable || config?.globalSettings?.autoStartRestoredQueue !== true || !window.AutoResume) return;
|
||||||
const jobs = window.AutoResume.getAutoResumeJobs(queueJobs);
|
const jobs = window.AutoResume.getAutoResumeJobs(queueJobs);
|
||||||
if (!jobs.length) return;
|
if (!jobs.length) return;
|
||||||
_startupAutoResumeCanceled = false;
|
_startupAutoResumeCanceled = false;
|
||||||
@@ -4275,7 +4326,12 @@ function serializeUploadJob(job) {
|
|||||||
sourceCleanupToken: job.sourceCleanupToken || null,
|
sourceCleanupToken: job.sourceCleanupToken || null,
|
||||||
sourceCleanupRequiredHosters: job.sourceCleanupRequiredHosters || [],
|
sourceCleanupRequiredHosters: job.sourceCleanupRequiredHosters || [],
|
||||||
sourceCleanupCompletedHosters: job.sourceCleanupCompletedHosters || [],
|
sourceCleanupCompletedHosters: job.sourceCleanupCompletedHosters || [],
|
||||||
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null
|
sourceCleanupFingerprint: job.sourceCleanupFingerprint || null,
|
||||||
|
automationAdmission: job.automationAdmission === true,
|
||||||
|
automationMtimeMs: job.automationMtimeMs,
|
||||||
|
automationSize: job.automationSize,
|
||||||
|
sourceMtimeMs: job.sourceMtimeMs,
|
||||||
|
sourceSize: job.sourceSize
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5363,7 +5419,8 @@ function syncSelectedFilesFromQueue() {
|
|||||||
fileMap.set(job.file, {
|
fileMap.set(job.file, {
|
||||||
path: job.file,
|
path: job.file,
|
||||||
name: job.fileName,
|
name: job.fileName,
|
||||||
size: job.bytesTotal || 0
|
size: job.sourceSize ?? job.automationSize ?? job.bytesTotal ?? 0,
|
||||||
|
mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs ?? null
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
selectedFiles = Array.from(fileMap.values());
|
selectedFiles = Array.from(fileMap.values());
|
||||||
@@ -9696,31 +9753,76 @@ function handleShutdownCountdown(data) {
|
|||||||
|
|
||||||
// --- Auto-deduplicate restored queue against own upload log on startup ---
|
// --- Auto-deduplicate restored queue against own upload log on startup ---
|
||||||
async function _autoDeduplicateFromLog() {
|
async function _autoDeduplicateFromLog() {
|
||||||
if (queueJobs.length === 0 && selectedFiles.length === 0) return;
|
if (queueJobs.length === 0 && selectedFiles.length === 0) {
|
||||||
|
_startupQueueEvidenceAvailable = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const entries = await window.api.readOwnUploadLog();
|
const [entries, completionRows] = await Promise.all([
|
||||||
if (!entries || entries.length === 0) return;
|
window.api.readOwnUploadLog(),
|
||||||
// Drops 'done' jobs present in the log (declutter) AND any job that the log
|
typeof window.api.getAutomationCompletions === 'function' ? window.api.getAutomationCompletions() : Promise.resolve([])
|
||||||
// shows completed at/after the snapshot's savedAt (a stale 'preview' ghost).
|
]);
|
||||||
// Pending jobs matching only OLDER log lines survive — intentional re-uploads.
|
const ledgerCandidates = queueJobs.map(job => ({
|
||||||
// Decision lives in lib/queue-dedup.js (Node-tested, see tests/queue-dedup.test.js)
|
path: job.file,
|
||||||
// so it can't silently regress to nuking the whole restored queue on restart.
|
size: job.sourceSize ?? job.automationSize ?? job.bytesTotal,
|
||||||
const { kept, removed } = window.QueueDedup.partitionRestoredJobsByLog(queueJobs, entries, _restoredSnapshotSavedAt);
|
mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs,
|
||||||
if (removed.length > 0) {
|
eligibleHosters: [job.hoster]
|
||||||
queueJobs = kept;
|
}));
|
||||||
|
const ledgerState = window.AutomationControl.classifyAutomationCompletionLedger({
|
||||||
|
candidates: ledgerCandidates,
|
||||||
|
completionRows
|
||||||
|
});
|
||||||
|
const ledgerRemovedIds = new Set();
|
||||||
|
for (let index = 0; index < ledgerState.remainingByPath.length; index++) {
|
||||||
|
if (ledgerState.remainingByPath[index].hosters.length === 0 && queueJobs[index]?.id) ledgerRemovedIds.add(queueJobs[index].id);
|
||||||
|
}
|
||||||
|
const removed = [];
|
||||||
|
if (ledgerRemovedIds.size > 0) {
|
||||||
|
queueJobs = queueJobs.filter(job => {
|
||||||
|
if (!ledgerRemovedIds.has(job.id)) return true;
|
||||||
|
removed.push(job);
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (Array.isArray(entries) && entries.length > 0) {
|
||||||
|
const partitioned = window.QueueDedup.partitionRestoredJobsByLog(queueJobs, entries, _restoredSnapshotSavedAt);
|
||||||
|
queueJobs = partitioned.kept;
|
||||||
|
removed.push(...partitioned.removed);
|
||||||
|
}
|
||||||
for (const job of removed) {
|
for (const job of removed) {
|
||||||
if (job.file && job.hoster) _completedUploadKeys.add(`${job.file}|${job.hoster}`);
|
if (job.file && job.hoster) _completedUploadKeys.add(`${job.file}|${job.hoster}`);
|
||||||
}
|
}
|
||||||
|
const selectedLedger = window.AutomationControl.classifyAutomationCompletionLedger({
|
||||||
|
candidates: selectedFiles.map(file => ({
|
||||||
|
path: file.path,
|
||||||
|
size: file.size,
|
||||||
|
mtimeMs: file.mtimeMs,
|
||||||
|
eligibleHosters: getSelectedHosters()
|
||||||
|
})),
|
||||||
|
completionRows
|
||||||
|
});
|
||||||
|
for (const entry of selectedLedger.completedByPath) {
|
||||||
|
for (const hoster of entry.hosters) _completedUploadKeys.add(`${entry.path}|${hoster}`);
|
||||||
|
}
|
||||||
|
if (removed.length > 0) {
|
||||||
rebuildJobIndex();
|
rebuildJobIndex();
|
||||||
syncSelectedFilesFromQueue();
|
syncSelectedFilesFromQueue();
|
||||||
window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`);
|
window.api.debugLog(`auto-dedup: removed ${removed.length} completed jobs from restored queue`);
|
||||||
}
|
}
|
||||||
|
if (Array.isArray(entries) && entries.length > 0) {
|
||||||
const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt);
|
const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt);
|
||||||
if (seedKeys.length > 0) {
|
if (seedKeys.length > 0) {
|
||||||
for (const k of seedKeys) _completedUploadKeys.add(k);
|
for (const key of seedKeys) _completedUploadKeys.add(key);
|
||||||
window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`);
|
window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`);
|
||||||
}
|
}
|
||||||
} catch {}
|
}
|
||||||
|
_startupQueueEvidenceAvailable = true;
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
_startupQueueEvidenceAvailable = false;
|
||||||
|
window.api.debugLog(`startup completion evidence failed: ${error?.message || String(error)}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Log import: remove already-uploaded file+hoster combos from queue ---
|
// --- Log import: remove already-uploaded file+hoster combos from queue ---
|
||||||
@@ -9739,6 +9841,47 @@ async function importUploadLog() {
|
|||||||
logKeys.add(`${entry.fileName.toLowerCase()}|${entry.hoster.toLowerCase()}`);
|
logKeys.add(`${entry.fileName.toLowerCase()}|${entry.hoster.toLowerCase()}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const hasFiniteMetadata = value => value !== null && value !== undefined && value !== '' && Number.isFinite(Number(value));
|
||||||
|
const matchingJobs = queueJobs.filter(job => {
|
||||||
|
const key = `${job.fileName.toLowerCase()}|${job.hoster.toLowerCase()}`;
|
||||||
|
return logKeys.has(key) && job.status !== 'done';
|
||||||
|
});
|
||||||
|
const missingMetadata = matchingJobs.filter(job => {
|
||||||
|
const size = job.sourceSize ?? job.automationSize ?? job.bytesTotal;
|
||||||
|
const mtimeMs = job.sourceMtimeMs ?? job.automationMtimeMs;
|
||||||
|
return !hasFiniteMetadata(size) || !hasFiniteMetadata(mtimeMs);
|
||||||
|
});
|
||||||
|
const inspectedMetadata = new Map();
|
||||||
|
if (missingMetadata.length > 0) {
|
||||||
|
const inspection = await window.api.inspectImportFiles(missingMetadata.map(job => ({ path: job.file, name: job.fileName })), []);
|
||||||
|
for (const file of Array.isArray(inspection?.accepted) ? inspection.accepted : []) {
|
||||||
|
inspectedMetadata.set(normalizeAutomationPath(file.path), file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const completionRows = matchingJobs.map(job => {
|
||||||
|
const inspected = inspectedMetadata.get(normalizeAutomationPath(job.file));
|
||||||
|
return {
|
||||||
|
path: job.file,
|
||||||
|
hoster: job.hoster,
|
||||||
|
size: job.sourceSize ?? job.automationSize ?? inspected?.size ?? job.bytesTotal,
|
||||||
|
mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs ?? inspected?.mtimeMs,
|
||||||
|
completedAt: Date.now()
|
||||||
|
};
|
||||||
|
}).filter(row => hasFiniteMetadata(row.size) && hasFiniteMetadata(row.mtimeMs));
|
||||||
|
if (completionRows.length !== matchingJobs.length) {
|
||||||
|
showCopyToast('Abschlussnachweise konnten nicht vollständig ermittelt werden.', 7000);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (completionRows.length > 0 && typeof window.api.recordAutomationCompletions === 'function') {
|
||||||
|
try {
|
||||||
|
await window.api.recordAutomationCompletions(completionRows);
|
||||||
|
invalidateAutomationEvidenceSnapshot();
|
||||||
|
} catch {
|
||||||
|
showCopyToast('Abschlussnachweise konnten nicht gespeichert werden.', 7000);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Find queue jobs that match (already uploaded)
|
// Find queue jobs that match (already uploaded)
|
||||||
let removed = 0;
|
let removed = 0;
|
||||||
queueJobs = queueJobs.filter(job => {
|
queueJobs = queueJobs.filter(job => {
|
||||||
|
|||||||
@@ -704,6 +704,21 @@
|
|||||||
['Updateprüfung fehlgeschlagen', 'Update check failed'],
|
['Updateprüfung fehlgeschlagen', 'Update check failed'],
|
||||||
['Upload läuft...', 'Uploading...'],
|
['Upload läuft...', 'Uploading...'],
|
||||||
['Upload-Log', 'Upload log'],
|
['Upload-Log', 'Upload log'],
|
||||||
|
['Automatik-Abschlussnachweis konnte nicht gespeichert werden', 'Automation completion evidence could not be saved'],
|
||||||
|
['Lokale Speicherung', 'Local persistence'],
|
||||||
|
['Automatik-Abschlussdatei ist ungültig', 'Automation completion file is invalid'],
|
||||||
|
['Automatik-Abschlussdatei enthält zu viele Einträge', 'Automation completion file contains too many entries'],
|
||||||
|
['Automatik-Abschlussnachweise sind ungültig', 'Automation completion evidence is invalid'],
|
||||||
|
['Abschlussnachweise konnten nicht gespeichert werden.', 'Completion evidence could not be saved.'],
|
||||||
|
['Abschlussnachweise konnten nicht vollständig ermittelt werden.', 'Completion evidence could not be determined completely.'],
|
||||||
|
['Automatische Wiederaufnahme wurde wegen nicht verfügbarer Abschlussnachweise blockiert.', 'Automatic resume was blocked because completion evidence is unavailable.'],
|
||||||
|
['Upload-Log-Verzeichnis enthält zu viele Einträge', 'Upload log directory contains too many entries'],
|
||||||
|
['Zu viele verwaltete Upload-Logs', 'Too many managed upload logs'],
|
||||||
|
['Upload-Logs überschreiten das Leselimit', 'Upload logs exceed the read limit'],
|
||||||
|
['Upload-Log überschreitet das Leselimit', 'Upload log exceeds the read limit'],
|
||||||
|
['Upload-Log enthält zu viele eindeutige Einträge', 'Upload log contains too many unique entries'],
|
||||||
|
['Upload-Log-Stream ist nicht verfügbar', 'Upload log stream is unavailable'],
|
||||||
|
['Upload-Log-Zeile ist zu lang', 'Upload log line is too long'],
|
||||||
['Upload-Status', 'Upload status'],
|
['Upload-Status', 'Upload status'],
|
||||||
['Upload-Übersicht', 'Upload overview'],
|
['Upload-Übersicht', 'Upload overview'],
|
||||||
['Verlauf als CSV exportieren?\n\nOK = CSV\nAbbrechen = JSON', 'Export history as CSV?\n\nOK = CSV\nCancel = JSON'],
|
['Verlauf als CSV exportieren?\n\nOK = CSV\nAbbrechen = JSON', 'Export history as CSV?\n\nOK = CSV\nCancel = JSON'],
|
||||||
|
|||||||
@@ -7,7 +7,12 @@ const {
|
|||||||
rollDailyTelemetry,
|
rollDailyTelemetry,
|
||||||
applyTelemetryDelta,
|
applyTelemetryDelta,
|
||||||
deriveAutomationState,
|
deriveAutomationState,
|
||||||
classifyProcessedCandidates
|
isPathWithinAutomationFolder,
|
||||||
|
classifyProcessedCandidates,
|
||||||
|
classifyAutomationCompletionLedger,
|
||||||
|
createAutomationCompletionWriter,
|
||||||
|
mergeAutomationCompletions,
|
||||||
|
removeAutomationCompletions
|
||||||
} = require('../lib/automation-control');
|
} = require('../lib/automation-control');
|
||||||
|
|
||||||
test('automation defaults use 15000 jobs and a five minute reconciliation interval', () => {
|
test('automation defaults use 15000 jobs and a five minute reconciliation interval', () => {
|
||||||
@@ -284,6 +289,14 @@ test('automation state follows inactive disconnected error queue-limited and act
|
|||||||
assert.equal(deriveAutomationState(null), 'inactive');
|
assert.equal(deriveAutomationState(null), 'inactive');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('watched-folder membership respects Windows casing and recursive scope', () => {
|
||||||
|
assert.equal(isPathWithinAutomationFolder('C:\\Watch\\episode.mkv', 'c:/watch', false), true);
|
||||||
|
assert.equal(isPathWithinAutomationFolder('C:\\Watch\\Season 1\\episode.mkv', 'c:/watch', false), false);
|
||||||
|
assert.equal(isPathWithinAutomationFolder('C:\\Watch\\Season 1\\episode.mkv', 'c:/watch', true), true);
|
||||||
|
assert.equal(isPathWithinAutomationFolder('C:\\Watcher\\episode.mkv', 'c:/watch', true), false);
|
||||||
|
assert.equal(isPathWithinAutomationFolder('', 'c:/watch', true), false);
|
||||||
|
});
|
||||||
|
|
||||||
test('exact queue and history paths mark candidates processed case-insensitively', () => {
|
test('exact queue and history paths mark candidates processed case-insensitively', () => {
|
||||||
const result = classifyProcessedCandidates({
|
const result = classifyProcessedCandidates({
|
||||||
candidates: [
|
candidates: [
|
||||||
@@ -345,3 +358,96 @@ test('processed classification tolerates malformed collections and does not muta
|
|||||||
unprocessedPaths: []
|
unprocessedPaths: []
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('durable completion ledger excludes only unchanged completed hosters', () => {
|
||||||
|
const candidate = {
|
||||||
|
path: 'C:\\Watch\\Episode.mkv',
|
||||||
|
size: 1048576,
|
||||||
|
mtimeMs: 1787828400123.75,
|
||||||
|
eligibleHosters: ['doodstream.com', 'voe.sx', 'byse.sx']
|
||||||
|
};
|
||||||
|
const completionRows = [
|
||||||
|
{ path: 'c:/watch/episode.mkv', size: 1048576, mtimeMs: 1787828400123, hoster: 'DOODSTREAM.COM', completedAt: 10 },
|
||||||
|
{ path: 'C:\\WATCH\\EPISODE.MKV', size: 1048576, mtimeMs: 1787828400123.9, hoster: 'voe.sx', completedAt: 20 }
|
||||||
|
];
|
||||||
|
|
||||||
|
assert.deepEqual(classifyAutomationCompletionLedger({ candidates: [candidate], completionRows }), {
|
||||||
|
processedPaths: [],
|
||||||
|
completedByPath: [{ path: candidate.path, hosters: ['doodstream.com', 'voe.sx'] }],
|
||||||
|
remainingByPath: [{ path: candidate.path, hosters: ['byse.sx'] }]
|
||||||
|
});
|
||||||
|
|
||||||
|
const complete = classifyAutomationCompletionLedger({
|
||||||
|
candidates: [candidate],
|
||||||
|
completionRows: completionRows.concat({
|
||||||
|
path: candidate.path,
|
||||||
|
size: candidate.size,
|
||||||
|
mtimeMs: candidate.mtimeMs,
|
||||||
|
hoster: 'byse.sx',
|
||||||
|
completedAt: 30
|
||||||
|
})
|
||||||
|
});
|
||||||
|
assert.deepEqual(complete.processedPaths, [candidate.path]);
|
||||||
|
assert.deepEqual(complete.remainingByPath, [{ path: candidate.path, hosters: [] }]);
|
||||||
|
|
||||||
|
const changed = classifyAutomationCompletionLedger({
|
||||||
|
candidates: [{ ...candidate, mtimeMs: candidate.mtimeMs + 1 }],
|
||||||
|
completionRows
|
||||||
|
});
|
||||||
|
assert.deepEqual(changed.completedByPath, [{ path: candidate.path, hosters: [] }]);
|
||||||
|
assert.deepEqual(changed.remainingByPath, [{ path: candidate.path, hosters: candidate.eligibleHosters }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completion ledger replaces only the same path and hoster without evicting unrelated entries', () => {
|
||||||
|
const existing = [
|
||||||
|
{ path: 'C:\\watch\\a.mkv', size: 1, mtimeMs: 1, hoster: 'voe.sx', completedAt: 10 },
|
||||||
|
{ path: 'C:\\watch\\b.mkv', size: 2, mtimeMs: 2, hoster: 'voe.sx', completedAt: 20 }
|
||||||
|
];
|
||||||
|
const merged = mergeAutomationCompletions(existing, [
|
||||||
|
{ path: 'c:/WATCH/a.mkv', size: 3, mtimeMs: 3, hoster: 'VOE.SX', completedAt: 30 },
|
||||||
|
{ path: 'C:\\watch\\c.mkv', size: 4, mtimeMs: 4, hoster: 'byse.sx', completedAt: 40 }
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.deepEqual(merged, [
|
||||||
|
{ path: 'C:\\watch\\b.mkv', size: 2, mtimeMs: 2, hoster: 'voe.sx', completedAt: 20 },
|
||||||
|
{ path: 'c:/WATCH/a.mkv', size: 3, mtimeMs: 3, hoster: 'voe.sx', completedAt: 30 },
|
||||||
|
{ path: 'C:\\watch\\c.mkv', size: 4, mtimeMs: 4, hoster: 'byse.sx', completedAt: 40 }
|
||||||
|
]);
|
||||||
|
assert.deepEqual(removeAutomationCompletions(merged, [{ path: 'C:\\WATCH\\A.MKV', hoster: 'voe.sx' }]), [merged[0], merged[2]]);
|
||||||
|
assert.deepEqual(removeAutomationCompletions(merged, [{ path: 'c:/watch/c.mkv' }]), [merged[0], merged[1]]);
|
||||||
|
assert.throws(() => mergeAutomationCompletions(existing, [
|
||||||
|
{ path: 'C:\\watch\\c.mkv', size: 4, mtimeMs: 4, hoster: 'byse.sx', completedAt: 40 }
|
||||||
|
], 2), /zu viele Einträge/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('completion writer coalesces successful jobs and retains a failed batch for retry', async () => {
|
||||||
|
const scheduled = [];
|
||||||
|
const writes = [];
|
||||||
|
const persisted = [];
|
||||||
|
const failures = [];
|
||||||
|
let fail = true;
|
||||||
|
const writer = createAutomationCompletionWriter({
|
||||||
|
schedule: callback => scheduled.push(callback),
|
||||||
|
onPersisted: entries => persisted.push(structuredClone(entries)),
|
||||||
|
onError: (error, entries) => failures.push({ message: error.message, entries: structuredClone(entries) }),
|
||||||
|
save: async entries => {
|
||||||
|
if (fail) throw new Error('disk unavailable');
|
||||||
|
writes.push(structuredClone(entries));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const first = { path: 'C:\\watch\\episode.mkv', size: 1, mtimeMs: 2, hoster: 'voe.sx', completedAt: 3 };
|
||||||
|
const newer = { ...first, completedAt: 4 };
|
||||||
|
|
||||||
|
writer.add(first);
|
||||||
|
writer.add(newer);
|
||||||
|
assert.equal(scheduled.length, 1);
|
||||||
|
await assert.rejects(writer.flush(), /disk unavailable/);
|
||||||
|
assert.deepEqual(persisted, []);
|
||||||
|
assert.deepEqual(failures, [{ message: 'disk unavailable', entries: [newer] }]);
|
||||||
|
|
||||||
|
fail = false;
|
||||||
|
await writer.flush();
|
||||||
|
assert.deepEqual(writes, [[newer]]);
|
||||||
|
assert.deepEqual(persisted, [[newer]]);
|
||||||
|
assert.equal(writer.pendingCount(), 0);
|
||||||
|
});
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ function createStore() {
|
|||||||
store = new ConfigStore(fakeApp);
|
store = new ConfigStore(fakeApp);
|
||||||
store.filePath = path.join(tmpDir, 'electron-config.json');
|
store.filePath = path.join(tmpDir, 'electron-config.json');
|
||||||
store.historyPath = path.join(tmpDir, 'electron-history.json');
|
store.historyPath = path.join(tmpDir, 'electron-history.json');
|
||||||
|
store.automationCompletionPath = path.join(tmpDir, 'automation-completions.json');
|
||||||
return store;
|
return store;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,6 +42,7 @@ function createStoreAt(filePath) {
|
|||||||
});
|
});
|
||||||
configuredStore.filePath = filePath;
|
configuredStore.filePath = filePath;
|
||||||
configuredStore.historyPath = path.join(path.dirname(filePath), 'electron-history.json');
|
configuredStore.historyPath = path.join(path.dirname(filePath), 'electron-history.json');
|
||||||
|
configuredStore.automationCompletionPath = path.join(path.dirname(filePath), 'automation-completions.json');
|
||||||
return configuredStore;
|
return configuredStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,6 +127,74 @@ describe('ConfigStore', () => {
|
|||||||
assert.equal(reloaded.globalSettings.folderMonitor.pausedAt, 1787712000000);
|
assert.equal(reloaded.globalSettings.folderMonitor.pausedAt, 1787712000000);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('automation completions survive queue clearing and can be removed explicitly', async () => {
|
||||||
|
const completion = {
|
||||||
|
path: 'C:\\watch\\episode.mkv',
|
||||||
|
size: 1024,
|
||||||
|
mtimeMs: 1787828400123,
|
||||||
|
hoster: 'doodstream.com',
|
||||||
|
completedAt: 1787828500000
|
||||||
|
};
|
||||||
|
|
||||||
|
await store.saveAutomationCompletions([completion]);
|
||||||
|
await store.savePendingQueue(null);
|
||||||
|
await store.appendHistory({ id: 'old', timestamp: '2026-01-01T00:00:00.000Z', files: [] });
|
||||||
|
await store.clearHistory();
|
||||||
|
|
||||||
|
const reloaded = createStoreAt(store.filePath);
|
||||||
|
assert.deepEqual(await reloaded.loadAutomationCompletions(), [completion]);
|
||||||
|
|
||||||
|
await reloaded.clearAutomationCompletions([{ path: completion.path, hoster: completion.hoster }]);
|
||||||
|
assert.deepEqual(await reloaded.loadAutomationCompletions(), []);
|
||||||
|
assert.equal(JSON.parse(fs.readFileSync(reloaded.automationCompletionPath, 'utf8')).version, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('automation completion writes remain serialized without evicting older unique paths', async () => {
|
||||||
|
const first = store.saveAutomationCompletions([
|
||||||
|
{ path: 'C:\\watch\\old.mkv', size: 1, mtimeMs: 1, hoster: 'voe.sx', completedAt: 1 }
|
||||||
|
], { maxEntries: 2 });
|
||||||
|
const second = store.saveAutomationCompletions([
|
||||||
|
{ path: 'C:\\watch\\middle.mkv', size: 2, mtimeMs: 2, hoster: 'voe.sx', completedAt: 2 },
|
||||||
|
{ path: 'C:\\watch\\new.mkv', size: 3, mtimeMs: 3, hoster: 'byse.sx', completedAt: 3 }
|
||||||
|
], { maxEntries: 2 });
|
||||||
|
|
||||||
|
await Promise.all([first, second]);
|
||||||
|
await store.drainAutomationCompletionWrites();
|
||||||
|
|
||||||
|
assert.deepEqual((await store.loadAutomationCompletions()).map(entry => entry.path), [
|
||||||
|
'C:\\watch\\old.mkv',
|
||||||
|
'C:\\watch\\middle.mkv',
|
||||||
|
'C:\\watch\\new.mkv'
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('corrupted automation completion evidence fails closed', async () => {
|
||||||
|
fs.writeFileSync(store.automationCompletionPath, '{broken', 'utf8');
|
||||||
|
await assert.rejects(store.loadAutomationCompletions());
|
||||||
|
const reloaded = createStoreAt(store.filePath);
|
||||||
|
fs.writeFileSync(reloaded.automationCompletionPath, JSON.stringify({ version: 1, entries: [{ path: 'C:\\watch\\invalid.mkv' }] }), 'utf8');
|
||||||
|
await assert.rejects(reloaded.loadAutomationCompletions(), /ungültig/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('automation completion drain waits for an active durable write', async () => {
|
||||||
|
const originalWrite = store._writeAutomationCompletionFile.bind(store);
|
||||||
|
let releaseWrite;
|
||||||
|
store._writeAutomationCompletionFile = entries => new Promise((resolve, reject) => {
|
||||||
|
releaseWrite = () => originalWrite(entries).then(resolve, reject);
|
||||||
|
});
|
||||||
|
const saving = store.saveAutomationCompletions([
|
||||||
|
{ path: 'C:\\watch\\drain.mkv', size: 1, mtimeMs: 2, hoster: 'voe.sx', completedAt: 3 }
|
||||||
|
]);
|
||||||
|
while (!releaseWrite) await new Promise(resolve => setImmediate(resolve));
|
||||||
|
let drained = false;
|
||||||
|
const draining = store.drainAutomationCompletionWrites().then(() => { drained = true; });
|
||||||
|
await new Promise(resolve => setImmediate(resolve));
|
||||||
|
assert.equal(drained, false);
|
||||||
|
releaseWrite();
|
||||||
|
await Promise.all([saving, draining]);
|
||||||
|
assert.equal(drained, true);
|
||||||
|
});
|
||||||
|
|
||||||
it('drops the retired plaintext credential setting from legacy configurations', () => {
|
it('drops the retired plaintext credential setting from legacy configurations', () => {
|
||||||
fs.writeFileSync(store.filePath, JSON.stringify({
|
fs.writeFileSync(store.filePath, JSON.stringify({
|
||||||
hosters: {},
|
hosters: {},
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ test('inspects duplicates, unavailable files, accepted files, and configured siz
|
|||||||
], {
|
], {
|
||||||
existingPaths: ['C:\\queue\\duplicate.mkv'],
|
existingPaths: ['C:\\queue\\duplicate.mkv'],
|
||||||
inspectPath: async filePath => {
|
inspectPath: async filePath => {
|
||||||
if (filePath.endsWith('accepted.mkv')) return { exists: true, readable: true, size: 2 * 1024 * 1024 };
|
if (filePath.endsWith('accepted.mkv')) return { exists: true, readable: true, size: 2 * 1024 * 1024, mtimeMs: 1787828400123 };
|
||||||
if (filePath.endsWith('empty.mkv')) return { exists: true, readable: true, size: 0 };
|
if (filePath.endsWith('empty.mkv')) return { exists: true, readable: true, size: 0 };
|
||||||
return { exists: false };
|
return { exists: false };
|
||||||
}
|
}
|
||||||
@@ -42,6 +42,7 @@ test('inspects duplicates, unavailable files, accepted files, and configured siz
|
|||||||
jobCount: 1,
|
jobCount: 1,
|
||||||
sizeLimitedJobCount: 1
|
sizeLimitedJobCount: 1
|
||||||
});
|
});
|
||||||
|
assert.equal(inspection.accepted[0].mtimeMs, 1787828400123);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('connects the import preflight through the main process, preload, renderer, and hoster dialog', () => {
|
test('connects the import preflight through the main process, preload, renderer, and hoster dialog', () => {
|
||||||
|
|||||||
@@ -92,8 +92,11 @@ test('managed upload-log discovery includes session logs and excludes unrelated
|
|||||||
assert.equal(typeof isManagedUploadLogFileName, 'function');
|
assert.equal(typeof isManagedUploadLogFileName, 'function');
|
||||||
const options = { baseName: 'fileuploader', ext: '.log' };
|
const options = { baseName: 'fileuploader', ext: '.log' };
|
||||||
assert.equal(isManagedUploadLogFileName('fileuploader.log', options), true);
|
assert.equal(isManagedUploadLogFileName('fileuploader.log', options), true);
|
||||||
|
assert.equal(isManagedUploadLogFileName('fileuploader.1.log', options), true);
|
||||||
assert.equal(isManagedUploadLogFileName('fileuploader-2026-08-27.log', options), true);
|
assert.equal(isManagedUploadLogFileName('fileuploader-2026-08-27.log', options), true);
|
||||||
|
assert.equal(isManagedUploadLogFileName('fileuploader-2026-08-27.2.log', options), true);
|
||||||
assert.equal(isManagedUploadLogFileName('fileuploader-session-2026-08-27_05-40-59-1234.log', options), true);
|
assert.equal(isManagedUploadLogFileName('fileuploader-session-2026-08-27_05-40-59-1234.log', options), true);
|
||||||
|
assert.equal(isManagedUploadLogFileName('27-08-2026-mdu-session-05-40-111111.3.log', options), true);
|
||||||
assert.equal(isManagedUploadLogFileName('27-08-2026-mdu-session-05-40-599797.log', options), true);
|
assert.equal(isManagedUploadLogFileName('27-08-2026-mdu-session-05-40-599797.log', options), true);
|
||||||
assert.equal(isManagedUploadLogFileName('FILEUPLOADER-2026-08-27.LOG', options), true);
|
assert.equal(isManagedUploadLogFileName('FILEUPLOADER-2026-08-27.LOG', options), true);
|
||||||
assert.equal(isManagedUploadLogFileName('27-08-2026-MDU-SESSION-05-40-599797.LOG', options), true);
|
assert.equal(isManagedUploadLogFileName('27-08-2026-MDU-SESSION-05-40-599797.LOG', options), true);
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ test('packages every Electron preload referenced by the main process', () => {
|
|||||||
|
|
||||||
test('read-own-upload-log discovers base daily session and both fallback directories without synchronous reads', async () => {
|
test('read-own-upload-log discovers base daily session and both fallback directories without synchronous reads', async () => {
|
||||||
const mainSource = fs.readFileSync(path.join(projectRoot, 'main.js'), 'utf8');
|
const mainSource = fs.readFileSync(path.join(projectRoot, 'main.js'), 'utf8');
|
||||||
const blockStart = mainSource.indexOf("ipcMain.handle('read-own-upload-log'");
|
const blockStart = mainSource.indexOf('let _uploadLogEvidenceCache');
|
||||||
const blockEnd = mainSource.indexOf("\nipcMain.handle('import-upload-log'", blockStart);
|
const blockEnd = mainSource.indexOf("\nipcMain.handle('import-upload-log'", blockStart);
|
||||||
assert.notEqual(blockStart, -1);
|
assert.notEqual(blockStart, -1);
|
||||||
assert.notEqual(blockEnd, -1);
|
assert.notEqual(blockEnd, -1);
|
||||||
@@ -233,9 +233,9 @@ test('read-own-upload-log discovers base daily session and both fallback directo
|
|||||||
const desktop = 'C:\\desktop';
|
const desktop = 'C:\\desktop';
|
||||||
const userData = 'C:\\user-data';
|
const userData = 'C:\\user-data';
|
||||||
const entriesByDirectory = new Map([
|
const entriesByDirectory = new Map([
|
||||||
[configured, ['custom.txt', 'custom-2026-08-27.txt', '27-08-2026-mdu-session-05-40-111111.txt', 'upload-audit.log']],
|
[configured, ['custom.txt', 'custom-2026-08-27.txt', 'custom.1.txt', '27-08-2026-mdu-session-05-40-111111.txt', 'upload-audit.log']],
|
||||||
[desktop, ['FILEUPLOADER-2026-08-26.LOG', '26-08-2026-MDU-SESSION-05-40-222222.LOG', 'account-rotation.log']],
|
[desktop, ['FILEUPLOADER-2026-08-26.LOG', 'FILEUPLOADER-2026-08-26.2.LOG', '26-08-2026-MDU-SESSION-05-40-222222.LOG', 'account-rotation.log']],
|
||||||
[userData, ['fileuploader-2026-08-25.log', '25-08-2026-mdu-session-05-40-333333.log', 'upload-debug.log']]
|
[userData, ['fileuploader-2026-08-25.log', 'fileuploader.3.log', '25-08-2026-mdu-session-05-40-333333.log', 'upload-debug.log']]
|
||||||
]);
|
]);
|
||||||
const fileNames = new Map();
|
const fileNames = new Map();
|
||||||
for (const [directory, names] of entriesByDirectory) {
|
for (const [directory, names] of entriesByDirectory) {
|
||||||
@@ -244,33 +244,84 @@ test('read-own-upload-log discovers base daily session and both fallback directo
|
|||||||
fileNames.set(path.win32.join(directory, name), `${name}.mkv`);
|
fileNames.set(path.win32.join(directory, name), `${name}.mkv`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let streamReads = 0;
|
||||||
|
let failedPath = '';
|
||||||
|
let scanLabel = '';
|
||||||
|
let holdNextRead = false;
|
||||||
|
let heldRead = null;
|
||||||
const fakeFs = {
|
const fakeFs = {
|
||||||
readdirSync: directory => entriesByDirectory.get(directory) || [],
|
readdirSync: () => { throw new Error('synchronous enumeration forbidden'); },
|
||||||
existsSync: filePath => fileNames.has(filePath),
|
existsSync: () => { throw new Error('synchronous existence check forbidden'); },
|
||||||
readFileSync: () => { throw new Error('synchronous read forbidden'); },
|
readFileSync: () => { throw new Error('synchronous read forbidden'); },
|
||||||
promises: {
|
promises: {
|
||||||
readFile: async filePath => require('../lib/upload-log').formatUploadLogLine(
|
readdir: async () => { throw new Error('materialized directory read forbidden'); },
|
||||||
new Date(2026, 7, 27, 5, 40, 0),
|
opendir: async directory => ({
|
||||||
'voe.sx',
|
async *[Symbol.asyncIterator]() {
|
||||||
'https://voe.sx/e/test',
|
for (const name of entriesByDirectory.get(directory) || []) yield { name };
|
||||||
fileNames.get(filePath)
|
}
|
||||||
)
|
}),
|
||||||
|
readFile: async () => { throw new Error('full-file read forbidden'); }
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
vm.runInNewContext(mainSource.slice(blockStart, blockEnd), {
|
const context = {
|
||||||
_resolveUploadLogTarget: () => ({ path: path.win32.join(configured, '27-08-2026-mdu-session-05-40-111111.txt') }),
|
_activeLogPath: path.win32.join(configured, '27-08-2026-mdu-session-05-40-111111.txt'),
|
||||||
|
_resolveUploadLogTarget: () => { throw new Error('write-target resolution forbidden'); },
|
||||||
app: { getPath: name => name === 'desktop' ? desktop : userData },
|
app: { getPath: name => name === 'desktop' ? desktop : userData },
|
||||||
fs: fakeFs,
|
fs: fakeFs,
|
||||||
getBaseLogFilePath: () => path.win32.join(configured, 'custom.txt'),
|
getBaseLogFilePath: () => path.win32.join(configured, 'custom.txt'),
|
||||||
getSafeDesktopDir: () => desktop,
|
getSafeDesktopDir: () => { throw new Error('synchronous desktop probe forbidden'); },
|
||||||
ipcMain: { handle: (channel, handler) => handlers.set(channel, handler) },
|
ipcMain: { handle: (channel, handler) => handlers.set(channel, handler) },
|
||||||
isManagedUploadLogFileName: require('../lib/log-mode').isManagedUploadLogFileName,
|
isManagedUploadLogFileName: require('../lib/log-mode').isManagedUploadLogFileName,
|
||||||
parseUploadLogLine: require('../lib/upload-log').parseUploadLogLine,
|
iterateUploadLogEntries: async function* (filePath) {
|
||||||
|
streamReads++;
|
||||||
|
const label = scanLabel;
|
||||||
|
if (filePath === failedPath) {
|
||||||
|
const error = new Error('managed log denied');
|
||||||
|
error.code = 'EACCES';
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (holdNextRead) {
|
||||||
|
holdNextRead = false;
|
||||||
|
heldRead = createDeferred();
|
||||||
|
await heldRead.promise;
|
||||||
|
}
|
||||||
|
yield require('../lib/upload-log').parseUploadLogLine(require('../lib/upload-log').formatUploadLogLine(
|
||||||
|
new Date(2026, 7, 27, 5, 40, 0),
|
||||||
|
'voe.sx',
|
||||||
|
'https://voe.sx/e/test',
|
||||||
|
`${fileNames.get(filePath)}${label}`
|
||||||
|
));
|
||||||
|
},
|
||||||
path: path.win32
|
path: path.win32
|
||||||
});
|
};
|
||||||
|
vm.runInNewContext(mainSource.slice(blockStart, blockEnd), context);
|
||||||
|
|
||||||
const entries = await handlers.get('read-own-upload-log')();
|
const handler = handlers.get('read-own-upload-log');
|
||||||
assert.deepEqual([...entries.map(entry => entry.fileName)].sort(), [...fileNames.values()].sort());
|
const [first, concurrent] = await Promise.all([handler(), handler()]);
|
||||||
|
const cached = await handler();
|
||||||
|
const expected = [...fileNames.values()].sort();
|
||||||
|
assert.deepEqual([...first.map(entry => entry.fileName)].sort(), expected);
|
||||||
|
assert.deepEqual([...concurrent.map(entry => entry.fileName)].sort(), expected);
|
||||||
|
assert.deepEqual([...cached.map(entry => entry.fileName)].sort(), expected);
|
||||||
|
assert.equal(streamReads, fileNames.size);
|
||||||
|
vm.runInNewContext('_invalidateUploadLogEvidenceCache()', context);
|
||||||
|
failedPath = [...fileNames.keys()][0];
|
||||||
|
await assert.rejects(handler(), /managed log denied/);
|
||||||
|
failedPath = '';
|
||||||
|
vm.runInNewContext('_invalidateUploadLogEvidenceCache()', context);
|
||||||
|
holdNextRead = true;
|
||||||
|
scanLabel = '.old';
|
||||||
|
const staleScan = handler();
|
||||||
|
while (!heldRead) await new Promise(resolve => setImmediate(resolve));
|
||||||
|
vm.runInNewContext('_invalidateUploadLogEvidenceCache()', context);
|
||||||
|
scanLabel = '.new';
|
||||||
|
const freshScan = handler();
|
||||||
|
heldRead.resolve();
|
||||||
|
const [staleEntries, freshEntries] = await Promise.all([staleScan, freshScan]);
|
||||||
|
const cachedFreshEntries = await handler();
|
||||||
|
assert.equal(staleEntries.some(entry => entry.fileName.endsWith('.old')), true);
|
||||||
|
assert.equal(freshEntries.every(entry => entry.fileName.endsWith('.new')), true);
|
||||||
|
assert.equal(cachedFreshEntries.every(entry => entry.fileName.endsWith('.new')), true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('exposes managed online backup operations through narrow IPC boundaries', () => {
|
test('exposes managed online backup operations through narrow IPC boundaries', () => {
|
||||||
@@ -633,20 +684,47 @@ test('preload exposes account cooldown snapshots and removes their listener duri
|
|||||||
test('exposes persistent automation controls and status through narrow IPC boundaries', () => {
|
test('exposes persistent automation controls and status through narrow IPC boundaries', () => {
|
||||||
const preloadSource = fs.readFileSync(path.join(projectRoot, 'preload.js'), 'utf8');
|
const preloadSource = fs.readFileSync(path.join(projectRoot, 'preload.js'), 'utf8');
|
||||||
const mainSource = fs.readFileSync(path.join(projectRoot, 'main.js'), 'utf8');
|
const mainSource = fs.readFileSync(path.join(projectRoot, 'main.js'), 'utf8');
|
||||||
|
const rendererSource = fs.readFileSync(path.join(projectRoot, 'renderer', 'app.js'), 'utf8');
|
||||||
|
|
||||||
assert.match(mainSource, /ipcMain\.handle\('automation:get-status'/u);
|
assert.match(mainSource, /ipcMain\.handle\('automation:get-status'/u);
|
||||||
|
assert.match(mainSource, /ipcMain\.handle\('automation:get-completions'/u);
|
||||||
|
assert.match(mainSource, /ipcMain\.handle\('automation:record-completions'/u);
|
||||||
assert.match(mainSource, /ipcMain\.handle\('automation:pause-after-active'/u);
|
assert.match(mainSource, /ipcMain\.handle\('automation:pause-after-active'/u);
|
||||||
assert.match(mainSource, /ipcMain\.handle\('automation:resume'/u);
|
assert.match(mainSource, /ipcMain\.handle\('automation:resume'/u);
|
||||||
assert.match(mainSource, /ipcMain\.handle\('folder-monitor:test-scan'/u);
|
assert.match(mainSource, /ipcMain\.handle\('folder-monitor:test-scan'/u);
|
||||||
assert.match(mainSource, /ipcMain\.handle\('folder-monitor:reconcile'/u);
|
assert.match(mainSource, /ipcMain\.handle\('folder-monitor:reconcile'/u);
|
||||||
assert.match(mainSource, /safeSend\('automation:status'/u);
|
assert.match(mainSource, /safeSend\('automation:status'/u);
|
||||||
assert.match(preloadSource, /automationGetStatus:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:get-status'\)/u);
|
assert.match(preloadSource, /automationGetStatus:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:get-status'\)/u);
|
||||||
|
assert.match(preloadSource, /getAutomationCompletions:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:get-completions'\)/u);
|
||||||
|
assert.match(preloadSource, /recordAutomationCompletions:\s*\(entries\)\s*=>\s*ipcRenderer\.invoke\('automation:record-completions',\s*entries\)/u);
|
||||||
assert.match(preloadSource, /automationPauseAfterActive:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:pause-after-active'\)/u);
|
assert.match(preloadSource, /automationPauseAfterActive:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:pause-after-active'\)/u);
|
||||||
assert.match(preloadSource, /automationResume:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:resume'\)/u);
|
assert.match(preloadSource, /automationResume:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:resume'\)/u);
|
||||||
assert.match(preloadSource, /folderMonitorTestScan:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('folder-monitor:test-scan'\)/u);
|
assert.match(preloadSource, /folderMonitorTestScan:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('folder-monitor:test-scan'\)/u);
|
||||||
assert.match(preloadSource, /folderMonitorReconcile:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('folder-monitor:reconcile'\)/u);
|
assert.match(preloadSource, /folderMonitorReconcile:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('folder-monitor:reconcile'\)/u);
|
||||||
assert.match(preloadSource, /onAutomationStatus:\s*\(callback\)\s*=>\s*\{[\s\S]*?ipcRenderer\.on\('automation:status'/u);
|
assert.match(preloadSource, /onAutomationStatus:\s*\(callback\)\s*=>\s*\{[\s\S]*?ipcRenderer\.on\('automation:status'/u);
|
||||||
assert.match(preloadSource, /ipcRenderer\.removeAllListeners\('automation:status'\)/u);
|
assert.match(preloadSource, /ipcRenderer\.removeAllListeners\('automation:status'\)/u);
|
||||||
|
assert.match(mainSource, /async function registerAutomationCompletionJobs/u);
|
||||||
|
assert.match(mainSource, /await fs\.promises\.stat\(job\.file\)/u);
|
||||||
|
assert.match(mainSource, /await registerAutomationCompletionJobs\(_thisManager,\s*jobs\)/u);
|
||||||
|
assert.match(mainSource, /await registerAutomationCompletionJobs\(batchManager,\s*jobs\)/u);
|
||||||
|
assert.match(mainSource, /_automationCompletionProgress\.set\(automationCompletionKey\(entry\),\s*data\)/u);
|
||||||
|
assert.match(mainSource, /_automationCompletionWriter\.add\(entry\)/u);
|
||||||
|
assert.match(mainSource, /await _thisManager\._automationCompletionWriter\?\.flush\(\)/u);
|
||||||
|
assert.match(mainSource, /requestUploadFinalization\(summary,\s*!automationCompletionsPersisted\)/u);
|
||||||
|
assert.match(rendererSource, /data\.preserveQueue\s*===\s*true\s*\|\|\s*queueJobs\.some/u);
|
||||||
|
assert.match(rendererSource, /await window\.api\.recordAutomationCompletions\(completionRows\)/u);
|
||||||
|
const serializerStart = rendererSource.indexOf('function serializeUploadJob');
|
||||||
|
const serializerEnd = rendererSource.indexOf('\n}', serializerStart);
|
||||||
|
const serializer = rendererSource.slice(serializerStart, serializerEnd);
|
||||||
|
assert.match(serializer, /automationAdmission:\s*job\.automationAdmission\s*===\s*true/u);
|
||||||
|
assert.match(serializer, /automationMtimeMs:\s*job\.automationMtimeMs/u);
|
||||||
|
assert.match(serializer, /automationSize:\s*job\.automationSize/u);
|
||||||
|
assert.match(serializer, /sourceMtimeMs:\s*job\.sourceMtimeMs/u);
|
||||||
|
assert.match(serializer, /sourceSize:\s*job\.sourceSize/u);
|
||||||
|
const syncStart = rendererSource.indexOf('function syncSelectedFilesFromQueue');
|
||||||
|
const syncEnd = rendererSource.indexOf('\n}', syncStart);
|
||||||
|
const syncSelected = rendererSource.slice(syncStart, syncEnd);
|
||||||
|
assert.match(syncSelected, /mtimeMs:\s*job\.sourceMtimeMs\s*\?\?\s*job\.automationMtimeMs/u);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('every batch start and extension IPC fails closed before account and cleanup side effects', () => {
|
test('every batch start and extension IPC fails closed before account and cleanup side effects', () => {
|
||||||
|
|||||||
@@ -69,6 +69,8 @@ let pendingAutomationTestScan = null;
|
|||||||
let automationProbe = {
|
let automationProbe = {
|
||||||
history: [],
|
history: [],
|
||||||
uploadLog: [],
|
uploadLog: [],
|
||||||
|
completionRows: [],
|
||||||
|
completionError: '',
|
||||||
paused: false,
|
paused: false,
|
||||||
runtimeStatus: {},
|
runtimeStatus: {},
|
||||||
automationStatusSequence: [],
|
automationStatusSequence: [],
|
||||||
@@ -85,7 +87,7 @@ let automationProbe = {
|
|||||||
activeInspections: 0,
|
activeInspections: 0,
|
||||||
maxConcurrentInspections: 0,
|
maxConcurrentInspections: 0,
|
||||||
dryScan: { files: [], reachable: true, trigger: 'test' },
|
dryScan: { files: [], reachable: true, trigger: 'test' },
|
||||||
readCalls: { history: 0, uploadLog: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 },
|
readCalls: { history: 0, uploadLog: 0, completions: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 },
|
||||||
mutationCalls: [],
|
mutationCalls: [],
|
||||||
logs: [],
|
logs: [],
|
||||||
savedSettings: []
|
savedSettings: []
|
||||||
@@ -175,6 +177,8 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
automationProbe = {
|
automationProbe = {
|
||||||
history: Array.isArray(value.history) ? value.history : [],
|
history: Array.isArray(value.history) ? value.history : [],
|
||||||
uploadLog: Array.isArray(value.uploadLog) ? value.uploadLog : [],
|
uploadLog: Array.isArray(value.uploadLog) ? value.uploadLog : [],
|
||||||
|
completionRows: Array.isArray(value.completionRows) ? value.completionRows : [],
|
||||||
|
completionError: String(value.completionError || ''),
|
||||||
paused: value.paused === true,
|
paused: value.paused === true,
|
||||||
runtimeStatus: value.runtimeStatus && typeof value.runtimeStatus === 'object' ? { ...value.runtimeStatus } : {},
|
runtimeStatus: value.runtimeStatus && typeof value.runtimeStatus === 'object' ? { ...value.runtimeStatus } : {},
|
||||||
automationStatusSequence: Array.isArray(value.automationStatusSequence) ? value.automationStatusSequence.map(entry => ({ ...entry })) : [],
|
automationStatusSequence: Array.isArray(value.automationStatusSequence) ? value.automationStatusSequence.map(entry => ({ ...entry })) : [],
|
||||||
@@ -191,7 +195,7 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
activeInspections: 0,
|
activeInspections: 0,
|
||||||
maxConcurrentInspections: 0,
|
maxConcurrentInspections: 0,
|
||||||
dryScan: value.dryScan || { files: [], reachable: true, trigger: 'test' },
|
dryScan: value.dryScan || { files: [], reachable: true, trigger: 'test' },
|
||||||
readCalls: { history: 0, uploadLog: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 },
|
readCalls: { history: 0, uploadLog: 0, completions: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 },
|
||||||
mutationCalls: [],
|
mutationCalls: [],
|
||||||
logs: [],
|
logs: [],
|
||||||
savedSettings: []
|
savedSettings: []
|
||||||
@@ -200,6 +204,7 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
setAutomationEvidence(value = {}) {
|
setAutomationEvidence(value = {}) {
|
||||||
if (Array.isArray(value.history)) automationProbe.history = value.history;
|
if (Array.isArray(value.history)) automationProbe.history = value.history;
|
||||||
if (Array.isArray(value.uploadLog)) automationProbe.uploadLog = value.uploadLog;
|
if (Array.isArray(value.uploadLog)) automationProbe.uploadLog = value.uploadLog;
|
||||||
|
if (Array.isArray(value.completionRows)) automationProbe.completionRows = value.completionRows;
|
||||||
},
|
},
|
||||||
getAutomationProbeState() {
|
getAutomationProbeState() {
|
||||||
return {
|
return {
|
||||||
@@ -251,6 +256,15 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
automationProbe.readCalls.uploadLog++;
|
automationProbe.readCalls.uploadLog++;
|
||||||
return Promise.resolve(automationProbe.uploadLog);
|
return Promise.resolve(automationProbe.uploadLog);
|
||||||
},
|
},
|
||||||
|
getAutomationCompletions() {
|
||||||
|
automationProbe.readCalls.completions++;
|
||||||
|
if (automationProbe.completionError) return Promise.reject(new Error(automationProbe.completionError));
|
||||||
|
return Promise.resolve(automationProbe.completionRows);
|
||||||
|
},
|
||||||
|
clearAutomationCompletions(removals) {
|
||||||
|
automationProbe.mutationCalls.push(['clear-completions', JSON.parse(JSON.stringify(removals || []))]);
|
||||||
|
return Promise.resolve(true);
|
||||||
|
},
|
||||||
automationGetStatus() {
|
automationGetStatus() {
|
||||||
automationProbe.readCalls.status++;
|
automationProbe.readCalls.status++;
|
||||||
if (automationProbe.automationStatusSequence.length > 0) return Promise.resolve(automationProbe.automationStatusSequence.shift());
|
if (automationProbe.automationStatusSequence.length > 0) return Promise.resolve(automationProbe.automationStatusSequence.shift());
|
||||||
@@ -686,7 +700,7 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
resultingJobs: historyEvaluation.summary.resultingJobs
|
resultingJobs: historyEvaluation.summary.resultingJobs
|
||||||
};
|
};
|
||||||
_completedUploadKeys.clear();
|
_completedUploadKeys.clear();
|
||||||
const completedFile = { path: 'C:\\history\\completed-in-session.mkv', name: 'completed-in-session.mkv', size: 1 };
|
const completedFile = { path: 'C:\\history\\completed-in-session.mkv', name: 'completed-in-session.mkv', size: 1, mtimeMs: 1787828400123 };
|
||||||
config.globalSettings.removeFromQueueOnDone = true;
|
config.globalSettings.removeFromQueueOnDone = true;
|
||||||
config.globalSettings.folderMonitor = {
|
config.globalSettings.folderMonitor = {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
@@ -703,6 +717,7 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
hoster: 'doodstream.com',
|
hoster: 'doodstream.com',
|
||||||
status: 'queued',
|
status: 'queued',
|
||||||
bytesTotal: 1,
|
bytesTotal: 1,
|
||||||
|
automationMtimeMs: completedFile.mtimeMs,
|
||||||
automationAdmission: true
|
automationAdmission: true
|
||||||
};
|
};
|
||||||
queueJobs = [completedJob];
|
queueJobs = [completedJob];
|
||||||
@@ -721,18 +736,127 @@ contextBridge.exposeInMainWorld('api', {
|
|||||||
result: { download_url: 'https://doodstream.com/d/completed-in-session' }
|
result: { download_url: 'https://doodstream.com/d/completed-in-session' }
|
||||||
});
|
});
|
||||||
_doneRemovalCoalescer?.drainSync();
|
_doneRemovalCoalescer?.drainSync();
|
||||||
|
window.api.setAutomationEvidence({
|
||||||
|
completionRows: [{
|
||||||
|
path: completedFile.path,
|
||||||
|
size: completedFile.size,
|
||||||
|
mtimeMs: completedFile.mtimeMs,
|
||||||
|
hoster: completedJob.hoster,
|
||||||
|
completedAt: 1787828500000
|
||||||
|
}]
|
||||||
|
});
|
||||||
automationEvidenceSnapshotGeneration++;
|
automationEvidenceSnapshotGeneration++;
|
||||||
automationEvidenceSnapshotCache = null;
|
automationEvidenceSnapshotCache = null;
|
||||||
const removedAfterDone = !queueJobs.some(job => job.id === completedJob.id);
|
const removedAfterDone = !queueJobs.some(job => job.id === completedJob.id);
|
||||||
const completedKeyPresent = _completedUploadKeys.has(completedFile.path + '|doodstream.com');
|
const completedKeyPresent = _completedUploadKeys.has(completedFile.path + '|doodstream.com');
|
||||||
const completedResult = await handleFolderMonitorFiles([completedFile]);
|
const completedResult = await handleFolderMonitorFiles([completedFile]);
|
||||||
const completedProbe = await window.api.getAutomationProbeState();
|
const completedProbe = await window.api.getAutomationProbeState();
|
||||||
|
_completedUploadKeys.clear();
|
||||||
|
queueJobs = [];
|
||||||
|
rebuildJobIndex();
|
||||||
|
window.api.configureAutomationProbe({
|
||||||
|
paused: false,
|
||||||
|
history: [],
|
||||||
|
uploadLog: [],
|
||||||
|
completionRows: [{
|
||||||
|
path: completedFile.path,
|
||||||
|
size: completedFile.size,
|
||||||
|
mtimeMs: completedFile.mtimeMs,
|
||||||
|
hoster: 'doodstream.com',
|
||||||
|
completedAt: 1787828500000
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
automationEvidenceSnapshotGeneration++;
|
||||||
|
automationEvidenceSnapshotCache = null;
|
||||||
|
const durableEvaluation = await evaluateAutomationCandidates([completedFile], { dryRun: true, trigger: 'startup' });
|
||||||
|
_completedUploadKeys.add(completedFile.path + '|doodstream.com');
|
||||||
|
automationEvidenceSnapshotGeneration++;
|
||||||
|
automationEvidenceSnapshotCache = null;
|
||||||
|
const changedEvaluation = await evaluateAutomationCandidates([{ ...completedFile, mtimeMs: completedFile.mtimeMs + 1 }], { dryRun: true, trigger: 'startup' });
|
||||||
|
const partialFile = { path: 'C:\\history\\partial-in-session.mkv', name: 'partial-in-session.mkv', size: 2, mtimeMs: 1787828400456 };
|
||||||
|
config.globalSettings.folderMonitor.hosters = ['doodstream.com', 'voe.sx'];
|
||||||
|
window.api.configureAutomationProbe({
|
||||||
|
paused: false,
|
||||||
|
history: [],
|
||||||
|
uploadLog: [{ fileName: partialFile.name, hoster: 'doodstream.com' }],
|
||||||
|
completionRows: [{
|
||||||
|
path: partialFile.path,
|
||||||
|
size: partialFile.size,
|
||||||
|
mtimeMs: partialFile.mtimeMs,
|
||||||
|
hoster: 'doodstream.com',
|
||||||
|
completedAt: 1787828500001
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
automationEvidenceSnapshotGeneration++;
|
||||||
|
automationEvidenceSnapshotCache = null;
|
||||||
|
const partialEvaluation = await evaluateAutomationCandidates([partialFile], { dryRun: true, trigger: 'startup' });
|
||||||
|
const restoredFile = { path: 'C:\\history\\restored-after-ledger.mkv', name: 'restored-after-ledger.mkv', size: 3, mtimeMs: 1787828400789 };
|
||||||
|
const restoredJob = {
|
||||||
|
id: 'restored-after-ledger',
|
||||||
|
file: restoredFile.path,
|
||||||
|
fileName: restoredFile.name,
|
||||||
|
hoster: 'doodstream.com',
|
||||||
|
status: 'preview',
|
||||||
|
bytesTotal: restoredFile.size,
|
||||||
|
sourceSize: restoredFile.size,
|
||||||
|
sourceMtimeMs: restoredFile.mtimeMs,
|
||||||
|
automationAdmission: true
|
||||||
|
};
|
||||||
|
queueJobs = [restoredJob];
|
||||||
|
selectedFiles = [];
|
||||||
|
rebuildJobIndex();
|
||||||
|
_completedUploadKeys.clear();
|
||||||
|
window.api.configureAutomationProbe({
|
||||||
|
paused: false,
|
||||||
|
history: [],
|
||||||
|
uploadLog: [],
|
||||||
|
completionRows: [{
|
||||||
|
path: restoredFile.path,
|
||||||
|
size: restoredFile.size,
|
||||||
|
mtimeMs: restoredFile.mtimeMs,
|
||||||
|
hoster: restoredJob.hoster,
|
||||||
|
completedAt: 1787828500002
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
await _autoDeduplicateFromLog();
|
||||||
|
queueJobs = [{
|
||||||
|
id: 'blocked-restored-evidence',
|
||||||
|
file: 'C:\\history\\blocked-restored-evidence.mkv',
|
||||||
|
fileName: 'blocked-restored-evidence.mkv',
|
||||||
|
hoster: 'doodstream.com',
|
||||||
|
status: 'preview',
|
||||||
|
bytesTotal: 4
|
||||||
|
}];
|
||||||
|
selectedFiles = [];
|
||||||
|
rebuildJobIndex();
|
||||||
|
config.globalSettings.autoStartRestoredQueue = true;
|
||||||
|
_startupAutoResumeController = null;
|
||||||
|
window.api.configureAutomationProbe({ paused: false, completionError: 'ledger unavailable' });
|
||||||
|
const failedEvidenceResult = await _autoDeduplicateFromLog();
|
||||||
|
scheduleRestoredQueueAutoStart();
|
||||||
|
const failedEvidence = {
|
||||||
|
result: failedEvidenceResult,
|
||||||
|
available: typeof _startupQueueEvidenceAvailable === 'undefined' ? null : _startupQueueEvidenceAvailable,
|
||||||
|
controllerCreated: _startupAutoResumeController !== null
|
||||||
|
};
|
||||||
|
cancelStartupQueueAutoStart();
|
||||||
|
config.globalSettings.autoStartRestoredQueue = false;
|
||||||
const completedEvidence = {
|
const completedEvidence = {
|
||||||
removedAfterDone,
|
removedAfterDone,
|
||||||
completedKeyPresent,
|
completedKeyPresent,
|
||||||
admittedFiles: completedResult.admittedFiles.length,
|
admittedFiles: completedResult.admittedFiles.length,
|
||||||
matchingQueueJobs: queueJobs.filter(job => normalizeAutomationPath(job.file) === normalizeAutomationPath(completedFile.path)).length,
|
matchingQueueJobs: queueJobs.filter(job => normalizeAutomationPath(job.file) === normalizeAutomationPath(completedFile.path)).length,
|
||||||
startOrInjectCalls: completedProbe.mutationCalls.filter(call => call[0] === 'start' || call[0] === 'inject').length
|
startOrInjectCalls: completedProbe.mutationCalls.filter(call => call[0] === 'start' || call[0] === 'inject').length,
|
||||||
|
durableAlreadyProcessed: durableEvaluation.summary.alreadyProcessed,
|
||||||
|
durableResultingJobs: durableEvaluation.summary.resultingJobs,
|
||||||
|
changedAlreadyProcessed: changedEvaluation.summary.alreadyProcessed,
|
||||||
|
changedResultingJobs: changedEvaluation.summary.resultingJobs,
|
||||||
|
partialAlreadyProcessed: partialEvaluation.summary.alreadyProcessed,
|
||||||
|
partialResultingJobs: partialEvaluation.summary.resultingJobs,
|
||||||
|
partialHosters: partialEvaluation.candidates[0]?.eligibleHosters || [],
|
||||||
|
restoredQueueRemoved: !queueJobs.some(job => job.id === restoredJob.id),
|
||||||
|
restoredCompletionKey: _completedUploadKeys.has(restoredJob.file + '|' + restoredJob.hoster),
|
||||||
|
failedEvidence
|
||||||
};
|
};
|
||||||
_completedUploadKeys.clear();
|
_completedUploadKeys.clear();
|
||||||
config.globalSettings.removeFromQueueOnDone = false;
|
config.globalSettings.removeFromQueueOnDone = false;
|
||||||
@@ -2827,7 +2951,7 @@ app.whenReady().then(async () => {
|
|||||||
deferredFiles: 70
|
deferredFiles: 70
|
||||||
},
|
},
|
||||||
frozen: true,
|
frozen: true,
|
||||||
reads: { history: 1, uploadLog: 1, inspect: 1, status: 0, testScan: 0, reconcile: 0 }
|
reads: { history: 1, uploadLog: 1, completions: 1, inspect: 1, status: 0, testScan: 0, reconcile: 0 }
|
||||||
});
|
});
|
||||||
assert.deepEqual(result.automationPipeline.manualTest, {
|
assert.deepEqual(result.automationPipeline.manualTest, {
|
||||||
fingerprintEqual: true,
|
fingerprintEqual: true,
|
||||||
@@ -2843,7 +2967,7 @@ app.whenReady().then(async () => {
|
|||||||
availableSlots: 1200,
|
availableSlots: 1200,
|
||||||
deferredFiles: 0
|
deferredFiles: 0
|
||||||
},
|
},
|
||||||
reads: { history: 1, uploadLog: 1, inspect: 1, status: 0, testScan: 1, reconcile: 0 }
|
reads: { history: 1, uploadLog: 1, completions: 1, inspect: 1, status: 0, testScan: 1, reconcile: 0 }
|
||||||
});
|
});
|
||||||
assert.deepEqual(result.automationPipeline.historyEvidence, {
|
assert.deepEqual(result.automationPipeline.historyEvidence, {
|
||||||
alreadyProcessed: 2,
|
alreadyProcessed: 2,
|
||||||
@@ -2855,7 +2979,17 @@ app.whenReady().then(async () => {
|
|||||||
completedKeyPresent: true,
|
completedKeyPresent: true,
|
||||||
admittedFiles: 0,
|
admittedFiles: 0,
|
||||||
matchingQueueJobs: 0,
|
matchingQueueJobs: 0,
|
||||||
startOrInjectCalls: 0
|
startOrInjectCalls: 0,
|
||||||
|
durableAlreadyProcessed: 1,
|
||||||
|
durableResultingJobs: 0,
|
||||||
|
changedAlreadyProcessed: 0,
|
||||||
|
changedResultingJobs: 1,
|
||||||
|
partialAlreadyProcessed: 0,
|
||||||
|
partialResultingJobs: 1,
|
||||||
|
partialHosters: ['voe.sx'],
|
||||||
|
restoredQueueRemoved: true,
|
||||||
|
restoredCompletionKey: true,
|
||||||
|
failedEvidence: { result: false, available: false, controllerCreated: false }
|
||||||
});
|
});
|
||||||
assert.deepEqual(result.automationPipeline.pendingDedup, {
|
assert.deepEqual(result.automationPipeline.pendingDedup, {
|
||||||
evaluatedNames: ['new.mkv'],
|
evaluatedNames: ['new.mkv'],
|
||||||
|
|||||||
@@ -105,6 +105,12 @@ test('classifyErrorCategory: aborted is its own bucket (not retryable)', () => {
|
|||||||
assert.strictEqual(isRetryableCategory('aborted'), false);
|
assert.strictEqual(isRetryableCategory('aborted'), false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('automation completion persistence failures are never retried as uploads', () => {
|
||||||
|
const category = classifyErrorCategory('Automatik-Abschlussnachweis konnte nicht gespeichert werden');
|
||||||
|
assert.strictEqual(category, 'local-persistence');
|
||||||
|
assert.strictEqual(isRetryableCategory(category), false);
|
||||||
|
});
|
||||||
|
|
||||||
test('classifyErrorCategory: unknown for everything else', () => {
|
test('classifyErrorCategory: unknown for everything else', () => {
|
||||||
assert.strictEqual(classifyErrorCategory(''), 'unknown');
|
assert.strictEqual(classifyErrorCategory(''), 'unknown');
|
||||||
assert.strictEqual(classifyErrorCategory(null), 'unknown');
|
assert.strictEqual(classifyErrorCategory(null), 'unknown');
|
||||||
@@ -176,4 +182,5 @@ test('isRetryableCategory: only transient + network + unknown retry-worthy', ()
|
|||||||
assert.strictEqual(isRetryableCategory('file-rejected'), false);
|
assert.strictEqual(isRetryableCategory('file-rejected'), false);
|
||||||
assert.strictEqual(isRetryableCategory('account-error'), false);
|
assert.strictEqual(isRetryableCategory('account-error'), false);
|
||||||
assert.strictEqual(isRetryableCategory('aborted'), false);
|
assert.strictEqual(isRetryableCategory('aborted'), false);
|
||||||
|
assert.strictEqual(isRetryableCategory('local-persistence'), false);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ const assert = require('node:assert');
|
|||||||
const {
|
const {
|
||||||
formatUploadLogLine,
|
formatUploadLogLine,
|
||||||
parseUploadLogLine,
|
parseUploadLogLine,
|
||||||
|
iterateUploadLogEntries,
|
||||||
|
readUploadLogEntries,
|
||||||
summarizeBatchPlan,
|
summarizeBatchPlan,
|
||||||
formatUploadPlanLogLine
|
formatUploadPlanLogLine
|
||||||
} = require('../lib/upload-log');
|
} = require('../lib/upload-log');
|
||||||
@@ -102,6 +104,11 @@ test('parseUploadLogLine skips comments, blanks and malformed lines', () => {
|
|||||||
assert.equal(parseUploadLogLine(42), null);
|
assert.equal(parseUploadLogLine(42), null);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('parser distinguishes confirmed uploads from filename-only rows', () => {
|
||||||
|
assert.equal(parseUploadLogLine('2026-08-27 05:40:00|voe.sx|||episode.mkv|').confirmed, false);
|
||||||
|
assert.equal(parseUploadLogLine('2026-08-27 05:40:00|voe.sx|https://voe.sx/e/code||episode.mkv|').confirmed, true);
|
||||||
|
});
|
||||||
|
|
||||||
test('parseUploadLogLine: missing/garbage timestamp yields ts=undefined (legacy lines still match by name)', () => {
|
test('parseUploadLogLine: missing/garbage timestamp yields ts=undefined (legacy lines still match by name)', () => {
|
||||||
const parsed = parseUploadLogLine('|voe.sx|link||a.mkv|');
|
const parsed = parseUploadLogLine('|voe.sx|link||a.mkv|');
|
||||||
assert.equal(parsed.hoster, 'voe.sx');
|
assert.equal(parsed.hoster, 'voe.sx');
|
||||||
@@ -136,3 +143,78 @@ test('SEAM: a leading-space filename round-trips and the gate still drops its gh
|
|||||||
const { removed } = partitionRestoredJobsByLog([job], [parsed], savedAt);
|
const { removed } = partitionRestoredJobsByLog([job], [parsed], savedAt);
|
||||||
assert.equal(removed.length, 1, 'leading-space filename now matches end-to-end (was a mismatch before)');
|
assert.equal(removed.length, 1, 'leading-space filename now matches end-to-end (was a mismatch before)');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('stream reader parses large logs incrementally and yields between bounded line batches', async () => {
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const os = require('node:os');
|
||||||
|
const path = require('node:path');
|
||||||
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'upload-log-stream-'));
|
||||||
|
const filePath = path.join(directory, 'fileuploader.log');
|
||||||
|
const lines = Array.from({ length: 2505 }, (_, index) => formatUploadLogLine(
|
||||||
|
new Date(2026, 7, 27, 5, 40, index % 60),
|
||||||
|
index % 2 === 0 ? 'voe.sx' : 'doodstream.com',
|
||||||
|
`https://example.invalid/${index}`,
|
||||||
|
`episode-${index}.mkv`
|
||||||
|
)).join('');
|
||||||
|
fs.writeFileSync(filePath, lines, 'utf8');
|
||||||
|
let yields = 0;
|
||||||
|
try {
|
||||||
|
const entries = await readUploadLogEntries(filePath, {
|
||||||
|
yieldEvery: 500,
|
||||||
|
yieldFn: async () => { yields++; }
|
||||||
|
});
|
||||||
|
assert.equal(entries.length, 2505);
|
||||||
|
assert.equal(entries[0].fileName, 'episode-0.mkv');
|
||||||
|
assert.equal(entries.at(-1).fileName, 'episode-2504.mkv');
|
||||||
|
assert.equal(yields, 5);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('upload-log iterator is lazy and rejects oversized lines', async () => {
|
||||||
|
let produced = 0;
|
||||||
|
async function* source() {
|
||||||
|
produced++;
|
||||||
|
yield formatUploadLogLine(new Date(2026, 7, 27, 5, 40, 0), 'voe.sx', 'https://example.invalid/1', 'one.mkv').trimEnd();
|
||||||
|
produced++;
|
||||||
|
yield formatUploadLogLine(new Date(2026, 7, 27, 5, 40, 1), 'voe.sx', 'https://example.invalid/2', 'two.mkv').trimEnd();
|
||||||
|
}
|
||||||
|
const iterator = iterateUploadLogEntries('', { lines: source(), maxLineLength: 65536 });
|
||||||
|
assert.deepEqual(await iterator.next(), {
|
||||||
|
done: false,
|
||||||
|
value: { hoster: 'voe.sx', fileName: 'one.mkv', ts: new Date(2026, 7, 27, 5, 40, 0).getTime(), confirmed: true }
|
||||||
|
});
|
||||||
|
assert.equal(produced, 1);
|
||||||
|
await iterator.return();
|
||||||
|
|
||||||
|
const oversized = iterateUploadLogEntries('', {
|
||||||
|
lines: (async function* () { yield 'x'.repeat(11); })(),
|
||||||
|
maxLineLength: 10
|
||||||
|
});
|
||||||
|
await assert.rejects(async () => { for await (const entry of oversized) void entry; }, /Zeile ist zu lang/);
|
||||||
|
|
||||||
|
let destroyed = 0;
|
||||||
|
const input = {
|
||||||
|
async *[Symbol.asyncIterator]() { yield 'x'.repeat(11); },
|
||||||
|
destroy: () => { destroyed++; }
|
||||||
|
};
|
||||||
|
const leaking = iterateUploadLogEntries('ignored.log', {
|
||||||
|
fs: { createReadStream: () => input },
|
||||||
|
maxLineLength: 10
|
||||||
|
});
|
||||||
|
await assert.rejects(async () => { for await (const entry of leaking) void entry; }, /Zeile ist zu lang/);
|
||||||
|
assert.equal(destroyed, 1);
|
||||||
|
|
||||||
|
const oversizedStream = iterateUploadLogEntries('ignored.log', {
|
||||||
|
fs: {
|
||||||
|
createReadStream: () => ({
|
||||||
|
async *[Symbol.asyncIterator]() { yield 'x'.repeat(11); },
|
||||||
|
destroy() {}
|
||||||
|
})
|
||||||
|
},
|
||||||
|
maxBytes: 10,
|
||||||
|
maxLineLength: 100
|
||||||
|
});
|
||||||
|
await assert.rejects(async () => { for await (const entry of oversizedStream) void entry; }, /Leselimit/);
|
||||||
|
});
|
||||||
|
|||||||
@@ -151,8 +151,8 @@ describe('UploadManager', () => {
|
|||||||
mgr.on('batch-done', (s) => { summary = s; });
|
mgr.on('batch-done', (s) => { summary = s; });
|
||||||
|
|
||||||
await mgr.startBatch([
|
await mgr.startBatch([
|
||||||
{ file: '/test/video1.mp4', hoster: 'doodstream.com', apiKey: 'key1' },
|
{ jobId: 'summary-1', file: '/test/video1.mp4', hoster: 'doodstream.com', apiKey: 'key1' },
|
||||||
{ file: '/test/video2.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
|
{ jobId: 'summary-2', file: '/test/video2.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
|
||||||
]);
|
]);
|
||||||
|
|
||||||
assert.ok(summary);
|
assert.ok(summary);
|
||||||
@@ -160,6 +160,7 @@ describe('UploadManager', () => {
|
|||||||
assert.equal(summary.succeeded, 2);
|
assert.equal(summary.succeeded, 2);
|
||||||
assert.equal(summary.failed, 0);
|
assert.equal(summary.failed, 0);
|
||||||
assert.equal(summary.files.length, 2);
|
assert.equal(summary.files.length, 2);
|
||||||
|
assert.deepEqual(summary.files.map(file => file.results[0].jobId).sort(), ['summary-1', 'summary-2']);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('emits a final idle stats snapshot after a normal batch', async () => {
|
it('emits a final idle stats snapshot after a normal batch', async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user