diff --git a/.gitignore b/.gitignore index 346bf1f..800ebc9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ electron-config.json.pre-history-split.bak electron-config.pre-import-*.json electron-history.json electron-history.json.tmp +automation-completions.json +automation-completions.json.tmp *.log debug.log fileuploader.log diff --git a/lib/automation-control.js b/lib/automation-control.js index 996372e..65642e6 100644 --- a/lib/automation-control.js +++ b/lib/automation-control.js @@ -147,10 +147,143 @@ return String(value || '').replace(/\\/g, '/').toLowerCase(); } + function isPathWithinAutomationFolder(filePath, folderPath, recursive = true) { + const file = normalizePath(filePath).replace(/\/+$/, ''); + const folder = normalizePath(folderPath).replace(/\/+$/, ''); + if (!file || !folder || !file.startsWith(`${folder}/`)) return false; + const relative = file.slice(folder.length + 1); + return Boolean(relative) && (recursive === true || !relative.includes('/')); + } + function baseName(value) { return String(value || '').split(/[\\/]/).pop().toLowerCase(); } + function normalizeAutomationCompletion(value) { + const row = asObject(value); + const path = String(row.path || '').trim(); + const hoster = String(row.hoster || '').trim().toLowerCase(); + const size = Number(row.size); + const mtimeMs = Number(row.mtimeMs); + const completedAt = Number(row.completedAt); + if (!path || !hoster || !Number.isFinite(size) || size < 0 || !Number.isFinite(mtimeMs) || mtimeMs < 0) return null; + return { + path, + size, + mtimeMs: Math.trunc(mtimeMs), + hoster, + completedAt: Number.isFinite(completedAt) && completedAt >= 0 ? Math.trunc(completedAt) : 0 + }; + } + + function automationCompletionKey(value) { + const row = normalizeAutomationCompletion(value); + return row ? `${normalizePath(row.path)}\u0000${row.hoster}` : ''; + } + + function mergeAutomationCompletions(existing, incoming, maxEntries = 250000) { + const merged = new Map(); + for (const source of [asArray(existing), asArray(incoming)]) { + for (const value of source) { + const row = normalizeAutomationCompletion(value); + const key = automationCompletionKey(row); + if (key) { + merged.delete(key); + merged.set(key, row); + } + } + } + const limit = Number.isFinite(Number(maxEntries)) ? Math.max(1, Math.floor(Number(maxEntries))) : 250000; + if (merged.size > limit) throw new Error('Automatik-Abschlussdatei enthält zu viele Einträge'); + return [...merged.values()]; + } + + function removeAutomationCompletions(existing, removals) { + const rules = asArray(removals).map(value => ({ + path: normalizePath(value?.path), + hoster: String(value?.hoster || '').trim().toLowerCase() + })).filter(value => value.path); + if (rules.length === 0) return mergeAutomationCompletions(existing, []); + return mergeAutomationCompletions(existing, []).filter(row => { + const path = normalizePath(row.path); + return !rules.some(rule => rule.path === path && (!rule.hoster || rule.hoster === row.hoster)); + }); + } + + function classifyAutomationCompletionLedger(input = {}) { + const value = asObject(input); + const rows = new Map(); + for (const entry of asArray(value.completionRows)) { + const row = normalizeAutomationCompletion(entry); + const key = automationCompletionKey(row); + if (key) rows.set(key, row); + } + const processedPaths = []; + const completedByPath = []; + const remainingByPath = []; + for (const candidate of asArray(value.candidates)) { + const path = String(candidate?.path || ''); + const size = Number(candidate?.size); + const mtimeMs = Number(candidate?.mtimeMs); + const hosters = [...new Set(asArray(candidate?.eligibleHosters).map(hoster => String(hoster || '').trim().toLowerCase()).filter(Boolean))]; + const completed = []; + const remaining = []; + for (const hoster of hosters) { + const row = rows.get(`${normalizePath(path)}\u0000${hoster}`); + if (row && Number.isFinite(size) && size === row.size && Number.isFinite(mtimeMs) && Math.trunc(mtimeMs) === row.mtimeMs) completed.push(hoster); + else remaining.push(hoster); + } + if (hosters.length > 0 && remaining.length === 0) processedPaths.push(path); + completedByPath.push({ path, hosters: completed }); + remainingByPath.push({ path, hosters: remaining }); + } + return { processedPaths, completedByPath, remainingByPath }; + } + + function createAutomationCompletionWriter(options = {}) { + const settings = asObject(options); + if (typeof settings.save !== 'function') throw new TypeError('save is required'); + const schedule = typeof settings.schedule === 'function' ? settings.schedule : queueMicrotask; + const pending = new Map(); + let scheduled = false; + let tail = Promise.resolve(); + const flush = () => { + scheduled = false; + if (pending.size === 0) return tail; + const snapshot = [...pending.entries()]; + const rows = snapshot.map(([, row]) => row); + const operation = tail.then(async () => { + try { + await settings.save(rows); + for (const [key, row] of snapshot) { + if (pending.get(key) === row) pending.delete(key); + } + try { settings.onPersisted?.(rows); } catch {} + } catch (error) { + try { settings.onError?.(error, rows); } catch {} + throw error; + } + }); + tail = operation.catch(() => {}); + return operation; + }; + return { + add(value) { + const row = normalizeAutomationCompletion(value); + const key = automationCompletionKey(row); + if (!key) return false; + pending.set(key, row); + if (!scheduled) { + scheduled = true; + schedule(() => { flush().catch(() => {}); }); + } + return true; + }, + flush, + pendingCount: () => pending.size + }; + } + function classifyProcessedCandidates(input = {}) { const value = asObject(input); const candidates = asArray(value.candidates); @@ -191,6 +324,13 @@ rollDailyTelemetry, applyTelemetryDelta, deriveAutomationState, - classifyProcessedCandidates + isPathWithinAutomationFolder, + classifyProcessedCandidates, + classifyAutomationCompletionLedger, + automationCompletionKey, + createAutomationCompletionWriter, + normalizeAutomationCompletion, + mergeAutomationCompletions, + removeAutomationCompletions }; }); diff --git a/lib/config-store.js b/lib/config-store.js index bec4f33..b23b82d 100644 --- a/lib/config-store.js +++ b/lib/config-store.js @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); const secretStore = require('./secret-store'); const { normalizeLogMode } = require('./log-mode'); +const { mergeAutomationCompletions, normalizeAutomationCompletion, removeAutomationCompletions } = require('./automation-control'); const HOSTER_SETTINGS_DEFAULTS = { retries: 3, @@ -196,8 +197,11 @@ class ConfigStore { : path.join(__dirname, '..'); this.filePath = path.join(dir, 'electron-config.json'); this.historyPath = path.join(dir, 'electron-history.json'); + this.automationCompletionPath = path.join(dir, 'automation-completions.json'); this._writeQueue = Promise.resolve(); // Serializes all writes to prevent race conditions this._historyWriteQueue = Promise.resolve(); + this._automationCompletionWriteQueue = Promise.resolve(); + this._automationCompletionCache = null; this._pendingWriteOperations = new Set(); this._writesQuiesced = false; this._historyMigrated = false; @@ -622,6 +626,71 @@ class ConfigStore { }, options); } + async _readAutomationCompletionFile() { + try { + const raw = await fs.promises.readFile(this.automationCompletionPath, 'utf8'); + const parsed = JSON.parse(raw); + if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.entries)) throw new Error('Automatik-Abschlussdatei ist ungültig'); + if (parsed.entries.some(entry => !normalizeAutomationCompletion(entry))) throw new Error('Automatik-Abschlussdatei ist ungültig'); + return mergeAutomationCompletions([], parsed.entries); + } catch (error) { + if (error?.code === 'ENOENT') return []; + throw error; + } + } + + async _writeAutomationCompletionFile(entries) { + const target = this.automationCompletionPath; + const temporary = `${target}.tmp`; + await fs.promises.mkdir(path.dirname(target), { recursive: true }); + const handle = await fs.promises.open(temporary, 'w'); + try { + await handle.writeFile(JSON.stringify({ version: 1, entries }), 'utf8'); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.promises.rename(temporary, target); + } + + _enqueueAutomationCompletionWrite(operation) { + const result = this._automationCompletionWriteQueue.then(operation); + this._automationCompletionWriteQueue = result.catch(() => {}); + return result; + } + + async loadAutomationCompletions() { + await this._automationCompletionWriteQueue; + if (!this._automationCompletionCache) this._automationCompletionCache = await this._readAutomationCompletionFile(); + return this._clone(this._automationCompletionCache); + } + + saveAutomationCompletions(entries) { + const incoming = this._clone(Array.isArray(entries) ? entries : []); + return this._enqueueAutomationCompletionWrite(async () => { + const current = this._automationCompletionCache || await this._readAutomationCompletionFile(); + const next = mergeAutomationCompletions(current, incoming); + await this._writeAutomationCompletionFile(next); + this._automationCompletionCache = next; + return this._clone(next); + }); + } + + clearAutomationCompletions(removals) { + const requested = this._clone(Array.isArray(removals) ? removals : []); + return this._enqueueAutomationCompletionWrite(async () => { + const current = this._automationCompletionCache || await this._readAutomationCompletionFile(); + const next = removeAutomationCompletions(current, requested); + await this._writeAutomationCompletionFile(next); + this._automationCompletionCache = next; + return this._clone(next); + }); + } + + async drainAutomationCompletionWrites() { + await this._automationCompletionWriteQueue; + } + saveLastBrowseDirectory(directory) { const snapshot = String(directory || '').trim(); return this._enqueueWrite(() => { diff --git a/lib/import-preflight.js b/lib/import-preflight.js index e29eac7..af40351 100644 --- a/lib/import-preflight.js +++ b/lib/import-preflight.js @@ -27,7 +27,8 @@ return { path: filePath, name: sourceName || path.basename(filePath), - size: Number.isFinite(Number(source.size)) ? Number(source.size) : null + size: Number.isFinite(Number(source.size)) ? Number(source.size) : null, + mtimeMs: Number.isFinite(Number(source.mtimeMs)) ? Number(source.mtimeMs) : null }; } @@ -62,8 +63,8 @@ try { fileHandle = await openPath(filePath, 'r'); const fileStat = await fileHandle.stat(); - if (!fileStat.isFile()) return { exists: true, readable: false, size: fileStat.size }; - return { exists: true, readable: true, size: fileStat.size }; + if (!fileStat.isFile()) return { exists: true, readable: false, size: fileStat.size, mtimeMs: fileStat.mtimeMs }; + return { exists: true, readable: true, size: fileStat.size, mtimeMs: fileStat.mtimeMs }; } catch (error) { if (error && (error.code === 'ENOENT' || error.code === 'ENOTDIR')) return { exists: false }; return { exists: true, readable: false }; @@ -109,8 +110,9 @@ try { const result = await inspectPath(entry.path, entry); const reason = unavailableReason(result); - if (reason) return { entry: { ...entry, size: Number(result?.size) || 0 }, reason }; - return { entry: { ...entry, size: Number(result.size) }, reason: '' }; + const mtimeMs = Number.isFinite(Number(result?.mtimeMs)) ? Number(result.mtimeMs) : entry.mtimeMs; + if (reason) return { entry: { ...entry, size: Number(result?.size) || 0, mtimeMs }, reason }; + return { entry: { ...entry, size: Number(result.size), mtimeMs }, reason: '' }; } catch (error) { return { entry, reason: error && error.code === 'ENOENT' ? 'missing' : 'unreadable' }; } diff --git a/lib/log-mode.js b/lib/log-mode.js index 42a61cc..5bf4334 100644 --- a/lib/log-mode.js +++ b/lib/log-mode.js @@ -106,9 +106,11 @@ const normalizedBaseName = baseName.toLowerCase(); const normalizedExt = ext.toLowerCase(); if (!normalizedExt || !normalizedValue.endsWith(normalizedExt)) return false; - if (stripModeStampFromFileName(normalizedValue) === `${normalizedBaseName}${normalizedExt}`) return true; const stem = normalizedValue.slice(0, -normalizedExt.length); - return /^\d{2}-\d{2}-\d{4}-mdu-session-\d{2}-\d{2}(?:-\d+)?$/.test(stem); + const managedStem = stem.replace(/\.\d+$/, ''); + const managedValue = `${managedStem}${normalizedExt}`; + if (stripModeStampFromFileName(managedValue) === `${normalizedBaseName}${normalizedExt}`) return true; + return /^\d{2}-\d{2}-\d{4}-mdu-session-\d{2}-\d{2}(?:-\d+)?$/.test(managedStem); } const api = { normalizeLogMode, resolveLogFileName, formatDateStamp, formatSessionStamp, stripModeStampFromFileName, isManagedUploadLogFileName, VALID_MODES }; diff --git a/lib/stats.js b/lib/stats.js index fade21b..0269681 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -42,6 +42,7 @@ function classifyErrorCategory(err) { if (!err || typeof err !== 'string') return 'unknown'; const s = err.toLowerCase(); + if (/automatik-abschlussnachweis.*nicht gespeichert|automation completion evidence.*not saved/.test(s)) return 'local-persistence'; if (/abgebrochen|aborted|cancel/.test(s)) return 'aborted'; if (/not video file format|kein videoformat|invalid file|wrong format|duplicate|already exists|file too (small|big|large)|datei zu (gro|klein)/.test(s)) return 'file-rejected'; if (/quota|storage (full|exhausted|voll)|account (full|banned|suspended)|disk (space )?full|insufficient (disk )?space|not enough (disk )?(space|storage)/.test(s)) return 'account-error'; @@ -57,6 +58,7 @@ 'hoster-transient': [], 'network': [], 'unknown': [], + 'local-persistence': [], 'aborted': [] }; if (!batchSummary || !Array.isArray(batchSummary.files)) return buckets; @@ -129,6 +131,7 @@ 'hoster-transient': 'Hoster-Flake', 'network': 'Netzwerk', 'unknown': 'Unbekannt', + 'local-persistence': 'Lokale Speicherung', 'aborted': 'Abgebrochen' }; diff --git a/lib/upload-log.js b/lib/upload-log.js index 3f01ee9..4d48252 100644 --- a/lib/upload-log.js +++ b/lib/upload-log.js @@ -18,14 +18,74 @@ if (parts.length < 5) return null; const hoster = (parts[1] || '').trim(); let fileName = ''; + let fileNameIndex = -1; for (let i = parts.length - 1; i >= 4; i--) { - if (parts[i].trim() !== '') { fileName = parts[i]; break; } + if (parts[i].trim() !== '') { fileName = parts[i]; fileNameIndex = i; break; } } if (!hoster || !fileName) return null; + const confirmed = parts.slice(2, fileNameIndex).some(value => value.trim() !== ''); const tsStr = (parts[0] || '').trim(); const tsParsed = tsStr ? Date.parse(tsStr.replace(' ', 'T')) : NaN; const ts = isNaN(tsParsed) ? undefined : tsParsed; - return { hoster, fileName, ts }; + return { hoster, fileName, ts, confirmed }; + } + + async function* iterateBoundedUploadLogLines(chunks, maxLineLength) { + let buffer = ''; + for await (const chunk of chunks) { + buffer += String(chunk); + for (;;) { + const separator = buffer.indexOf('\n'); + if (separator < 0) break; + const line = buffer.slice(0, separator).replace(/\r$/, ''); + if (line.length > maxLineLength) throw new Error('Upload-Log-Zeile ist zu lang'); + yield line; + buffer = buffer.slice(separator + 1); + } + if (buffer.length > maxLineLength) throw new Error('Upload-Log-Zeile ist zu lang'); + } + if (buffer) yield buffer.replace(/\r$/, ''); + } + + async function* iterateBoundedUploadLogChunks(chunks, maxBytes, onBytes) { + const BufferImpl = typeof require === 'function' ? require('node:buffer').Buffer : null; + let total = 0; + for await (const chunk of chunks) { + const bytes = BufferImpl ? BufferImpl.byteLength(String(chunk), 'utf8') : String(chunk).length; + total += bytes; + if (total > maxBytes) throw new Error('Upload-Log überschreitet das Leselimit'); + if (typeof onBytes === 'function') onBytes(bytes); + yield chunk; + } + } + + async function* iterateUploadLogEntries(filePath, options = {}) { + const fsImpl = options.fs || (typeof require === 'function' ? require('node:fs') : null); + if (!options.lines && !fsImpl?.createReadStream) throw new Error('Upload-Log-Stream ist nicht verfügbar'); + const yieldEvery = Number.isFinite(Number(options.yieldEvery)) ? Math.max(1, Math.floor(Number(options.yieldEvery))) : 1000; + const maxLineLength = Number.isFinite(Number(options.maxLineLength)) ? Math.max(1, Math.floor(Number(options.maxLineLength))) : 65536; + const maxBytes = Number.isFinite(Number(options.maxBytes)) ? Math.max(1, Math.floor(Number(options.maxBytes))) : 256 * 1024 * 1024; + const yieldFn = typeof options.yieldFn === 'function' ? options.yieldFn : (() => new Promise(resolve => setImmediate(resolve))); + const input = options.lines ? null : fsImpl.createReadStream(filePath, { encoding: 'utf8', highWaterMark: 32768 }); + const lines = options.lines || iterateBoundedUploadLogLines(iterateBoundedUploadLogChunks(input, maxBytes, options.onBytes), maxLineLength); + let count = 0; + try { + for await (const line of lines) { + if (String(line).length > maxLineLength) throw new Error('Upload-Log-Zeile ist zu lang'); + const parsed = parseUploadLogLine(line); + if (parsed) yield parsed; + count++; + if (count % yieldEvery === 0) await yieldFn(); + } + } finally { + if (input && !input.destroyed && typeof input.destroy === 'function') input.destroy(); + } + } + + async function readUploadLogEntries(filePath, options = {}) { + const entries = []; + for await (const entry of iterateUploadLogEntries(filePath, options)) entries.push(entry); + return entries; } function summarizeBatchPlan(payload) { @@ -73,7 +133,7 @@ })}\r\n`; } - const api = { formatUploadLogLine, parseUploadLogLine, summarizeBatchPlan, formatUploadPlanLogLine }; + const api = { formatUploadLogLine, parseUploadLogLine, iterateUploadLogEntries, readUploadLogEntries, summarizeBatchPlan, formatUploadPlanLogLine }; if (typeof module !== 'undefined' && module.exports) module.exports = api; else if (root) root.UploadLog = api; })(typeof window !== 'undefined' ? window : this); diff --git a/lib/upload-manager.js b/lib/upload-manager.js index 1e86f8e..0635971 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -489,6 +489,7 @@ class UploadManager extends EventEmitter { finalStatus = status; const result = { + jobId, hoster: task.hoster, status, error: payload.error || null, diff --git a/main.js b/main.js index 636dcbe..a917802 100644 --- a/main.js +++ b/main.js @@ -29,7 +29,7 @@ const { walkFolderAsync } = require('./lib/file-discovery'); const RemoteServer = require('./lib/remote-server'); const { maybeRotateLogFile } = require('./lib/log-rotation'); const { hosterLogToFileEnabled } = require('./lib/log-policy'); -const { formatUploadLogLine, parseUploadLogLine, summarizeBatchPlan, formatUploadPlanLogLine } = require('./lib/upload-log'); +const { formatUploadLogLine, iterateUploadLogEntries, summarizeBatchPlan, formatUploadPlanLogLine } = require('./lib/upload-log'); const { createInternalLogPathResolver, createInternalLogWriter, createUploadAuditWriter, createBufferedInternalLogFlusher, getLogOpenDirectory } = require('./lib/upload-audit'); const { selectOrphanTmps } = require('./lib/orphan-tmp'); const { sanitizeConfig, buildSupportBundleText, collectSecretValues, redactLogText, valueScrub, collectFile, REDACTED } = require('./lib/support-bundle'); @@ -39,7 +39,7 @@ const { createCollectors } = require('./lib/diagnostics-collectors'); const { createAgent } = require('./lib/diagnostics-agent'); const { buildSessionReport, buildSessionReportCsv } = require('./lib/session-report'); const { inspectImportEntries, inspectReadableImportPath } = require('./lib/import-preflight'); -const { normalizeAutomationSettings } = require('./lib/automation-control'); +const { normalizeAutomationSettings, automationCompletionKey, createAutomationCompletionWriter, isPathWithinAutomationFolder, normalizeAutomationCompletion } = require('./lib/automation-control'); const _eventLoopDelay = monitorEventLoopDelay({ resolution: 10 }); _eventLoopDelay.enable(); @@ -125,8 +125,10 @@ const updateAnnouncementState = createUpdateAnnouncementState(); let _lastImportPath = null; let dropTargetWindow = null; let tray = null; +let _cachedLogSettings = null; const configStore = new ConfigStore(app); configStore.setPerfLog((m) => { try { logInfo(m); } catch {} }); +_setLogSettingsSnapshot((configStore.load() || {}).globalSettings); const onlineBackupKeyring = createOnlineBackupKeyring({ filePath: path.join(app.getPath('userData'), 'online-backup-keys.json') }); @@ -153,7 +155,7 @@ async function waitForUploadManagerRelease(manager, timeoutMs = 300000) { } } -function requestUploadFinalization(summary) { +function requestUploadFinalization(summary, preserveQueue = false) { const finalizationId = `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; return new Promise((resolve) => { const timer = setTimeout(() => { @@ -167,7 +169,7 @@ function requestUploadFinalization(summary) { resolve(value); } }); - safeSend('upload-batch-done', { summary, finalizationId }); + safeSend('upload-batch-done', { summary, finalizationId, preserveQueue }); }); } const activeUploadProducerTrackers = new Set(); @@ -181,6 +183,7 @@ function assertConfigWriteAllowed() { async function waitForConfigStoreWrites() { await configStore.drainWrites(); + await configStore.drainAutomationCompletionWrites(); } const ONLINE_BACKUP_RENDERER_URL = pathToFileURL(path.join(__dirname, 'renderer', 'index.html')); @@ -790,18 +793,20 @@ function getDefaultLogFilePath() { // (incl. an 8 MB+ history) on every flush — a major long-running main-thread // drag. logFilePath/logMode change only when the user saves settings, so cache // the two strings and invalidate on those saves (see _invalidateLogSettings). -let _cachedLogSettings = null; +function _setLogSettingsSnapshot(globalSettings) { + const settings = globalSettings || {}; + _cachedLogSettings = { + logFilePath: String(settings.logFilePath || '').trim(), + logMode: settings.logMode || 'single' + }; +} function _getLogSettings() { - if (!_cachedLogSettings) { - const gs = (configStore.load() || {}).globalSettings || {}; - _cachedLogSettings = { - logFilePath: String(gs.logFilePath || '').trim(), - logMode: gs.logMode || 'single' - }; - } - return _cachedLogSettings; + return _cachedLogSettings || { logFilePath: '', logMode: 'single' }; +} +function _invalidateLogSettings(globalSettings) { + _setLogSettingsSnapshot(globalSettings); + _invalidateUploadLogEvidenceCache(); } -function _invalidateLogSettings() { _cachedLogSettings = null; } function getBaseLogFilePath() { const customPath = _getLogSettings().logFilePath; @@ -958,13 +963,16 @@ function _flushUploadLog() { _flushUploadLog(); }, 1000); } - } else if (target.isFallback && !_uploadLogFallbackWarned) { - _uploadLogFallbackWarned = true; - // Auto-persist the working fallback into the user's config so the - // next session writes here directly (no more fallback ladder) and - // the Settings input reflects reality. - _persistFallbackLogPath(target.path); - safeSend('upload-log-fallback', { fallbackPath: target.path }); + } else { + _invalidateUploadLogEvidenceCache(); + if (target.isFallback && !_uploadLogFallbackWarned) { + _uploadLogFallbackWarned = true; + // Auto-persist the working fallback into the user's config so the + // next session writes here directly (no more fallback ladder) and + // the Settings input reflects reality. + _persistFallbackLogPath(target.path); + safeSend('upload-log-fallback', { fallbackPath: target.path }); + } } if (_uploadLogBuffer.length && !_uploadLogFlushTimer) setImmediate(_flushUploadLog); }); @@ -993,7 +1001,7 @@ async function _persistFallbackLogPath(workingPath) { cfg.globalSettings = gs; await configStore.save({ globalSettings: gs }); _invalidateUploadLogTargetCache(); - _invalidateLogSettings(); + _invalidateLogSettings(gs); safeSend('log-path-auto-updated', { logFilePath: toSave }); return true; } catch (err) { @@ -1225,6 +1233,42 @@ function buildUploadTasksFromJobs(config, jobs, pick) { return tasks; } +async function registerAutomationCompletionJobs(manager, jobs) { + if (!manager || !Array.isArray(jobs)) return; + if (!manager._automationCompletionMetadata) manager._automationCompletionMetadata = new Map(); + const folderSettings = configStore.load().globalSettings?.folderMonitor || {}; + const candidates = jobs.filter(job => { + const monitoredManualJob = folderSettings.enabled === true && isPathWithinAutomationFolder(job?.file, folderSettings.folderPath, folderSettings.recursive === true); + return (job?.automationAdmission === true || monitoredManualJob) && job.id && job.file && job.hoster; + }); + let cursor = 0; + const hasFiniteMetadata = value => value !== null && value !== undefined && value !== '' && Number.isFinite(Number(value)); + async function worker() { + while (cursor < candidates.length) { + const job = candidates[cursor++]; + const sourceSize = job.sourceSize ?? job.automationSize ?? job.bytesTotal; + const sourceMtimeMs = job.sourceMtimeMs ?? job.automationMtimeMs; + let size = hasFiniteMetadata(sourceSize) ? Number(sourceSize) : Number.NaN; + let mtimeMs = hasFiniteMetadata(sourceMtimeMs) ? Number(sourceMtimeMs) : Number.NaN; + if (!Number.isFinite(size) || !Number.isFinite(mtimeMs)) { + try { + const stat = await fs.promises.stat(job.file); + size = Number(stat.size); + mtimeMs = Number(stat.mtimeMs); + } catch {} + } + if (!Number.isFinite(size) || !Number.isFinite(mtimeMs)) continue; + manager._automationCompletionMetadata.set(job.id, { + path: job.file, + size, + mtimeMs, + hoster: job.hoster + }); + } + } + await Promise.all(Array.from({ length: Math.min(16, candidates.length) }, worker)); +} + async function checkDoodstreamHealth(hosterConfig, otp) { const username = hosterConfig && hosterConfig.username ? String(hosterConfig.username).trim() @@ -1828,7 +1872,7 @@ ipcMain.handle('get-config', () => { ipcMain.handle('save-config', async (_event, config) => { assertConfigWriteAllowed(); await configStore.save(config); - if (config && config.globalSettings) _invalidateLogSettings(); + if (config && config.globalSettings) _invalidateLogSettings(config.globalSettings); try { if (config && config.globalSettings && Object.prototype.hasOwnProperty.call(config.globalSettings, 'logVerbose')) { setLogVerbose(!!config.globalSettings.logVerbose); @@ -2294,6 +2338,7 @@ ipcMain.handle('start-upload', async (_event, payload) => { uploadManager = new UploadManager(config.hosterSettings || {}, config.globalSettings || {}, buildAccountPools(config)); globalThis._mhuUploadManagerRef = uploadManager; const _thisManager = uploadManager; + await registerAutomationCompletionJobs(_thisManager, jobs); await appendUploadPlanAudit(batchPlan, 'start'); if (configStore.load().globalSettings?.folderMonitor?.paused === true) { @@ -2379,6 +2424,35 @@ ipcMain.handle('start-upload', async (_event, payload) => { }, PROGRESS_BATCH_INTERVAL_MS); } + function _queueProgressForRenderer(data) { + const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped'; + if (isTerminal) { + if (data.jobId) _progressByJob.delete(data.jobId); + _progressTerminalQueue.push(data); + } else if (data.jobId) { + _progressByJob.set(data.jobId, data); + } else { + _progressTerminalQueue.push(data); + } + _scheduleProgressFlush(); + } + + _thisManager._automationCompletionProgress = new Map(); + _thisManager._automationCompletionWriter = createAutomationCompletionWriter({ + schedule: callback => setTimeout(callback, 100), + save: entries => configStore.saveAutomationCompletions(entries), + onPersisted: entries => { + for (const entry of entries) { + const key = automationCompletionKey(entry); + const progress = _thisManager._automationCompletionProgress.get(key); + if (!progress) continue; + _thisManager._automationCompletionProgress.delete(key); + _queueProgressForRenderer(progress); + } + }, + onError: error => debugLog(`automation completion ledger failed: ${error.message}`) + }); + uploadManager.on('progress', (data) => { if (data.status !== 'uploading') { debugLog(`progress: ${data.fileName} ${data.hoster} ${data.status} ${data.error || ''}`); @@ -2389,6 +2463,7 @@ ipcMain.handle('start-upload', async (_event, payload) => { }); } if (data.status === 'done' && data.result) { + _invalidateUploadLogEvidenceCache(); const link = data.result.download_url || data.result.embed_url || data.result.file_code || ''; if (link) { if (shouldLogHosterToFile(data.hoster)) { @@ -2400,16 +2475,16 @@ ipcMain.handle('start-upload', async (_event, payload) => { debugLog(`WARNING: done but no link for ${data.fileName} @ ${data.hoster}: ${JSON.stringify(data.result)}`); } } - const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped'; - if (isTerminal) { - if (data.jobId) _progressByJob.delete(data.jobId); - _progressTerminalQueue.push(data); - } else if (data.jobId) { - _progressByJob.set(data.jobId, data); - } else { - _progressTerminalQueue.push(data); + if (data.status === 'done' && data.jobId) { + const completion = _thisManager._automationCompletionMetadata?.get(data.jobId); + if (completion) { + const entry = { ...completion, completedAt: Date.now() }; + _thisManager._automationCompletionProgress.set(automationCompletionKey(entry), data); + _thisManager._automationCompletionWriter.add(entry); + return; + } } - _scheduleProgressFlush(); + _queueProgressForRenderer(data); }); uploadManager.on('stats', (data) => { @@ -2493,6 +2568,34 @@ ipcMain.handle('start-upload', async (_event, payload) => { // orphans (cancel/addJobs see null, the new batch keeps running invisibly). uploadManager.on('batch-done', async (summary) => { summary = stats.mergeSkippedIntoSummary(summary, skippedJobs); + let automationCompletionsPersisted = true; + try { await _thisManager._automationCompletionWriter?.flush(); } catch (error) { + automationCompletionsPersisted = false; + debugLog(`automation completion ledger failed: ${error.message}`); + } + if (!automationCompletionsPersisted) { + const failedJobIds = new Set(); + for (const progress of _thisManager._automationCompletionProgress.values()) { + if (progress.jobId) failedJobIds.add(progress.jobId); + _queueProgressForRenderer({ + ...progress, + status: 'error', + error: 'Automatik-Abschlussnachweis konnte nicht gespeichert werden' + }); + } + _thisManager._automationCompletionProgress.clear(); + let changed = 0; + for (const file of summary.files || []) { + for (const result of file.results || []) { + if (!failedJobIds.has(result.jobId)) continue; + result.status = 'error'; + result.error = 'Automatik-Abschlussnachweis konnte nicht gespeichert werden'; + changed++; + } + } + summary.succeeded = Math.max(0, Number(summary.succeeded) - changed); + summary.failed = Math.max(0, Number(summary.failed) + changed); + } lastSessionSummary = summary; debugLog(`batch-done: total=${summary.total} ok=${summary.succeeded} fail=${summary.failed}`); logMarker('BATCH END', { total: summary.total, ok: summary.succeeded, fail: summary.failed }); @@ -2513,10 +2616,13 @@ ipcMain.handle('start-upload', async (_event, payload) => { for (const value of _progressByJob.values()) finalProgressBatch.push(value); _progressByJob.clear(); if (finalProgressBatch.length) safeSend('upload-progress-batch', finalProgressBatch); - const queuePersisted = await requestUploadFinalization(summary); - try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); } - if (!queuePersisted) debugLog('upload finalization blocked: renderer queue acknowledgement missing'); - await sourceCleanup.finishBatch({ historyPersisted, queuePersisted }); + const queuePersisted = await requestUploadFinalization(summary, !automationCompletionsPersisted); + const finalizationPersisted = queuePersisted && automationCompletionsPersisted; + if (finalizationPersisted) { + try { await configStore.saveUploadRecovery(null); } catch (error) { debugLog(`upload recovery state could not be cleared: ${error.message}`); } + } + if (!finalizationPersisted) debugLog('upload finalization blocked: queue or automation completion evidence was not persisted'); + await sourceCleanup.finishBatch({ historyPersisted, queuePersisted: finalizationPersisted }); _producerTracker.finish(); const fullyAborted = isAllAborted(summary); @@ -2615,12 +2721,12 @@ ipcMain.handle('add-jobs-to-batch', async (_event, payload) => { return { added: 0, skippedJobs, alreadyInBatchJobIds: [], sourceCleanupFingerprints }; } + await registerAutomationCompletionJobs(batchManager, jobs); const addResult = batchManager.addJobs(tasks); const added = typeof addResult === 'number' ? addResult : (addResult && addResult.added) || 0; const alreadyInBatchJobIds = (addResult && Array.isArray(addResult.alreadyInBatchJobIds)) ? addResult.alreadyInBatchJobIds : []; - debugLog( `add-jobs-to-batch: ${added} of ${tasks.length} tasks added (${alreadyInBatchJobIds.length} already in batch, ${skippedJobs.length} skipped)` ); @@ -2883,8 +2989,8 @@ async function applyImportedSettings(imported) { _rotationCursors = {}; _accountCooldowns.clear(); _sessionAccountOverrides.clear(); - _invalidateLogSettings(); const config = configStore.load(); + _invalidateLogSettings(config.globalSettings); const warnings = await syncImportedRuntime(config); return { config, warnings }; } finally { @@ -3015,23 +3121,37 @@ ipcMain.handle('online-backup:restore', async (_event, key) => { } }); -ipcMain.handle('read-own-upload-log', async () => { +let _uploadLogEvidenceCache = null; +let _uploadLogEvidenceInFlight = null; +let _uploadLogEvidenceGeneration = 0; + +function _invalidateUploadLogEvidenceCache() { + _uploadLogEvidenceCache = null; + _uploadLogEvidenceGeneration++; +} + +async function _scanOwnUploadLog() { const entries = new Map(); const basePath = getBaseLogFilePath(); const dir = path.dirname(basePath); const ext = path.extname(basePath); const name = path.basename(basePath, ext); - - const activeTarget = _resolveUploadLogTarget(); const directories = new Set([dir]); - if (activeTarget?.path) directories.add(path.dirname(activeTarget.path)); - const desktop = getSafeDesktopDir(); - if (desktop) directories.add(desktop); + if (_activeLogPath) directories.add(path.dirname(_activeLogPath)); + try { + const desktop = app.getPath('desktop'); + if (desktop) directories.add(desktop); + } catch {} try { directories.add(app.getPath('userData')); } catch {} const logFiles = new Set(); for (const directory of directories) { try { - for (const file of fs.readdirSync(directory)) { + const directoryHandle = await fs.promises.opendir(directory); + let directoryEntries = 0; + for await (const entry of directoryHandle) { + directoryEntries++; + if (directoryEntries > 50000) throw new Error('Upload-Log-Verzeichnis enthält zu viele Einträge'); + const file = entry.name; if ( isManagedUploadLogFileName(file, { baseName: name, ext }) || isManagedUploadLogFileName(file, { baseName: 'fileuploader', ext: '.log' }) @@ -3039,26 +3159,63 @@ ipcMain.handle('read-own-upload-log', async () => { logFiles.add(path.join(directory, file)); } } - } catch {} + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } } - if (activeTarget?.path && fs.existsSync(activeTarget.path)) logFiles.add(activeTarget.path); - if (fs.existsSync(basePath)) logFiles.add(basePath); + if (_activeLogPath) logFiles.add(_activeLogPath); + logFiles.add(basePath); + if (logFiles.size > 256) throw new Error('Zu viele verwaltete Upload-Logs'); - for (const logPath of logFiles) { + let expectedBytes = 0; + let actualBytes = 0; + for (const logPath of [...logFiles].sort()) { try { - const content = await fs.promises.readFile(logPath, 'utf-8'); - for (const line of content.split('\n')) { - const parsed = parseUploadLogLine(line); - if (!parsed) continue; + if (typeof fs.promises.stat === 'function') { + const stat = await fs.promises.stat(logPath); + expectedBytes += Number(stat.size) || 0; + if (expectedBytes > 256 * 1024 * 1024) throw new Error('Upload-Logs überschreiten das Leselimit'); + } + for await (const parsed of iterateUploadLogEntries(logPath, { + maxBytes: 256 * 1024 * 1024, + onBytes(bytes) { + actualBytes += bytes; + if (actualBytes > 256 * 1024 * 1024) throw new Error('Upload-Logs überschreiten das Leselimit'); + } + })) { + if (parsed.confirmed !== true) continue; const key = `${parsed.hoster.toLowerCase()}\u0000${parsed.fileName.toLowerCase()}`; const previous = entries.get(key); const timestamp = Number.isFinite(parsed.ts) ? parsed.ts : -Infinity; const previousTimestamp = Number.isFinite(previous?.ts) ? previous.ts : -Infinity; if (!previous || timestamp >= previousTimestamp) entries.set(key, parsed); + if (entries.size > 250000) throw new Error('Upload-Log enthält zu viele eindeutige Einträge'); } - } catch {} + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } } return [...entries.values()]; +} + +ipcMain.handle('read-own-upload-log', async () => { + const now = Date.now(); + if (_uploadLogEvidenceCache?.expiresAt > now) return _uploadLogEvidenceCache.entries; + const generation = _uploadLogEvidenceGeneration; + if (_uploadLogEvidenceInFlight?.generation === generation) return _uploadLogEvidenceInFlight.promise; + const pending = _scanOwnUploadLog().then(entries => { + if (generation === _uploadLogEvidenceGeneration) { + _uploadLogEvidenceCache = { entries, expiresAt: Date.now() + 5000 }; + } + return entries; + }); + const inFlight = { generation, promise: pending }; + _uploadLogEvidenceInFlight = inFlight; + try { + return await pending; + } finally { + if (_uploadLogEvidenceInFlight === inFlight) _uploadLogEvidenceInFlight = null; + } }); ipcMain.handle('import-upload-log', async () => { @@ -3252,7 +3409,7 @@ ipcMain.handle('save-global-settings', async (_event, globalSettings) => { assertConfigWriteAllowed(); await configStore.saveRendererGlobalSettings(globalSettings); globalSettings = configStore.load().globalSettings; - _invalidateLogSettings(); + _invalidateLogSettings(globalSettings); if (uploadManager) { try { uploadManager.updateSettings(null, globalSettings); } catch (error) { debugLog(`global settings runtime update failed: ${error.message}`); } } @@ -3467,6 +3624,18 @@ ipcMain.handle('automation:get-status', () => { return automationStatusSnapshot(); }); +ipcMain.handle('automation:get-completions', () => { + return configStore.loadAutomationCompletions(); +}); + +ipcMain.handle('automation:record-completions', async (_event, entries) => { + const source = Array.isArray(entries) ? entries.slice(0, 10000) : []; + const normalized = source.map(normalizeAutomationCompletion); + if (source.length === 0 || normalized.some(entry => !entry)) throw new Error('Automatik-Abschlussnachweise sind ungültig'); + await configStore.saveAutomationCompletions(normalized); + return true; +}); + ipcMain.handle('automation:pause-after-active', () => enqueueAutomationLifecycle(async generation => { let monitorError = ''; await withAutomationStatusSuppressed(async () => { diff --git a/preload.js b/preload.js index 4355fcc..d811731 100644 --- a/preload.js +++ b/preload.js @@ -110,6 +110,8 @@ contextBridge.exposeInMainWorld('api', { folderMonitorTestScan: () => ipcRenderer.invoke('folder-monitor:test-scan'), folderMonitorReconcile: () => ipcRenderer.invoke('folder-monitor:reconcile'), automationGetStatus: () => ipcRenderer.invoke('automation:get-status'), + getAutomationCompletions: () => ipcRenderer.invoke('automation:get-completions'), + recordAutomationCompletions: (entries) => ipcRenderer.invoke('automation:record-completions', entries), automationPauseAfterActive: () => ipcRenderer.invoke('automation:pause-after-active'), automationResume: () => ipcRenderer.invoke('automation:resume'), onFolderMonitorNewFiles: (callback) => { diff --git a/renderer/app.js b/renderer/app.js index 0b5ee96..5b3151b 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -397,6 +397,7 @@ let _updateDialogReturnFocus = null; let _updateDialogInertState = []; let _startupAutoResumeController = null; let _startupAutoResumeCanceled = false; +let _startupQueueEvidenceAvailable = true; // Session-specific files for the "Files" panel (resets each session) let sessionFilesData = []; @@ -556,11 +557,12 @@ function createAutomationStatusSnapshot() { } async function loadAutomationEvidenceSnapshot() { - const [history, uploadLog] = await Promise.all([ + const [history, uploadLog, automationCompletions] = await Promise.all([ window.api.getHistory(), - window.api.readOwnUploadLog() + window.api.readOwnUploadLog(), + typeof window.api.getAutomationCompletions === 'function' ? window.api.getAutomationCompletions() : Promise.resolve([]) ]); - return { history, uploadLog }; + return { history, uploadLog, automationCompletions }; } function invalidateAutomationEvidenceSnapshot() { @@ -598,14 +600,16 @@ async function evaluateAutomationCandidates(files, options = {}) { const selectedHosters = Array.from(new Set((Array.isArray(options.selectedHosters) ? options.selectedHosters : folderSettings.hosters || []) .map(value => String(value || '').trim()) .filter(Boolean))); - const { history, uploadLog } = options.evidenceSnapshot || await loadAutomationEvidenceSnapshot(); - const completedPaths = [..._completedUploadKeys].map(key => { - const separator = key.lastIndexOf('|'); - return separator > 0 ? key.slice(0, separator) : ''; - }).filter(Boolean); + const { history, uploadLog, automationCompletions } = options.evidenceSnapshot || await loadAutomationEvidenceSnapshot(); + const currentPaths = new Set([...queueJobs.map(job => job.file), ...selectedFiles.map(file => file.path)].map(normalizeAutomationPath)); + const durableCompletionPaths = new Set((Array.isArray(automationCompletions) ? automationCompletions : []).map(row => normalizeAutomationPath(row?.path)).filter(Boolean)); + const legacyCandidates = matched.filter(candidate => { + const key = normalizeAutomationPath(candidate.path); + return currentPaths.has(key) || !durableCompletionPaths.has(key); + }); const processed = window.AutomationControl.classifyProcessedCandidates({ - candidates: matched, - queuePaths: [...queueJobs.map(job => job.file), ...selectedFiles.map(file => file.path), ...completedPaths], + candidates: legacyCandidates, + queuePaths: [...queueJobs.map(job => job.file), ...selectedFiles.map(file => file.path)], historyRows: flattenAutomationHistoryRows(history), uploadLogRows: uploadLog }); @@ -625,7 +629,7 @@ async function evaluateAutomationCandidates(files, options = {}) { ...(metadata.get(normalizeAutomationPath(file?.path)) || {}), ...file })); - const plannedCandidates = accepted.map(file => { + const initialPlannedCandidates = accepted.map(file => { const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings); return { path: file.path, @@ -636,6 +640,28 @@ async function evaluateAutomationCandidates(files, options = {}) { eligibleJobCount: eligibleHosters.length }; }); + const completionState = window.AutomationControl.classifyAutomationCompletionLedger({ + candidates: initialPlannedCandidates, + completionRows: automationCompletions + }); + const ledgerProcessedPaths = new Set(completionState.processedPaths.map(normalizeAutomationPath)); + const completedHostersByPath = new Map(completionState.completedByPath.map(entry => [normalizeAutomationPath(entry.path), new Set(entry.hosters)])); + for (const path of ledgerProcessedPaths) { + processedPaths.add(path); + reasons.set(path, 'processed'); + } + const plannedCandidates = initialPlannedCandidates + .filter(candidate => !ledgerProcessedPaths.has(normalizeAutomationPath(candidate.path))) + .map(candidate => { + const completedHosters = completedHostersByPath.get(normalizeAutomationPath(candidate.path)) || new Set(); + const eligibleHosters = candidate.eligibleHosters.filter(hoster => !completedHosters.has(String(hoster).toLowerCase())); + return { + ...candidate, + eligibleHosters, + eligibleJobCount: eligibleHosters.length, + completedHosters: [...completedHosters] + }; + }); const normalizedSettings = automationSettings(); const currentJobCount = window.AutomationControl.countAutomaticQueueJobs(queueJobs); const availableSlots = normalizedSettings.queueLimitJobs === 0 ? null : Math.max(0, normalizedSettings.queueLimitJobs - currentJobCount); @@ -657,6 +683,7 @@ async function evaluateAutomationCandidates(files, options = {}) { } const actionableCandidates = plannedCandidates.filter(candidate => candidate.eligibleJobCount > 0); const resultingJobs = actionableCandidates.reduce((total, candidate) => total + candidate.eligibleJobCount, 0); + const sizeLimitedJobs = initialPlannedCandidates.reduce((total, candidate) => total + Math.max(0, selectedHosters.length - candidate.eligibleHosters.length), 0); const classifications = candidates.map(candidate => ({ path: candidate.path, name: candidate.name, @@ -666,9 +693,9 @@ async function evaluateAutomationCandidates(files, options = {}) { const summary = { found: candidates.length, filterMatched: matched.length, - alreadyProcessed: processed.processedPaths.length + inspectionDuplicatePaths.size, + alreadyProcessed: processed.processedPaths.length + inspectionDuplicatePaths.size + ledgerProcessedPaths.size, unavailable: unavailablePaths.size, - sizeLimitedJobs: plannedCandidates.length * selectedHosters.length - resultingJobs, + sizeLimitedJobs, acceptedFiles: selectedHosters.length === 0 ? plannedCandidates.length : actionableCandidates.length, selectedTargets: selectedHosters.length, resultingJobs, @@ -714,6 +741,10 @@ function createAutomationPreviewJob(file, hoster) { attempt: 0, maxAttempts: 0, link: '', + automationMtimeMs: file.mtimeMs, + automationSize: file.size, + sourceMtimeMs: file.mtimeMs, + sourceSize: file.size, automationAdmission: true }; } @@ -750,14 +781,17 @@ async function applyAutomationEvaluation(evaluation) { .map(value => String(value || '').trim()) .filter(Boolean))); const replannedCandidates = evaluation.candidates.map(file => { - const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings); + const completedHosters = new Set((Array.isArray(file.completedHosters) ? file.completedHosters : []).map(hoster => String(hoster || '').toLowerCase())); + const eligibleHosters = window.ImportPreflight.getEligibleImportHosters(file, selectedHosters, hosterSettings) + .filter(hoster => !completedHosters.has(String(hoster).toLowerCase())); return { path: file.path, name: file.name, size: file.size, mtimeMs: file.mtimeMs, eligibleHosters, - eligibleJobCount: eligibleHosters.length + eligibleJobCount: eligibleHosters.length, + completedHosters: [...completedHosters] }; }); const ownedPendingPaths = new Set(Array.isArray(evaluation.ownedPendingPaths) ? evaluation.ownedPendingPaths : []); @@ -1319,6 +1353,7 @@ async function init() { renderHosterSummary(); renderHosterModal(); renderSettings(); + if (!_startupQueueEvidenceAvailable) showCopyToast('Automatische Wiederaufnahme wurde wegen nicht verfügbarer Abschlussnachweise blockiert.', 9000); renderAccounts(); setupListeners(); importEntryCoordinator.ready(); @@ -1355,7 +1390,7 @@ async function init() { queuePersistThrottle.cancel(); await window.api.completeUploadFinalization({ finalizationId: data.finalizationId, - pendingQueue: queueJobs.some((job) => !['done', 'skipped'].includes(job.status)) + pendingQueue: data.preserveQueue === true || queueJobs.some((job) => !['done', 'skipped'].includes(job.status)) ? buildPersistedQueueState() : null }); @@ -2149,7 +2184,12 @@ function restoreQueueStateFromConfig() { selectedFiles = Array.isArray(pending.selectedFiles) ? pending.selectedFiles .filter(file => file && file.path) - .map(file => ({ path: file.path, name: file.name || file.path.split(/[\\/]/).pop(), size: file.size || 0 })) + .map(file => ({ + path: file.path, + name: file.name || file.path.split(/[\\/]/).pop(), + size: file.size || 0, + mtimeMs: Number.isFinite(Number(file.mtimeMs)) ? Number(file.mtimeMs) : null + })) : []; const interruptedJobIds = new Set(Array.isArray(config?.globalSettings?.uploadRecovery?.jobIds) ? config.globalSettings.uploadRecovery.jobIds : []); @@ -2177,6 +2217,10 @@ function restoreQueueStateFromConfig() { sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [], sourceCleanupFingerprint: job.sourceCleanupFingerprint || null, automationAdmission: job.automationAdmission === true, + automationMtimeMs: Number.isFinite(Number(job.automationMtimeMs)) ? Number(job.automationMtimeMs) : null, + automationSize: Number.isFinite(Number(job.automationSize)) ? Number(job.automationSize) : null, + sourceMtimeMs: Number.isFinite(Number(job.sourceMtimeMs)) ? Number(job.sourceMtimeMs) : null, + sourceSize: Number.isFinite(Number(job.sourceSize)) ? Number(job.sourceSize) : null, ...(job.automationPaused === true ? { automationPaused: true } : {}), attempt: 0, maxAttempts: job.maxAttempts || 0, @@ -2208,7 +2252,8 @@ function buildPersistedQueueState() { selectedFileMap.set(job.file, { path: job.file, name: job.fileName, - size: job.bytesTotal || 0 + size: job.sourceSize ?? job.automationSize ?? job.bytesTotal ?? 0, + mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs ?? null }); } } @@ -2257,6 +2302,10 @@ function buildPersistedQueueState() { sourceCleanupCompletedHosters: Array.isArray(job.sourceCleanupCompletedHosters) ? [...job.sourceCleanupCompletedHosters] : [], sourceCleanupFingerprint: job.sourceCleanupFingerprint || null, automationAdmission: job.automationAdmission === true, + automationMtimeMs: Number.isFinite(Number(job.automationMtimeMs)) ? Number(job.automationMtimeMs) : null, + automationSize: Number.isFinite(Number(job.automationSize)) ? Number(job.automationSize) : null, + sourceMtimeMs: Number.isFinite(Number(job.sourceMtimeMs)) ? Number(job.sourceMtimeMs) : null, + sourceSize: Number.isFinite(Number(job.sourceSize)) ? Number(job.sourceSize) : null, ...(automationPaused ? { automationPaused: true } : {}), maxAttempts: job.maxAttempts || 0 }; @@ -2657,6 +2706,8 @@ function buildQueuePreview() { id: `preview-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, file: file.path, fileName: file.name, hoster, status: 'preview', bytesUploaded: 0, bytesTotal: file.size || 0, + sourceMtimeMs: file.mtimeMs, + sourceSize: file.size, speedKbs: 0, elapsed: 0, remaining: 0, error: null, result: null, attempt: 0, maxAttempts: 0, link: '' }; @@ -2702,7 +2753,7 @@ async function startRestoredQueueAfterChecks(jobIds) { } function scheduleRestoredQueueAutoStart() { - if (config?.globalSettings?.autoStartRestoredQueue !== true || !window.AutoResume) return; + if (!_startupQueueEvidenceAvailable || config?.globalSettings?.autoStartRestoredQueue !== true || !window.AutoResume) return; const jobs = window.AutoResume.getAutoResumeJobs(queueJobs); if (!jobs.length) return; _startupAutoResumeCanceled = false; @@ -4275,7 +4326,12 @@ function serializeUploadJob(job) { sourceCleanupToken: job.sourceCleanupToken || null, sourceCleanupRequiredHosters: job.sourceCleanupRequiredHosters || [], sourceCleanupCompletedHosters: job.sourceCleanupCompletedHosters || [], - sourceCleanupFingerprint: job.sourceCleanupFingerprint || null + sourceCleanupFingerprint: job.sourceCleanupFingerprint || null, + automationAdmission: job.automationAdmission === true, + automationMtimeMs: job.automationMtimeMs, + automationSize: job.automationSize, + sourceMtimeMs: job.sourceMtimeMs, + sourceSize: job.sourceSize }; } @@ -5363,7 +5419,8 @@ function syncSelectedFilesFromQueue() { fileMap.set(job.file, { path: job.file, name: job.fileName, - size: job.bytesTotal || 0 + size: job.sourceSize ?? job.automationSize ?? job.bytesTotal ?? 0, + mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs ?? null }); }); selectedFiles = Array.from(fileMap.values()); @@ -9696,31 +9753,76 @@ function handleShutdownCountdown(data) { // --- Auto-deduplicate restored queue against own upload log on startup --- async function _autoDeduplicateFromLog() { - if (queueJobs.length === 0 && selectedFiles.length === 0) return; + if (queueJobs.length === 0 && selectedFiles.length === 0) { + _startupQueueEvidenceAvailable = true; + return true; + } try { - const entries = await window.api.readOwnUploadLog(); - if (!entries || entries.length === 0) return; - // Drops 'done' jobs present in the log (declutter) AND any job that the log - // shows completed at/after the snapshot's savedAt (a stale 'preview' ghost). - // Pending jobs matching only OLDER log lines survive — intentional re-uploads. - // Decision lives in lib/queue-dedup.js (Node-tested, see tests/queue-dedup.test.js) - // so it can't silently regress to nuking the whole restored queue on restart. - const { kept, removed } = window.QueueDedup.partitionRestoredJobsByLog(queueJobs, entries, _restoredSnapshotSavedAt); + const [entries, completionRows] = await Promise.all([ + window.api.readOwnUploadLog(), + typeof window.api.getAutomationCompletions === 'function' ? window.api.getAutomationCompletions() : Promise.resolve([]) + ]); + const ledgerCandidates = queueJobs.map(job => ({ + path: job.file, + size: job.sourceSize ?? job.automationSize ?? job.bytesTotal, + mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs, + eligibleHosters: [job.hoster] + })); + const ledgerState = window.AutomationControl.classifyAutomationCompletionLedger({ + candidates: ledgerCandidates, + completionRows + }); + const ledgerRemovedIds = new Set(); + for (let index = 0; index < ledgerState.remainingByPath.length; index++) { + if (ledgerState.remainingByPath[index].hosters.length === 0 && queueJobs[index]?.id) ledgerRemovedIds.add(queueJobs[index].id); + } + const removed = []; + if (ledgerRemovedIds.size > 0) { + queueJobs = queueJobs.filter(job => { + if (!ledgerRemovedIds.has(job.id)) return true; + removed.push(job); + return false; + }); + } + if (Array.isArray(entries) && entries.length > 0) { + const partitioned = window.QueueDedup.partitionRestoredJobsByLog(queueJobs, entries, _restoredSnapshotSavedAt); + queueJobs = partitioned.kept; + removed.push(...partitioned.removed); + } + for (const job of removed) { + if (job.file && job.hoster) _completedUploadKeys.add(`${job.file}|${job.hoster}`); + } + const selectedLedger = window.AutomationControl.classifyAutomationCompletionLedger({ + candidates: selectedFiles.map(file => ({ + path: file.path, + size: file.size, + mtimeMs: file.mtimeMs, + eligibleHosters: getSelectedHosters() + })), + completionRows + }); + for (const entry of selectedLedger.completedByPath) { + for (const hoster of entry.hosters) _completedUploadKeys.add(`${entry.path}|${hoster}`); + } if (removed.length > 0) { - queueJobs = kept; - for (const job of removed) { - if (job.file && job.hoster) _completedUploadKeys.add(`${job.file}|${job.hoster}`); - } rebuildJobIndex(); syncSelectedFilesFromQueue(); - window.api.debugLog(`auto-dedup: removed ${removed.length} already-uploaded (done) jobs from restored queue (${entries.length} log entries)`); + window.api.debugLog(`auto-dedup: removed ${removed.length} completed jobs from restored queue`); } - const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt); - if (seedKeys.length > 0) { - for (const k of seedKeys) _completedUploadKeys.add(k); - window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`); + if (Array.isArray(entries) && entries.length > 0) { + const seedKeys = window.QueueDedup.completedSelectionKeys(selectedFiles, getSelectedHosters(), entries, _restoredSnapshotSavedAt); + if (seedKeys.length > 0) { + for (const key of seedKeys) _completedUploadKeys.add(key); + window.api.debugLog(`auto-dedup: seeded ${seedKeys.length} completed file|hoster keys from log so buildQueuePreview won't re-create ghosts`); + } } - } catch {} + _startupQueueEvidenceAvailable = true; + return true; + } catch (error) { + _startupQueueEvidenceAvailable = false; + window.api.debugLog(`startup completion evidence failed: ${error?.message || String(error)}`); + return false; + } } // --- Log import: remove already-uploaded file+hoster combos from queue --- @@ -9739,6 +9841,47 @@ async function importUploadLog() { logKeys.add(`${entry.fileName.toLowerCase()}|${entry.hoster.toLowerCase()}`); } + const hasFiniteMetadata = value => value !== null && value !== undefined && value !== '' && Number.isFinite(Number(value)); + const matchingJobs = queueJobs.filter(job => { + const key = `${job.fileName.toLowerCase()}|${job.hoster.toLowerCase()}`; + return logKeys.has(key) && job.status !== 'done'; + }); + const missingMetadata = matchingJobs.filter(job => { + const size = job.sourceSize ?? job.automationSize ?? job.bytesTotal; + const mtimeMs = job.sourceMtimeMs ?? job.automationMtimeMs; + return !hasFiniteMetadata(size) || !hasFiniteMetadata(mtimeMs); + }); + const inspectedMetadata = new Map(); + if (missingMetadata.length > 0) { + const inspection = await window.api.inspectImportFiles(missingMetadata.map(job => ({ path: job.file, name: job.fileName })), []); + for (const file of Array.isArray(inspection?.accepted) ? inspection.accepted : []) { + inspectedMetadata.set(normalizeAutomationPath(file.path), file); + } + } + const completionRows = matchingJobs.map(job => { + const inspected = inspectedMetadata.get(normalizeAutomationPath(job.file)); + return { + path: job.file, + hoster: job.hoster, + size: job.sourceSize ?? job.automationSize ?? inspected?.size ?? job.bytesTotal, + mtimeMs: job.sourceMtimeMs ?? job.automationMtimeMs ?? inspected?.mtimeMs, + completedAt: Date.now() + }; + }).filter(row => hasFiniteMetadata(row.size) && hasFiniteMetadata(row.mtimeMs)); + if (completionRows.length !== matchingJobs.length) { + showCopyToast('Abschlussnachweise konnten nicht vollständig ermittelt werden.', 7000); + return; + } + if (completionRows.length > 0 && typeof window.api.recordAutomationCompletions === 'function') { + try { + await window.api.recordAutomationCompletions(completionRows); + invalidateAutomationEvidenceSnapshot(); + } catch { + showCopyToast('Abschlussnachweise konnten nicht gespeichert werden.', 7000); + return; + } + } + // Find queue jobs that match (already uploaded) let removed = 0; queueJobs = queueJobs.filter(job => { diff --git a/renderer/i18n.js b/renderer/i18n.js index 647c6e7..577c8e9 100644 --- a/renderer/i18n.js +++ b/renderer/i18n.js @@ -704,6 +704,21 @@ ['Updateprüfung fehlgeschlagen', 'Update check failed'], ['Upload läuft...', 'Uploading...'], ['Upload-Log', 'Upload log'], + ['Automatik-Abschlussnachweis konnte nicht gespeichert werden', 'Automation completion evidence could not be saved'], + ['Lokale Speicherung', 'Local persistence'], + ['Automatik-Abschlussdatei ist ungültig', 'Automation completion file is invalid'], + ['Automatik-Abschlussdatei enthält zu viele Einträge', 'Automation completion file contains too many entries'], + ['Automatik-Abschlussnachweise sind ungültig', 'Automation completion evidence is invalid'], + ['Abschlussnachweise konnten nicht gespeichert werden.', 'Completion evidence could not be saved.'], + ['Abschlussnachweise konnten nicht vollständig ermittelt werden.', 'Completion evidence could not be determined completely.'], + ['Automatische Wiederaufnahme wurde wegen nicht verfügbarer Abschlussnachweise blockiert.', 'Automatic resume was blocked because completion evidence is unavailable.'], + ['Upload-Log-Verzeichnis enthält zu viele Einträge', 'Upload log directory contains too many entries'], + ['Zu viele verwaltete Upload-Logs', 'Too many managed upload logs'], + ['Upload-Logs überschreiten das Leselimit', 'Upload logs exceed the read limit'], + ['Upload-Log überschreitet das Leselimit', 'Upload log exceeds the read limit'], + ['Upload-Log enthält zu viele eindeutige Einträge', 'Upload log contains too many unique entries'], + ['Upload-Log-Stream ist nicht verfügbar', 'Upload log stream is unavailable'], + ['Upload-Log-Zeile ist zu lang', 'Upload log line is too long'], ['Upload-Status', 'Upload status'], ['Upload-Übersicht', 'Upload overview'], ['Verlauf als CSV exportieren?\n\nOK = CSV\nAbbrechen = JSON', 'Export history as CSV?\n\nOK = CSV\nCancel = JSON'], diff --git a/tests/automation-control.test.js b/tests/automation-control.test.js index 67b52c8..789913f 100644 --- a/tests/automation-control.test.js +++ b/tests/automation-control.test.js @@ -7,7 +7,12 @@ const { rollDailyTelemetry, applyTelemetryDelta, deriveAutomationState, - classifyProcessedCandidates + isPathWithinAutomationFolder, + classifyProcessedCandidates, + classifyAutomationCompletionLedger, + createAutomationCompletionWriter, + mergeAutomationCompletions, + removeAutomationCompletions } = require('../lib/automation-control'); test('automation defaults use 15000 jobs and a five minute reconciliation interval', () => { @@ -284,6 +289,14 @@ test('automation state follows inactive disconnected error queue-limited and act assert.equal(deriveAutomationState(null), 'inactive'); }); +test('watched-folder membership respects Windows casing and recursive scope', () => { + assert.equal(isPathWithinAutomationFolder('C:\\Watch\\episode.mkv', 'c:/watch', false), true); + assert.equal(isPathWithinAutomationFolder('C:\\Watch\\Season 1\\episode.mkv', 'c:/watch', false), false); + assert.equal(isPathWithinAutomationFolder('C:\\Watch\\Season 1\\episode.mkv', 'c:/watch', true), true); + assert.equal(isPathWithinAutomationFolder('C:\\Watcher\\episode.mkv', 'c:/watch', true), false); + assert.equal(isPathWithinAutomationFolder('', 'c:/watch', true), false); +}); + test('exact queue and history paths mark candidates processed case-insensitively', () => { const result = classifyProcessedCandidates({ candidates: [ @@ -345,3 +358,96 @@ test('processed classification tolerates malformed collections and does not muta unprocessedPaths: [] }); }); + +test('durable completion ledger excludes only unchanged completed hosters', () => { + const candidate = { + path: 'C:\\Watch\\Episode.mkv', + size: 1048576, + mtimeMs: 1787828400123.75, + eligibleHosters: ['doodstream.com', 'voe.sx', 'byse.sx'] + }; + const completionRows = [ + { path: 'c:/watch/episode.mkv', size: 1048576, mtimeMs: 1787828400123, hoster: 'DOODSTREAM.COM', completedAt: 10 }, + { path: 'C:\\WATCH\\EPISODE.MKV', size: 1048576, mtimeMs: 1787828400123.9, hoster: 'voe.sx', completedAt: 20 } + ]; + + assert.deepEqual(classifyAutomationCompletionLedger({ candidates: [candidate], completionRows }), { + processedPaths: [], + completedByPath: [{ path: candidate.path, hosters: ['doodstream.com', 'voe.sx'] }], + remainingByPath: [{ path: candidate.path, hosters: ['byse.sx'] }] + }); + + const complete = classifyAutomationCompletionLedger({ + candidates: [candidate], + completionRows: completionRows.concat({ + path: candidate.path, + size: candidate.size, + mtimeMs: candidate.mtimeMs, + hoster: 'byse.sx', + completedAt: 30 + }) + }); + assert.deepEqual(complete.processedPaths, [candidate.path]); + assert.deepEqual(complete.remainingByPath, [{ path: candidate.path, hosters: [] }]); + + const changed = classifyAutomationCompletionLedger({ + candidates: [{ ...candidate, mtimeMs: candidate.mtimeMs + 1 }], + completionRows + }); + assert.deepEqual(changed.completedByPath, [{ path: candidate.path, hosters: [] }]); + assert.deepEqual(changed.remainingByPath, [{ path: candidate.path, hosters: candidate.eligibleHosters }]); +}); + +test('completion ledger replaces only the same path and hoster without evicting unrelated entries', () => { + const existing = [ + { path: 'C:\\watch\\a.mkv', size: 1, mtimeMs: 1, hoster: 'voe.sx', completedAt: 10 }, + { path: 'C:\\watch\\b.mkv', size: 2, mtimeMs: 2, hoster: 'voe.sx', completedAt: 20 } + ]; + const merged = mergeAutomationCompletions(existing, [ + { path: 'c:/WATCH/a.mkv', size: 3, mtimeMs: 3, hoster: 'VOE.SX', completedAt: 30 }, + { path: 'C:\\watch\\c.mkv', size: 4, mtimeMs: 4, hoster: 'byse.sx', completedAt: 40 } + ]); + + assert.deepEqual(merged, [ + { path: 'C:\\watch\\b.mkv', size: 2, mtimeMs: 2, hoster: 'voe.sx', completedAt: 20 }, + { path: 'c:/WATCH/a.mkv', size: 3, mtimeMs: 3, hoster: 'voe.sx', completedAt: 30 }, + { path: 'C:\\watch\\c.mkv', size: 4, mtimeMs: 4, hoster: 'byse.sx', completedAt: 40 } + ]); + assert.deepEqual(removeAutomationCompletions(merged, [{ path: 'C:\\WATCH\\A.MKV', hoster: 'voe.sx' }]), [merged[0], merged[2]]); + assert.deepEqual(removeAutomationCompletions(merged, [{ path: 'c:/watch/c.mkv' }]), [merged[0], merged[1]]); + assert.throws(() => mergeAutomationCompletions(existing, [ + { path: 'C:\\watch\\c.mkv', size: 4, mtimeMs: 4, hoster: 'byse.sx', completedAt: 40 } + ], 2), /zu viele Einträge/); +}); + +test('completion writer coalesces successful jobs and retains a failed batch for retry', async () => { + const scheduled = []; + const writes = []; + const persisted = []; + const failures = []; + let fail = true; + const writer = createAutomationCompletionWriter({ + schedule: callback => scheduled.push(callback), + onPersisted: entries => persisted.push(structuredClone(entries)), + onError: (error, entries) => failures.push({ message: error.message, entries: structuredClone(entries) }), + save: async entries => { + if (fail) throw new Error('disk unavailable'); + writes.push(structuredClone(entries)); + } + }); + const first = { path: 'C:\\watch\\episode.mkv', size: 1, mtimeMs: 2, hoster: 'voe.sx', completedAt: 3 }; + const newer = { ...first, completedAt: 4 }; + + writer.add(first); + writer.add(newer); + assert.equal(scheduled.length, 1); + await assert.rejects(writer.flush(), /disk unavailable/); + assert.deepEqual(persisted, []); + assert.deepEqual(failures, [{ message: 'disk unavailable', entries: [newer] }]); + + fail = false; + await writer.flush(); + assert.deepEqual(writes, [[newer]]); + assert.deepEqual(persisted, [[newer]]); + assert.equal(writer.pendingCount(), 0); +}); diff --git a/tests/config-store.test.js b/tests/config-store.test.js index 74001e8..0a43c30 100644 --- a/tests/config-store.test.js +++ b/tests/config-store.test.js @@ -31,6 +31,7 @@ function createStore() { store = new ConfigStore(fakeApp); store.filePath = path.join(tmpDir, 'electron-config.json'); store.historyPath = path.join(tmpDir, 'electron-history.json'); + store.automationCompletionPath = path.join(tmpDir, 'automation-completions.json'); return store; } @@ -41,6 +42,7 @@ function createStoreAt(filePath) { }); configuredStore.filePath = filePath; configuredStore.historyPath = path.join(path.dirname(filePath), 'electron-history.json'); + configuredStore.automationCompletionPath = path.join(path.dirname(filePath), 'automation-completions.json'); return configuredStore; } @@ -125,6 +127,74 @@ describe('ConfigStore', () => { assert.equal(reloaded.globalSettings.folderMonitor.pausedAt, 1787712000000); }); + it('automation completions survive queue clearing and can be removed explicitly', async () => { + const completion = { + path: 'C:\\watch\\episode.mkv', + size: 1024, + mtimeMs: 1787828400123, + hoster: 'doodstream.com', + completedAt: 1787828500000 + }; + + await store.saveAutomationCompletions([completion]); + await store.savePendingQueue(null); + await store.appendHistory({ id: 'old', timestamp: '2026-01-01T00:00:00.000Z', files: [] }); + await store.clearHistory(); + + const reloaded = createStoreAt(store.filePath); + assert.deepEqual(await reloaded.loadAutomationCompletions(), [completion]); + + await reloaded.clearAutomationCompletions([{ path: completion.path, hoster: completion.hoster }]); + assert.deepEqual(await reloaded.loadAutomationCompletions(), []); + assert.equal(JSON.parse(fs.readFileSync(reloaded.automationCompletionPath, 'utf8')).version, 1); + }); + + it('automation completion writes remain serialized without evicting older unique paths', async () => { + const first = store.saveAutomationCompletions([ + { path: 'C:\\watch\\old.mkv', size: 1, mtimeMs: 1, hoster: 'voe.sx', completedAt: 1 } + ], { maxEntries: 2 }); + const second = store.saveAutomationCompletions([ + { path: 'C:\\watch\\middle.mkv', size: 2, mtimeMs: 2, hoster: 'voe.sx', completedAt: 2 }, + { path: 'C:\\watch\\new.mkv', size: 3, mtimeMs: 3, hoster: 'byse.sx', completedAt: 3 } + ], { maxEntries: 2 }); + + await Promise.all([first, second]); + await store.drainAutomationCompletionWrites(); + + assert.deepEqual((await store.loadAutomationCompletions()).map(entry => entry.path), [ + 'C:\\watch\\old.mkv', + 'C:\\watch\\middle.mkv', + 'C:\\watch\\new.mkv' + ]); + }); + + it('corrupted automation completion evidence fails closed', async () => { + fs.writeFileSync(store.automationCompletionPath, '{broken', 'utf8'); + await assert.rejects(store.loadAutomationCompletions()); + const reloaded = createStoreAt(store.filePath); + fs.writeFileSync(reloaded.automationCompletionPath, JSON.stringify({ version: 1, entries: [{ path: 'C:\\watch\\invalid.mkv' }] }), 'utf8'); + await assert.rejects(reloaded.loadAutomationCompletions(), /ungültig/); + }); + + it('automation completion drain waits for an active durable write', async () => { + const originalWrite = store._writeAutomationCompletionFile.bind(store); + let releaseWrite; + store._writeAutomationCompletionFile = entries => new Promise((resolve, reject) => { + releaseWrite = () => originalWrite(entries).then(resolve, reject); + }); + const saving = store.saveAutomationCompletions([ + { path: 'C:\\watch\\drain.mkv', size: 1, mtimeMs: 2, hoster: 'voe.sx', completedAt: 3 } + ]); + while (!releaseWrite) await new Promise(resolve => setImmediate(resolve)); + let drained = false; + const draining = store.drainAutomationCompletionWrites().then(() => { drained = true; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(drained, false); + releaseWrite(); + await Promise.all([saving, draining]); + assert.equal(drained, true); + }); + it('drops the retired plaintext credential setting from legacy configurations', () => { fs.writeFileSync(store.filePath, JSON.stringify({ hosters: {}, diff --git a/tests/import-preflight.test.js b/tests/import-preflight.test.js index 3f3810f..3c373d1 100644 --- a/tests/import-preflight.test.js +++ b/tests/import-preflight.test.js @@ -13,7 +13,7 @@ test('inspects duplicates, unavailable files, accepted files, and configured siz ], { existingPaths: ['C:\\queue\\duplicate.mkv'], inspectPath: async filePath => { - if (filePath.endsWith('accepted.mkv')) return { exists: true, readable: true, size: 2 * 1024 * 1024 }; + if (filePath.endsWith('accepted.mkv')) return { exists: true, readable: true, size: 2 * 1024 * 1024, mtimeMs: 1787828400123 }; if (filePath.endsWith('empty.mkv')) return { exists: true, readable: true, size: 0 }; return { exists: false }; } @@ -42,6 +42,7 @@ test('inspects duplicates, unavailable files, accepted files, and configured siz jobCount: 1, sizeLimitedJobCount: 1 }); + assert.equal(inspection.accepted[0].mtimeMs, 1787828400123); }); test('connects the import preflight through the main process, preload, renderer, and hoster dialog', () => { diff --git a/tests/log-mode.test.js b/tests/log-mode.test.js index 0899f3d..5d943cc 100644 --- a/tests/log-mode.test.js +++ b/tests/log-mode.test.js @@ -92,8 +92,11 @@ test('managed upload-log discovery includes session logs and excludes unrelated assert.equal(typeof isManagedUploadLogFileName, 'function'); const options = { baseName: 'fileuploader', ext: '.log' }; assert.equal(isManagedUploadLogFileName('fileuploader.log', options), true); + assert.equal(isManagedUploadLogFileName('fileuploader.1.log', options), true); assert.equal(isManagedUploadLogFileName('fileuploader-2026-08-27.log', options), true); + assert.equal(isManagedUploadLogFileName('fileuploader-2026-08-27.2.log', options), true); assert.equal(isManagedUploadLogFileName('fileuploader-session-2026-08-27_05-40-59-1234.log', options), true); + assert.equal(isManagedUploadLogFileName('27-08-2026-mdu-session-05-40-111111.3.log', options), true); assert.equal(isManagedUploadLogFileName('27-08-2026-mdu-session-05-40-599797.log', options), true); assert.equal(isManagedUploadLogFileName('FILEUPLOADER-2026-08-27.LOG', options), true); assert.equal(isManagedUploadLogFileName('27-08-2026-MDU-SESSION-05-40-599797.LOG', options), true); diff --git a/tests/package-build-files.test.js b/tests/package-build-files.test.js index d002351..0e18427 100644 --- a/tests/package-build-files.test.js +++ b/tests/package-build-files.test.js @@ -224,7 +224,7 @@ test('packages every Electron preload referenced by the main process', () => { test('read-own-upload-log discovers base daily session and both fallback directories without synchronous reads', async () => { const mainSource = fs.readFileSync(path.join(projectRoot, 'main.js'), 'utf8'); - const blockStart = mainSource.indexOf("ipcMain.handle('read-own-upload-log'"); + const blockStart = mainSource.indexOf('let _uploadLogEvidenceCache'); const blockEnd = mainSource.indexOf("\nipcMain.handle('import-upload-log'", blockStart); assert.notEqual(blockStart, -1); assert.notEqual(blockEnd, -1); @@ -233,9 +233,9 @@ test('read-own-upload-log discovers base daily session and both fallback directo const desktop = 'C:\\desktop'; const userData = 'C:\\user-data'; const entriesByDirectory = new Map([ - [configured, ['custom.txt', 'custom-2026-08-27.txt', '27-08-2026-mdu-session-05-40-111111.txt', 'upload-audit.log']], - [desktop, ['FILEUPLOADER-2026-08-26.LOG', '26-08-2026-MDU-SESSION-05-40-222222.LOG', 'account-rotation.log']], - [userData, ['fileuploader-2026-08-25.log', '25-08-2026-mdu-session-05-40-333333.log', 'upload-debug.log']] + [configured, ['custom.txt', 'custom-2026-08-27.txt', 'custom.1.txt', '27-08-2026-mdu-session-05-40-111111.txt', 'upload-audit.log']], + [desktop, ['FILEUPLOADER-2026-08-26.LOG', 'FILEUPLOADER-2026-08-26.2.LOG', '26-08-2026-MDU-SESSION-05-40-222222.LOG', 'account-rotation.log']], + [userData, ['fileuploader-2026-08-25.log', 'fileuploader.3.log', '25-08-2026-mdu-session-05-40-333333.log', 'upload-debug.log']] ]); const fileNames = new Map(); for (const [directory, names] of entriesByDirectory) { @@ -244,33 +244,84 @@ test('read-own-upload-log discovers base daily session and both fallback directo fileNames.set(path.win32.join(directory, name), `${name}.mkv`); } } + let streamReads = 0; + let failedPath = ''; + let scanLabel = ''; + let holdNextRead = false; + let heldRead = null; const fakeFs = { - readdirSync: directory => entriesByDirectory.get(directory) || [], - existsSync: filePath => fileNames.has(filePath), + readdirSync: () => { throw new Error('synchronous enumeration forbidden'); }, + existsSync: () => { throw new Error('synchronous existence check forbidden'); }, readFileSync: () => { throw new Error('synchronous read forbidden'); }, promises: { - readFile: async filePath => require('../lib/upload-log').formatUploadLogLine( - new Date(2026, 7, 27, 5, 40, 0), - 'voe.sx', - 'https://voe.sx/e/test', - fileNames.get(filePath) - ) + readdir: async () => { throw new Error('materialized directory read forbidden'); }, + opendir: async directory => ({ + async *[Symbol.asyncIterator]() { + for (const name of entriesByDirectory.get(directory) || []) yield { name }; + } + }), + readFile: async () => { throw new Error('full-file read forbidden'); } } }; - vm.runInNewContext(mainSource.slice(blockStart, blockEnd), { - _resolveUploadLogTarget: () => ({ path: path.win32.join(configured, '27-08-2026-mdu-session-05-40-111111.txt') }), + const context = { + _activeLogPath: path.win32.join(configured, '27-08-2026-mdu-session-05-40-111111.txt'), + _resolveUploadLogTarget: () => { throw new Error('write-target resolution forbidden'); }, app: { getPath: name => name === 'desktop' ? desktop : userData }, fs: fakeFs, getBaseLogFilePath: () => path.win32.join(configured, 'custom.txt'), - getSafeDesktopDir: () => desktop, + getSafeDesktopDir: () => { throw new Error('synchronous desktop probe forbidden'); }, ipcMain: { handle: (channel, handler) => handlers.set(channel, handler) }, isManagedUploadLogFileName: require('../lib/log-mode').isManagedUploadLogFileName, - parseUploadLogLine: require('../lib/upload-log').parseUploadLogLine, + iterateUploadLogEntries: async function* (filePath) { + streamReads++; + const label = scanLabel; + if (filePath === failedPath) { + const error = new Error('managed log denied'); + error.code = 'EACCES'; + throw error; + } + if (holdNextRead) { + holdNextRead = false; + heldRead = createDeferred(); + await heldRead.promise; + } + yield require('../lib/upload-log').parseUploadLogLine(require('../lib/upload-log').formatUploadLogLine( + new Date(2026, 7, 27, 5, 40, 0), + 'voe.sx', + 'https://voe.sx/e/test', + `${fileNames.get(filePath)}${label}` + )); + }, path: path.win32 - }); + }; + vm.runInNewContext(mainSource.slice(blockStart, blockEnd), context); - const entries = await handlers.get('read-own-upload-log')(); - assert.deepEqual([...entries.map(entry => entry.fileName)].sort(), [...fileNames.values()].sort()); + const handler = handlers.get('read-own-upload-log'); + const [first, concurrent] = await Promise.all([handler(), handler()]); + const cached = await handler(); + const expected = [...fileNames.values()].sort(); + assert.deepEqual([...first.map(entry => entry.fileName)].sort(), expected); + assert.deepEqual([...concurrent.map(entry => entry.fileName)].sort(), expected); + assert.deepEqual([...cached.map(entry => entry.fileName)].sort(), expected); + assert.equal(streamReads, fileNames.size); + vm.runInNewContext('_invalidateUploadLogEvidenceCache()', context); + failedPath = [...fileNames.keys()][0]; + await assert.rejects(handler(), /managed log denied/); + failedPath = ''; + vm.runInNewContext('_invalidateUploadLogEvidenceCache()', context); + holdNextRead = true; + scanLabel = '.old'; + const staleScan = handler(); + while (!heldRead) await new Promise(resolve => setImmediate(resolve)); + vm.runInNewContext('_invalidateUploadLogEvidenceCache()', context); + scanLabel = '.new'; + const freshScan = handler(); + heldRead.resolve(); + const [staleEntries, freshEntries] = await Promise.all([staleScan, freshScan]); + const cachedFreshEntries = await handler(); + assert.equal(staleEntries.some(entry => entry.fileName.endsWith('.old')), true); + assert.equal(freshEntries.every(entry => entry.fileName.endsWith('.new')), true); + assert.equal(cachedFreshEntries.every(entry => entry.fileName.endsWith('.new')), true); }); test('exposes managed online backup operations through narrow IPC boundaries', () => { @@ -633,20 +684,47 @@ test('preload exposes account cooldown snapshots and removes their listener duri test('exposes persistent automation controls and status through narrow IPC boundaries', () => { const preloadSource = fs.readFileSync(path.join(projectRoot, 'preload.js'), 'utf8'); const mainSource = fs.readFileSync(path.join(projectRoot, 'main.js'), 'utf8'); + const rendererSource = fs.readFileSync(path.join(projectRoot, 'renderer', 'app.js'), 'utf8'); assert.match(mainSource, /ipcMain\.handle\('automation:get-status'/u); + assert.match(mainSource, /ipcMain\.handle\('automation:get-completions'/u); + assert.match(mainSource, /ipcMain\.handle\('automation:record-completions'/u); assert.match(mainSource, /ipcMain\.handle\('automation:pause-after-active'/u); assert.match(mainSource, /ipcMain\.handle\('automation:resume'/u); assert.match(mainSource, /ipcMain\.handle\('folder-monitor:test-scan'/u); assert.match(mainSource, /ipcMain\.handle\('folder-monitor:reconcile'/u); assert.match(mainSource, /safeSend\('automation:status'/u); assert.match(preloadSource, /automationGetStatus:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:get-status'\)/u); + assert.match(preloadSource, /getAutomationCompletions:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:get-completions'\)/u); + assert.match(preloadSource, /recordAutomationCompletions:\s*\(entries\)\s*=>\s*ipcRenderer\.invoke\('automation:record-completions',\s*entries\)/u); assert.match(preloadSource, /automationPauseAfterActive:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:pause-after-active'\)/u); assert.match(preloadSource, /automationResume:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('automation:resume'\)/u); assert.match(preloadSource, /folderMonitorTestScan:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('folder-monitor:test-scan'\)/u); assert.match(preloadSource, /folderMonitorReconcile:\s*\(\)\s*=>\s*ipcRenderer\.invoke\('folder-monitor:reconcile'\)/u); assert.match(preloadSource, /onAutomationStatus:\s*\(callback\)\s*=>\s*\{[\s\S]*?ipcRenderer\.on\('automation:status'/u); assert.match(preloadSource, /ipcRenderer\.removeAllListeners\('automation:status'\)/u); + assert.match(mainSource, /async function registerAutomationCompletionJobs/u); + assert.match(mainSource, /await fs\.promises\.stat\(job\.file\)/u); + assert.match(mainSource, /await registerAutomationCompletionJobs\(_thisManager,\s*jobs\)/u); + assert.match(mainSource, /await registerAutomationCompletionJobs\(batchManager,\s*jobs\)/u); + assert.match(mainSource, /_automationCompletionProgress\.set\(automationCompletionKey\(entry\),\s*data\)/u); + assert.match(mainSource, /_automationCompletionWriter\.add\(entry\)/u); + assert.match(mainSource, /await _thisManager\._automationCompletionWriter\?\.flush\(\)/u); + assert.match(mainSource, /requestUploadFinalization\(summary,\s*!automationCompletionsPersisted\)/u); + assert.match(rendererSource, /data\.preserveQueue\s*===\s*true\s*\|\|\s*queueJobs\.some/u); + assert.match(rendererSource, /await window\.api\.recordAutomationCompletions\(completionRows\)/u); + const serializerStart = rendererSource.indexOf('function serializeUploadJob'); + const serializerEnd = rendererSource.indexOf('\n}', serializerStart); + const serializer = rendererSource.slice(serializerStart, serializerEnd); + assert.match(serializer, /automationAdmission:\s*job\.automationAdmission\s*===\s*true/u); + assert.match(serializer, /automationMtimeMs:\s*job\.automationMtimeMs/u); + assert.match(serializer, /automationSize:\s*job\.automationSize/u); + assert.match(serializer, /sourceMtimeMs:\s*job\.sourceMtimeMs/u); + assert.match(serializer, /sourceSize:\s*job\.sourceSize/u); + const syncStart = rendererSource.indexOf('function syncSelectedFilesFromQueue'); + const syncEnd = rendererSource.indexOf('\n}', syncStart); + const syncSelected = rendererSource.slice(syncStart, syncEnd); + assert.match(syncSelected, /mtimeMs:\s*job\.sourceMtimeMs\s*\?\?\s*job\.automationMtimeMs/u); }); test('every batch start and extension IPC fails closed before account and cleanup side effects', () => { diff --git a/tests/startup-renderer.test.js b/tests/startup-renderer.test.js index a573b55..4cefc4a 100644 --- a/tests/startup-renderer.test.js +++ b/tests/startup-renderer.test.js @@ -69,6 +69,8 @@ let pendingAutomationTestScan = null; let automationProbe = { history: [], uploadLog: [], + completionRows: [], + completionError: '', paused: false, runtimeStatus: {}, automationStatusSequence: [], @@ -85,7 +87,7 @@ let automationProbe = { activeInspections: 0, maxConcurrentInspections: 0, dryScan: { files: [], reachable: true, trigger: 'test' }, - readCalls: { history: 0, uploadLog: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 }, + readCalls: { history: 0, uploadLog: 0, completions: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 }, mutationCalls: [], logs: [], savedSettings: [] @@ -175,6 +177,8 @@ contextBridge.exposeInMainWorld('api', { automationProbe = { history: Array.isArray(value.history) ? value.history : [], uploadLog: Array.isArray(value.uploadLog) ? value.uploadLog : [], + completionRows: Array.isArray(value.completionRows) ? value.completionRows : [], + completionError: String(value.completionError || ''), paused: value.paused === true, runtimeStatus: value.runtimeStatus && typeof value.runtimeStatus === 'object' ? { ...value.runtimeStatus } : {}, automationStatusSequence: Array.isArray(value.automationStatusSequence) ? value.automationStatusSequence.map(entry => ({ ...entry })) : [], @@ -191,7 +195,7 @@ contextBridge.exposeInMainWorld('api', { activeInspections: 0, maxConcurrentInspections: 0, dryScan: value.dryScan || { files: [], reachable: true, trigger: 'test' }, - readCalls: { history: 0, uploadLog: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 }, + readCalls: { history: 0, uploadLog: 0, completions: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 }, mutationCalls: [], logs: [], savedSettings: [] @@ -200,6 +204,7 @@ contextBridge.exposeInMainWorld('api', { setAutomationEvidence(value = {}) { if (Array.isArray(value.history)) automationProbe.history = value.history; if (Array.isArray(value.uploadLog)) automationProbe.uploadLog = value.uploadLog; + if (Array.isArray(value.completionRows)) automationProbe.completionRows = value.completionRows; }, getAutomationProbeState() { return { @@ -251,6 +256,15 @@ contextBridge.exposeInMainWorld('api', { automationProbe.readCalls.uploadLog++; return Promise.resolve(automationProbe.uploadLog); }, + getAutomationCompletions() { + automationProbe.readCalls.completions++; + if (automationProbe.completionError) return Promise.reject(new Error(automationProbe.completionError)); + return Promise.resolve(automationProbe.completionRows); + }, + clearAutomationCompletions(removals) { + automationProbe.mutationCalls.push(['clear-completions', JSON.parse(JSON.stringify(removals || []))]); + return Promise.resolve(true); + }, automationGetStatus() { automationProbe.readCalls.status++; if (automationProbe.automationStatusSequence.length > 0) return Promise.resolve(automationProbe.automationStatusSequence.shift()); @@ -686,7 +700,7 @@ contextBridge.exposeInMainWorld('api', { resultingJobs: historyEvaluation.summary.resultingJobs }; _completedUploadKeys.clear(); - const completedFile = { path: 'C:\\history\\completed-in-session.mkv', name: 'completed-in-session.mkv', size: 1 }; + const completedFile = { path: 'C:\\history\\completed-in-session.mkv', name: 'completed-in-session.mkv', size: 1, mtimeMs: 1787828400123 }; config.globalSettings.removeFromQueueOnDone = true; config.globalSettings.folderMonitor = { enabled: true, @@ -703,6 +717,7 @@ contextBridge.exposeInMainWorld('api', { hoster: 'doodstream.com', status: 'queued', bytesTotal: 1, + automationMtimeMs: completedFile.mtimeMs, automationAdmission: true }; queueJobs = [completedJob]; @@ -721,18 +736,127 @@ contextBridge.exposeInMainWorld('api', { result: { download_url: 'https://doodstream.com/d/completed-in-session' } }); _doneRemovalCoalescer?.drainSync(); + window.api.setAutomationEvidence({ + completionRows: [{ + path: completedFile.path, + size: completedFile.size, + mtimeMs: completedFile.mtimeMs, + hoster: completedJob.hoster, + completedAt: 1787828500000 + }] + }); automationEvidenceSnapshotGeneration++; automationEvidenceSnapshotCache = null; const removedAfterDone = !queueJobs.some(job => job.id === completedJob.id); const completedKeyPresent = _completedUploadKeys.has(completedFile.path + '|doodstream.com'); const completedResult = await handleFolderMonitorFiles([completedFile]); const completedProbe = await window.api.getAutomationProbeState(); + _completedUploadKeys.clear(); + queueJobs = []; + rebuildJobIndex(); + window.api.configureAutomationProbe({ + paused: false, + history: [], + uploadLog: [], + completionRows: [{ + path: completedFile.path, + size: completedFile.size, + mtimeMs: completedFile.mtimeMs, + hoster: 'doodstream.com', + completedAt: 1787828500000 + }] + }); + automationEvidenceSnapshotGeneration++; + automationEvidenceSnapshotCache = null; + const durableEvaluation = await evaluateAutomationCandidates([completedFile], { dryRun: true, trigger: 'startup' }); + _completedUploadKeys.add(completedFile.path + '|doodstream.com'); + automationEvidenceSnapshotGeneration++; + automationEvidenceSnapshotCache = null; + const changedEvaluation = await evaluateAutomationCandidates([{ ...completedFile, mtimeMs: completedFile.mtimeMs + 1 }], { dryRun: true, trigger: 'startup' }); + const partialFile = { path: 'C:\\history\\partial-in-session.mkv', name: 'partial-in-session.mkv', size: 2, mtimeMs: 1787828400456 }; + config.globalSettings.folderMonitor.hosters = ['doodstream.com', 'voe.sx']; + window.api.configureAutomationProbe({ + paused: false, + history: [], + uploadLog: [{ fileName: partialFile.name, hoster: 'doodstream.com' }], + completionRows: [{ + path: partialFile.path, + size: partialFile.size, + mtimeMs: partialFile.mtimeMs, + hoster: 'doodstream.com', + completedAt: 1787828500001 + }] + }); + automationEvidenceSnapshotGeneration++; + automationEvidenceSnapshotCache = null; + const partialEvaluation = await evaluateAutomationCandidates([partialFile], { dryRun: true, trigger: 'startup' }); + const restoredFile = { path: 'C:\\history\\restored-after-ledger.mkv', name: 'restored-after-ledger.mkv', size: 3, mtimeMs: 1787828400789 }; + const restoredJob = { + id: 'restored-after-ledger', + file: restoredFile.path, + fileName: restoredFile.name, + hoster: 'doodstream.com', + status: 'preview', + bytesTotal: restoredFile.size, + sourceSize: restoredFile.size, + sourceMtimeMs: restoredFile.mtimeMs, + automationAdmission: true + }; + queueJobs = [restoredJob]; + selectedFiles = []; + rebuildJobIndex(); + _completedUploadKeys.clear(); + window.api.configureAutomationProbe({ + paused: false, + history: [], + uploadLog: [], + completionRows: [{ + path: restoredFile.path, + size: restoredFile.size, + mtimeMs: restoredFile.mtimeMs, + hoster: restoredJob.hoster, + completedAt: 1787828500002 + }] + }); + await _autoDeduplicateFromLog(); + queueJobs = [{ + id: 'blocked-restored-evidence', + file: 'C:\\history\\blocked-restored-evidence.mkv', + fileName: 'blocked-restored-evidence.mkv', + hoster: 'doodstream.com', + status: 'preview', + bytesTotal: 4 + }]; + selectedFiles = []; + rebuildJobIndex(); + config.globalSettings.autoStartRestoredQueue = true; + _startupAutoResumeController = null; + window.api.configureAutomationProbe({ paused: false, completionError: 'ledger unavailable' }); + const failedEvidenceResult = await _autoDeduplicateFromLog(); + scheduleRestoredQueueAutoStart(); + const failedEvidence = { + result: failedEvidenceResult, + available: typeof _startupQueueEvidenceAvailable === 'undefined' ? null : _startupQueueEvidenceAvailable, + controllerCreated: _startupAutoResumeController !== null + }; + cancelStartupQueueAutoStart(); + config.globalSettings.autoStartRestoredQueue = false; const completedEvidence = { removedAfterDone, completedKeyPresent, admittedFiles: completedResult.admittedFiles.length, matchingQueueJobs: queueJobs.filter(job => normalizeAutomationPath(job.file) === normalizeAutomationPath(completedFile.path)).length, - startOrInjectCalls: completedProbe.mutationCalls.filter(call => call[0] === 'start' || call[0] === 'inject').length + startOrInjectCalls: completedProbe.mutationCalls.filter(call => call[0] === 'start' || call[0] === 'inject').length, + durableAlreadyProcessed: durableEvaluation.summary.alreadyProcessed, + durableResultingJobs: durableEvaluation.summary.resultingJobs, + changedAlreadyProcessed: changedEvaluation.summary.alreadyProcessed, + changedResultingJobs: changedEvaluation.summary.resultingJobs, + partialAlreadyProcessed: partialEvaluation.summary.alreadyProcessed, + partialResultingJobs: partialEvaluation.summary.resultingJobs, + partialHosters: partialEvaluation.candidates[0]?.eligibleHosters || [], + restoredQueueRemoved: !queueJobs.some(job => job.id === restoredJob.id), + restoredCompletionKey: _completedUploadKeys.has(restoredJob.file + '|' + restoredJob.hoster), + failedEvidence }; _completedUploadKeys.clear(); config.globalSettings.removeFromQueueOnDone = false; @@ -2827,7 +2951,7 @@ app.whenReady().then(async () => { deferredFiles: 70 }, frozen: true, - reads: { history: 1, uploadLog: 1, inspect: 1, status: 0, testScan: 0, reconcile: 0 } + reads: { history: 1, uploadLog: 1, completions: 1, inspect: 1, status: 0, testScan: 0, reconcile: 0 } }); assert.deepEqual(result.automationPipeline.manualTest, { fingerprintEqual: true, @@ -2843,7 +2967,7 @@ app.whenReady().then(async () => { availableSlots: 1200, deferredFiles: 0 }, - reads: { history: 1, uploadLog: 1, inspect: 1, status: 0, testScan: 1, reconcile: 0 } + reads: { history: 1, uploadLog: 1, completions: 1, inspect: 1, status: 0, testScan: 1, reconcile: 0 } }); assert.deepEqual(result.automationPipeline.historyEvidence, { alreadyProcessed: 2, @@ -2855,7 +2979,17 @@ app.whenReady().then(async () => { completedKeyPresent: true, admittedFiles: 0, matchingQueueJobs: 0, - startOrInjectCalls: 0 + startOrInjectCalls: 0, + durableAlreadyProcessed: 1, + durableResultingJobs: 0, + changedAlreadyProcessed: 0, + changedResultingJobs: 1, + partialAlreadyProcessed: 0, + partialResultingJobs: 1, + partialHosters: ['voe.sx'], + restoredQueueRemoved: true, + restoredCompletionKey: true, + failedEvidence: { result: false, available: false, controllerCreated: false } }); assert.deepEqual(result.automationPipeline.pendingDedup, { evaluatedNames: ['new.mkv'], diff --git a/tests/stats.test.js b/tests/stats.test.js index 80dc60f..9e1e5a1 100644 --- a/tests/stats.test.js +++ b/tests/stats.test.js @@ -105,6 +105,12 @@ test('classifyErrorCategory: aborted is its own bucket (not retryable)', () => { assert.strictEqual(isRetryableCategory('aborted'), false); }); +test('automation completion persistence failures are never retried as uploads', () => { + const category = classifyErrorCategory('Automatik-Abschlussnachweis konnte nicht gespeichert werden'); + assert.strictEqual(category, 'local-persistence'); + assert.strictEqual(isRetryableCategory(category), false); +}); + test('classifyErrorCategory: unknown for everything else', () => { assert.strictEqual(classifyErrorCategory(''), 'unknown'); assert.strictEqual(classifyErrorCategory(null), 'unknown'); @@ -176,4 +182,5 @@ test('isRetryableCategory: only transient + network + unknown retry-worthy', () assert.strictEqual(isRetryableCategory('file-rejected'), false); assert.strictEqual(isRetryableCategory('account-error'), false); assert.strictEqual(isRetryableCategory('aborted'), false); + assert.strictEqual(isRetryableCategory('local-persistence'), false); }); diff --git a/tests/upload-log.test.js b/tests/upload-log.test.js index a979646..ffbae6d 100644 --- a/tests/upload-log.test.js +++ b/tests/upload-log.test.js @@ -3,6 +3,8 @@ const assert = require('node:assert'); const { formatUploadLogLine, parseUploadLogLine, + iterateUploadLogEntries, + readUploadLogEntries, summarizeBatchPlan, formatUploadPlanLogLine } = require('../lib/upload-log'); @@ -102,6 +104,11 @@ test('parseUploadLogLine skips comments, blanks and malformed lines', () => { assert.equal(parseUploadLogLine(42), null); }); +test('parser distinguishes confirmed uploads from filename-only rows', () => { + assert.equal(parseUploadLogLine('2026-08-27 05:40:00|voe.sx|||episode.mkv|').confirmed, false); + assert.equal(parseUploadLogLine('2026-08-27 05:40:00|voe.sx|https://voe.sx/e/code||episode.mkv|').confirmed, true); +}); + test('parseUploadLogLine: missing/garbage timestamp yields ts=undefined (legacy lines still match by name)', () => { const parsed = parseUploadLogLine('|voe.sx|link||a.mkv|'); assert.equal(parsed.hoster, 'voe.sx'); @@ -136,3 +143,78 @@ test('SEAM: a leading-space filename round-trips and the gate still drops its gh const { removed } = partitionRestoredJobsByLog([job], [parsed], savedAt); assert.equal(removed.length, 1, 'leading-space filename now matches end-to-end (was a mismatch before)'); }); + +test('stream reader parses large logs incrementally and yields between bounded line batches', async () => { + const fs = require('node:fs'); + const os = require('node:os'); + const path = require('node:path'); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'upload-log-stream-')); + const filePath = path.join(directory, 'fileuploader.log'); + const lines = Array.from({ length: 2505 }, (_, index) => formatUploadLogLine( + new Date(2026, 7, 27, 5, 40, index % 60), + index % 2 === 0 ? 'voe.sx' : 'doodstream.com', + `https://example.invalid/${index}`, + `episode-${index}.mkv` + )).join(''); + fs.writeFileSync(filePath, lines, 'utf8'); + let yields = 0; + try { + const entries = await readUploadLogEntries(filePath, { + yieldEvery: 500, + yieldFn: async () => { yields++; } + }); + assert.equal(entries.length, 2505); + assert.equal(entries[0].fileName, 'episode-0.mkv'); + assert.equal(entries.at(-1).fileName, 'episode-2504.mkv'); + assert.equal(yields, 5); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('upload-log iterator is lazy and rejects oversized lines', async () => { + let produced = 0; + async function* source() { + produced++; + yield formatUploadLogLine(new Date(2026, 7, 27, 5, 40, 0), 'voe.sx', 'https://example.invalid/1', 'one.mkv').trimEnd(); + produced++; + yield formatUploadLogLine(new Date(2026, 7, 27, 5, 40, 1), 'voe.sx', 'https://example.invalid/2', 'two.mkv').trimEnd(); + } + const iterator = iterateUploadLogEntries('', { lines: source(), maxLineLength: 65536 }); + assert.deepEqual(await iterator.next(), { + done: false, + value: { hoster: 'voe.sx', fileName: 'one.mkv', ts: new Date(2026, 7, 27, 5, 40, 0).getTime(), confirmed: true } + }); + assert.equal(produced, 1); + await iterator.return(); + + const oversized = iterateUploadLogEntries('', { + lines: (async function* () { yield 'x'.repeat(11); })(), + maxLineLength: 10 + }); + await assert.rejects(async () => { for await (const entry of oversized) void entry; }, /Zeile ist zu lang/); + + let destroyed = 0; + const input = { + async *[Symbol.asyncIterator]() { yield 'x'.repeat(11); }, + destroy: () => { destroyed++; } + }; + const leaking = iterateUploadLogEntries('ignored.log', { + fs: { createReadStream: () => input }, + maxLineLength: 10 + }); + await assert.rejects(async () => { for await (const entry of leaking) void entry; }, /Zeile ist zu lang/); + assert.equal(destroyed, 1); + + const oversizedStream = iterateUploadLogEntries('ignored.log', { + fs: { + createReadStream: () => ({ + async *[Symbol.asyncIterator]() { yield 'x'.repeat(11); }, + destroy() {} + }) + }, + maxBytes: 10, + maxLineLength: 100 + }); + await assert.rejects(async () => { for await (const entry of oversizedStream) void entry; }, /Leselimit/); +}); diff --git a/tests/upload-manager.test.js b/tests/upload-manager.test.js index 58edfbe..2624483 100644 --- a/tests/upload-manager.test.js +++ b/tests/upload-manager.test.js @@ -151,8 +151,8 @@ describe('UploadManager', () => { mgr.on('batch-done', (s) => { summary = s; }); await mgr.startBatch([ - { file: '/test/video1.mp4', hoster: 'doodstream.com', apiKey: 'key1' }, - { file: '/test/video2.mp4', hoster: 'doodstream.com', apiKey: 'key1' } + { jobId: 'summary-1', file: '/test/video1.mp4', hoster: 'doodstream.com', apiKey: 'key1' }, + { jobId: 'summary-2', file: '/test/video2.mp4', hoster: 'doodstream.com', apiKey: 'key1' } ]); assert.ok(summary); @@ -160,6 +160,7 @@ describe('UploadManager', () => { assert.equal(summary.succeeded, 2); assert.equal(summary.failed, 0); assert.equal(summary.files.length, 2); + assert.deepEqual(summary.files.map(file => file.results[0].jobId).sort(), ['summary-1', 'summary-2']); }); it('emits a final idle stats snapshot after a normal batch', async () => {