From afa7da11da361d2318e0ad96ff1c92b4166eb808 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Sat, 22 Aug 2026 13:02:27 +0200 Subject: [PATCH] fix: harden package post-processing provenance Version package-owned output counts so unversioned global legacy values cannot remain authoritative after session load. Capture extracted outputs through same-volume package staging, preserve overwrite, skip, and rename conflicts, retain partial abort output deterministically, and eliminate per-package full scans of shared roots while keeping extraction work parallel. Carry normalized archive path provenance, prune explicitly deleted package generations without dropping package_done run evidence, and transfer run-owned post-process slots without releasing foreign waiters or exceeding maxParallelExtract. --- .../residual-hardening-report.md | 46 ++ src/main/download-manager.ts | 421 +++++++++++++----- src/main/extractor.ts | 37 +- src/main/storage.ts | 20 +- src/shared/types.ts | 2 + tests/download-manager.test.ts | 204 ++++++++- tests/notify-hooks.test.ts | 35 ++ tests/storage.test.ts | 39 ++ 8 files changed, 647 insertions(+), 157 deletions(-) create mode 100644 .superpowers/sdd/2026-08-22-daily-schedule-stop-ui-polish/residual-hardening-report.md diff --git a/.superpowers/sdd/2026-08-22-daily-schedule-stop-ui-polish/residual-hardening-report.md b/.superpowers/sdd/2026-08-22-daily-schedule-stop-ui-polish/residual-hardening-report.md new file mode 100644 index 0000000..195bfa6 --- /dev/null +++ b/.superpowers/sdd/2026-08-22-daily-schedule-stop-ui-polish/residual-hardening-report.md @@ -0,0 +1,46 @@ +# Residual Hardening Report + +## Status + +PASS. Alle fünf Residuals sind mit beobachteten REDs, fokussierten GREENs und frischen Abschlussgates umgesetzt. + +Basis: `7a052e995c38cd8d98589d2485f7b49b8c2d4c70` + +## Umsetzung + +1. Persistierte Outputzahlen besitzen jetzt einen expliziten Provenienzvertrag in Version 1. Beim Laden wird `outputCount` ausschließlich aus gültiger Paketprovenienz abgeleitet; unversionierte globale Legacy-Zahlen ohne Provenienz werden auf 0 invalidiert. +2. Archivfortschritt transportiert den tatsächlichen Archivpfad. Die Zuordnung verwendet normalisierte verschachtelte Itempfade, gleiche Basenames in verschiedenen Unterordnern bleiben getrennt und Items ohne Pfadprovenienz erhöhen `partCount` nicht. +3. Explizites Paketlöschen entfernt sämtliche Generationen aus finalisierten, standalone, unterdrückten und Digest-Resultaten sowie Run-Referenzen und Outputprovenienz. Automatische `package_done`-Bereinigung behält bereits finalisierte Run- und Digest-Belege. +4. Post-Process-Waiter besitzen einen Run-Owner. Selektiver Stop verwirft nur Waiter des gestoppten Runs. Ein freier Slot wird atomar an genau einen Waiter übertragen, ohne den Aktivzähler zwischenzeitlich auf 0 zu setzen. +5. Outputprovenienz entsteht aus paket-eigenen Staging-Ausgaben im selben Zielroot. Entpacker verschiedener Pakete dürfen parallel arbeiten; nur der konfliktbehaftete Merge in denselben Root wird in Aufrufreihenfolge serialisiert. Der Merge verwendet Rename/Move, erhält `overwrite`, `skip` und `rename`, übernimmt Partial-Ausgaben bei Abort deterministisch und traversiert ausschließlich das jeweilige Staging-Verzeichnis. + +## RED + +- `npx vitest run tests/storage.test.ts -t "invalidates an unversioned legacy package output count on load"`: 1/1 fehlgeschlagen; geladen wurden `outputCount: 48000` und keine Provenienzversion. +- `npx vitest run tests/download-manager.test.ts -t "captures shared-root provenance|preserves .* conflicts|retains partial staged outputs|uses normalized nested item paths|keeps foreign post-process waiters"`: 6 Fehler. Die Traversalinjektion wurde nicht verwendet, `skip` und `rename` überschrieben die Fremddatei, Abort verwendete keinen bounded Traversal, zwei Nested-Archive kollidierten zu zwei statt drei Operationen und der gestoppte Waiter wurde mit `undefined` statt `false` freigegeben. Der isolierte `overwrite`-Fall war bereits grün. +- `npx vitest run tests/notify-hooks.test.ts -t "prunes only the removed package result generations"`: 1/1 fehlgeschlagen; beide Generationen des entfernten Pakets blieben erhalten. +- Der erste fokussierte Regressionlauf zeigte zusätzlich zwei Fehler in bestehenden `package_done`-Notification-Fällen. Ursache war zu breites Pruning bei automatischer Bereinigung. Nach Begrenzung auf explizites Löschen bestanden die beiden Regressionen zusammen mit dem neuen Removal-Test 3/3. + +## GREEN + +- Neue Residualfälle in `tests/download-manager.test.ts`: 7/7. +- Legacy-Invalidierung und versionierte Persistenz in `tests/storage.test.ts`: 2/2. +- Explizites Removal-Pruning in `tests/notify-hooks.test.ts`: 1/1. +- Shared-Root-Lasttest: 2.000 Fremddateien, zwei parallele Paketoperationen, kein Traversal des Shared Roots und eine literale Traversalobergrenze von höchstens 4 Aufrufen. +- Konflikt- und Abbruchabdeckung: `overwrite`, `skip`, `rename` sowie Partial-Abort und vollständiges Entfernen der Staging-Verzeichnisse. + +## Abschlussgates + +- `npx vitest run tests/notify-hooks.test.ts tests/package-telemetry.test.ts tests/notification-outbox.test.ts tests/history-reveal.test.ts tests/history-view.test.tsx tests/storage.test.ts tests/extractor.test.ts tests/extractor-jvm.test.ts tests/main-shutdown-lifecycle.test.ts`: 324/324. +- `npx vitest run tests/download-manager.test.ts -t "deterministic stop and restart lifecycle|package lifecycle telemetry boundaries|recovers pending extraction on startup"`: 23/23. +- Fokussierte Gesamtsumme: 347/347. +- `npx tsc --noEmit`: Exit 0. +- `npm run build:main`: Exit 0. +- `npm run build:renderer`: Exit 0. +- `git diff --check`: Exit 0 nach Produktions-, Test- und Berichtsänderungen. + +## Restbedenken + +- Der Renderer-Build meldet weiterhin den bestehenden JavaScript-Chunk über 500 kB. +- Native WinRAR-/7-Zip-Prozesse wurden nicht als reales Windows-End-to-End-Szenario gestartet; die fokussierten Extractor- und JVM-Suites bestanden 86/86. +- Auf Vorgabe wurden keine Vollsuite und keine GUI-, RDP-, Maus-, Fenster- oder Zwischenablageprüfungen ausgeführt. diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 617ee14..8fe052d 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -18,6 +18,7 @@ import { HistoryEntry, ArchiveOperationMetric, PackageEntry, + PACKAGE_OUTPUT_PROVENANCE_VERSION, PackagePriority, PackageResult, ParsedPackageInput, @@ -476,6 +477,7 @@ type DownloadManagerOptions = { onHistoryEntry?: HistoryEntryCallback; enqueueNotification?: (event: NotificationEvent) => Promise; protectEmptyClobber?: boolean; + readOutputDirectory?: (directory: string) => Promise; }; type RunLifecycleContext = { @@ -1619,10 +1621,28 @@ const ARCHIVE_GENERIC_001_RE = /^(.*)\.001$/; const ARCHIVE_KNOWN_001_RE = /\.(zip|7z)\.001$/; const REGEX_ESCAPE_RE = /[.*+?^${}()|[\]\\]/g; -export function resolveArchiveItemsFromList(archiveName: string, items: DownloadItem[]): DownloadItem[] { - const normalizeArchiveMatchName = (value: string): string => - stripDuplicateSuffixBeforeExtension(path.basename(String(value || ""))); - const entryLower = normalizeArchiveMatchName(archiveName).toLowerCase(); +export function resolveArchiveItemsFromList(archiveName: string, items: DownloadItem[], archivePath = ""): DownloadItem[] { + const normalizeArchiveMatchName = (value: string): string => + stripDuplicateSuffixBeforeExtension(path.basename(String(value || ""))); + const entryLower = normalizeArchiveMatchName(archiveName).toLowerCase(); + + const normalizedArchivePath = String(archivePath || "").trim(); + if (normalizedArchivePath) { + const archivePathKey = pathKey(path.join( + path.dirname(path.resolve(normalizedArchivePath)), + normalizeArchiveMatchName(normalizedArchivePath) + )); + const pathMatches = items.filter((item) => { + const targetPath = String(item.targetPath || "").trim(); + if (!targetPath) { + return false; + } + return pathKey(path.join(path.dirname(path.resolve(targetPath)), normalizeArchiveMatchName(targetPath))) === archivePathKey; + }); + if (pathMatches.length > 0) { + return pathMatches; + } + } const itemBaseName = (item: DownloadItem): string => normalizeArchiveMatchName(item.targetPath || item.fileName || ""); @@ -1916,7 +1936,7 @@ export class DownloadManager extends EventEmitter { private packagePostProcessActive = 0; - private packagePostProcessWaiters: Array<{ packageId: string; resolve: () => void }> = []; + private packagePostProcessWaiters: Array<{ packageId: string; runOwnerId: string | null; resolve: (acquired: boolean) => void }> = []; private packagePostProcessTasks = new Map>(); @@ -2048,6 +2068,8 @@ export class DownloadManager extends EventEmitter { private onHistoryEntryCallback?: HistoryEntryCallback; private enqueueNotificationCallback?: (event: NotificationEvent) => Promise; + + private readOutputDirectoryFn: (directory: string) => Promise; public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) { super(); @@ -2078,6 +2100,7 @@ export class DownloadManager extends EventEmitter { this.invalidateMegaSessionFn = options.invalidateMegaSession; this.onHistoryEntryCallback = options.onHistoryEntry; this.enqueueNotificationCallback = options.enqueueNotification; + this.readOutputDirectoryFn = options.readOutputDirectory || ((directory) => fs.promises.readdir(directory, { withFileTypes: true })); logger.info(`DownloadManager Init: ${Object.keys(this.session.packages).length} Pakete, ${this.itemCount} Items, cleanupPolicy=${this.settings.completedCleanupPolicy}`); for (const pkg of Object.values(this.session.packages)) { this.ensurePackageLogForPackage(pkg); @@ -2639,11 +2662,9 @@ export class DownloadManager extends EventEmitter { return this.session.running; } - public abortAllPostProcessing(): void { - this.abortPostProcessing("external"); - for (const waiter of this.packagePostProcessWaiters) { waiter.resolve(); } - this.packagePostProcessWaiters = []; - this.packagePostProcessActive = 0; + public abortAllPostProcessing(): void { + this.abortPostProcessing("external"); + this.cancelPostProcessWaiters(); } public triggerIdleExtractions(): void { @@ -3327,11 +3348,9 @@ export class DownloadManager extends EventEmitter { this.hybridExtractRequeue.clear(); this.hybridExtractedPaths.clear(); this.hybridFailedArchives.clear(); - this.providerFailures.clear(); - this.packagePostProcessQueue = Promise.resolve(); - this.packagePostProcessActive = 0; - for (const waiter of this.packagePostProcessWaiters) { waiter.resolve(); } - this.packagePostProcessWaiters = []; + this.providerFailures.clear(); + this.packagePostProcessQueue = Promise.resolve(); + this.cancelPostProcessWaiters(); this.summary = null; this.nonResumableActive = 0; this.resetSessionTotalsIfQueueEmpty(true); @@ -4365,17 +4384,14 @@ export class DownloadManager extends EventEmitter { return false; } - private async snapshotPackageOutputFiles(rootDir: string): Promise> { - const snapshot = new Map(); - if (!rootDir) { - return snapshot; - } - const stack = [rootDir]; + private async listStagedOutputFiles(stagingDir: string): Promise { + const files: string[] = []; + const stack = [stagingDir]; while (stack.length > 0) { const current = stack.pop() as string; let entries: fs.Dirent[] = []; try { - entries = await fs.promises.readdir(current, { withFileTypes: true }); + entries = await this.readOutputDirectoryFn(current); } catch { continue; } @@ -4386,54 +4402,169 @@ export class DownloadManager extends EventEmitter { } if (entry.isDirectory()) { stack.push(fullPath); - } else if (entry.isFile() && !isArchiveLikePath(fullPath) && !isIgnorableEmptyDirFileName(entry.name)) { - try { - const stat = await fs.promises.stat(fullPath); - const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/"); - const key = process.platform === "win32" ? relativePath.toLowerCase() : relativePath; - snapshot.set(key, `${stat.size}:${stat.mtimeMs}`); - } catch { - } + } else if (entry.isFile()) { + files.push(fullPath); } } } - return snapshot; + return files.sort((left, right) => path.relative(stagingDir, left).localeCompare(path.relative(stagingDir, right))); } - private async recordPackageOutputFiles(pkg: PackageEntry, before: ReadonlyMap): Promise { - const after = await this.snapshotPackageOutputFiles(pkg.extractDir); - const provenance = new Set(pkg.outputProvenance || []); - for (const [relativePath, signature] of after) { - if (before.get(relativePath) !== signature) { - provenance.add(createHash("sha256").update(relativePath).digest("hex")); + private async resolveStagedOutputDestination(targetDir: string, relativePath: string): Promise { + const targetRoot = path.resolve(targetDir); + const destination = path.resolve(targetRoot, relativePath); + if (destination !== targetRoot && !destination.startsWith(`${targetRoot}${path.sep}`)) { + throw new Error(`Ungültiger Staging-Ausgabepfad: ${relativePath}`); + } + let existing: fs.Stats | null = null; + try { + existing = await fs.promises.lstat(destination); + } catch { + } + if (!existing) { + return destination; + } + if (this.settings.extractConflictMode === "skip" || this.settings.extractConflictMode === "ask") { + return null; + } + if (this.settings.extractConflictMode === "overwrite") { + return existing.isFile() ? destination : null; + } + const parsed = path.parse(destination); + for (let index = 1; index <= 10_000; index += 1) { + const candidate = path.join(parsed.dir, `${parsed.name} (${index})${parsed.ext}`); + try { + await fs.promises.lstat(candidate); + } catch { + return candidate; } } - pkg.outputProvenance = [...provenance]; - pkg.outputCount = Math.max(pkg.outputCount || 0, provenance.size); + throw new Error(`Staging-Rename-Limit erreicht für ${relativePath}`); } - private async runWithPackageOutputProvenance(pkg: PackageEntry, operation: () => Promise): Promise { + private async moveStagedOutputFile(sourcePath: string, destinationPath: string): Promise { + await fs.promises.mkdir(path.dirname(destinationPath), { recursive: true }); + try { + await fs.promises.rename(sourcePath, destinationPath); + return; + } catch (error) { + const code = String((error as NodeJS.ErrnoException)?.code || ""); + if (this.settings.extractConflictMode !== "overwrite" || (code !== "EEXIST" && code !== "EPERM" && code !== "EACCES")) { + throw error; + } + } + const displacedPath = path.join(path.dirname(destinationPath), `.rd-replace-${uuidv4()}`); + await fs.promises.rename(destinationPath, displacedPath); + try { + await fs.promises.rename(sourcePath, destinationPath); + } catch (error) { + await fs.promises.rename(displacedPath, destinationPath); + throw error; + } + await fs.promises.rm(displacedPath, { force: true }); + } + + private async mergeStagedPackageOutputs(stagingDir: string, targetDir: string): Promise { + const movedOutputs: string[] = []; + const stagedFiles = await this.listStagedOutputFiles(stagingDir); + for (const stagedFile of stagedFiles) { + const relativePath = path.relative(stagingDir, stagedFile); + const destination = await this.resolveStagedOutputDestination(targetDir, relativePath); + if (!destination) { + continue; + } + await this.moveStagedOutputFile(stagedFile, destination); + if (!isArchiveLikePath(destination) && !isIgnorableEmptyDirFileName(path.basename(destination))) { + movedOutputs.push(destination); + } + } + return movedOutputs; + } + + private normalizePackageProvenancePath(pkg: PackageEntry, sourcePath: string): string { + const absolutePath = path.resolve(sourcePath); + for (const rootDir of [pkg.outputDir, pkg.extractDir]) { + const root = String(rootDir || "").trim(); + if (!root) { + continue; + } + const relativePath = path.relative(path.resolve(root), absolutePath); + if (!relativePath.startsWith(`..${path.sep}`) && relativePath !== ".." && !path.isAbsolute(relativePath)) { + const segments = relativePath.replace(/\\/g, "/").split("/"); + if (/^\.rd-output-[^/]+$/i.test(segments[0] || "")) { + segments.shift(); + } + return segments.join("/").toLocaleLowerCase("de-DE"); + } + } + return absolutePath.replace(/\\/g, "/").toLocaleLowerCase("de-DE"); + } + + private recordPackageOutputFiles(pkg: PackageEntry, outputFiles: readonly string[]): void { + const provenance = new Set(pkg.outputProvenance || []); + for (const outputFile of outputFiles) { + const key = this.normalizePackageProvenancePath(pkg, outputFile); + provenance.add(createHash("sha256").update(key).digest("hex")); + } + pkg.outputProvenance = [...provenance]; + pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION; + pkg.outputCount = provenance.size; + } + + private async runWithPackageOutputProvenance(pkg: PackageEntry, operation: (targetDir: string) => Promise): Promise { const key = pathKey(pkg.extractDir); + const packageWasInSession = this.session.packages[pkg.id] === pkg; const previous = this.packageOutputProvenanceTails.get(key) || Promise.resolve(); let release!: () => void; const current = new Promise((resolve) => { release = resolve; }); this.packageOutputProvenanceTails.set(key, current); - await previous; - const before = await this.snapshotPackageOutputFiles(pkg.extractDir); + let stagingDir = ""; + let result: T | undefined; + let operationError: unknown; + let mergeError: unknown; + let mergeTurnReached = false; try { - return await operation(); + await fs.promises.mkdir(pkg.extractDir, { recursive: true }); + stagingDir = await fs.promises.mkdtemp(path.join(pkg.extractDir, ".rd-output-")); + try { + result = await operation(stagingDir); + } catch (error) { + operationError = error; + } + await previous; + mergeTurnReached = true; + try { + if (!packageWasInSession || this.session.packages[pkg.id] === pkg) { + const outputFiles = await this.mergeStagedPackageOutputs(stagingDir, pkg.extractDir); + this.recordPackageOutputFiles(pkg, outputFiles); + } + } catch (error) { + mergeError = error; + } } finally { try { - await this.recordPackageOutputFiles(pkg, before); + if (stagingDir) { + await fs.promises.rm(stagingDir, { recursive: true, force: true }); + } } finally { + if (!mergeTurnReached) { + await previous; + } release(); if (this.packageOutputProvenanceTails.get(key) === current) { this.packageOutputProvenanceTails.delete(key); } } } + if (operationError) { + throw operationError; + } + if (mergeError) { + throw mergeError; + } + return result as T; } private async removeEmptyDirectoryTree(rootDir: string): Promise { @@ -6854,11 +6985,9 @@ export class DownloadManager extends EventEmitter { this.speedEvents = []; this.speedBytesLastWindow = 0; this.speedBytesPerPackage.clear(); - this.speedEventsHead = 0; + this.speedEventsHead = 0; this.abortPostProcessing("stop", stoppedRunContext?.id); - for (const waiter of this.packagePostProcessWaiters) { waiter.resolve(); } - this.packagePostProcessWaiters = []; - this.packagePostProcessActive = 0; + this.cancelPostProcessWaiters(stoppedRunContext?.id); for (const active of this.activeTasks.values()) { active.abortReason = abortReason; active.abortController.abort(abortReason); @@ -8434,7 +8563,7 @@ export class DownloadManager extends EventEmitter { private abortPostProcessing(reason: string, runContextId?: string): void { for (const [packageId, controller] of this.packagePostProcessAbortControllers.entries()) { const owner = this.packagePostProcessRunOwnerByController.get(controller); - if (runContextId !== undefined && owner !== undefined && owner !== null && owner !== runContextId) { + if (runContextId !== undefined && owner !== runContextId) { continue; } if (!controller.signal.aborted) { @@ -8468,7 +8597,7 @@ export class DownloadManager extends EventEmitter { for (const controller of this.packageDeferredPostProcessAbortControllers.values()) { const owner = this.packageDeferredRunOwnerByController.get(controller); - if (runContextId !== undefined && owner !== undefined && owner !== null && owner !== runContextId) { + if (runContextId !== undefined && owner !== runContextId) { continue; } if (!controller.signal.aborted) { @@ -8478,7 +8607,7 @@ export class DownloadManager extends EventEmitter { for (const hybridSet of this.packageHybridPostProcessControllers.values()) { for (const controller of hybridSet) { const owner = this.packageHybridRunOwnerByController.get(controller); - if (runContextId !== undefined && owner !== undefined && owner !== null && owner !== runContextId) { + if (runContextId !== undefined && owner !== runContextId) { continue; } if (!controller.signal.aborted) { @@ -8488,27 +8617,39 @@ export class DownloadManager extends EventEmitter { } } - private async acquirePostProcessSlot(packageId: string): Promise { - const maxConcurrent = Math.max(1, Math.min(8, this.settings.maxParallelExtract || 1)); - if (this.packagePostProcessActive < maxConcurrent) { - this.packagePostProcessActive += 1; - return; - } - await new Promise((resolve) => { - this.packagePostProcessWaiters.push({ packageId, resolve }); - }); - if (this.packagePostProcessActive < maxConcurrent) { - this.packagePostProcessActive += 1; - } - } + private cancelPostProcessWaiters(runOwnerId?: string): void { + const retained: typeof this.packagePostProcessWaiters = []; + for (const waiter of this.packagePostProcessWaiters) { + if (runOwnerId !== undefined && waiter.runOwnerId !== runOwnerId) { + retained.push(waiter); + } else { + waiter.resolve(false); + } + } + this.packagePostProcessWaiters = retained; + } + + private async acquirePostProcessSlot(packageId: string, runOwnerId: string | null = this.getPackageResultRunOwner(packageId)): Promise { + const maxConcurrent = Math.max(1, Math.min(8, this.settings.maxParallelExtract || 1)); + if (this.packagePostProcessActive < maxConcurrent) { + this.packagePostProcessActive += 1; + return true; + } + return new Promise((resolve) => { + this.packagePostProcessWaiters.push({ packageId, runOwnerId, resolve }); + }); + } private releasePostProcessSlot(): void { if (this.packagePostProcessActive <= 0) { this.packagePostProcessActive = 0; return; } - this.packagePostProcessActive -= 1; - if (this.packagePostProcessWaiters.length === 0) return; + const maxConcurrent = Math.max(1, Math.min(8, this.settings.maxParallelExtract || 1)); + if (this.packagePostProcessWaiters.length === 0 || this.packagePostProcessActive > maxConcurrent) { + this.packagePostProcessActive -= 1; + return; + } const order = this.session.packageOrder; let bestIdx = 0; let bestOrder = order.indexOf(this.packagePostProcessWaiters[0].packageId); @@ -8520,10 +8661,10 @@ export class DownloadManager extends EventEmitter { bestOrder = pos; bestIdx = i; } - } - const [next] = this.packagePostProcessWaiters.splice(bestIdx, 1); - next.resolve(); - } + } + const [next] = this.packagePostProcessWaiters.splice(bestIdx, 1); + next.resolve(true); + } private runPackagePostProcessing(packageId: string): Promise { this.trackPackagePostProcessResult(packageId); @@ -8544,27 +8685,34 @@ export class DownloadManager extends EventEmitter { // Holder so the task's own finally can identity-check itself (the task Promise // cannot reference its own const inside its initializer). Assigned right after. - const handle: { task?: Promise } = {}; - const task = (async () => { + const handle: { task?: Promise } = {}; + const task = (async () => { const slotWaitStart = nowMs(); - await this.acquirePostProcessSlot(packageId); - const startedPackage = this.session.packages[packageId]; - if (startedPackage) { - startedPackage.postProcessStartedAt = startedPackage.postProcessStartedAt || nowMs(); - startedPackage.updatedAt = nowMs(); - } - const slotWaitMs = nowMs() - slotWaitStart; - if (slotWaitMs > 100) { - logger.info(`Post-Process Slot erhalten nach ${(slotWaitMs / 1000).toFixed(1)}s Wartezeit: pkg=${packageId.slice(0, 8)}`); - const pkg = this.session.packages[packageId]; - if (pkg) { - this.logPackageForPackage(pkg, "INFO", "Post-Process-Slot erhalten", { - slotWaitMs - }); - } - } - try { - let round = 0; + let slotAcquired = false; + try { + slotAcquired = await this.acquirePostProcessSlot( + packageId, + this.packagePostProcessRunOwnerByController.get(abortController) ?? null + ); + if (!slotAcquired) { + return; + } + const startedPackage = this.session.packages[packageId]; + if (startedPackage) { + startedPackage.postProcessStartedAt = startedPackage.postProcessStartedAt || nowMs(); + startedPackage.updatedAt = nowMs(); + } + const slotWaitMs = nowMs() - slotWaitStart; + if (slotWaitMs > 100) { + logger.info(`Post-Process Slot erhalten nach ${(slotWaitMs / 1000).toFixed(1)}s Wartezeit: pkg=${packageId.slice(0, 8)}`); + const pkg = this.session.packages[packageId]; + if (pkg) { + this.logPackageForPackage(pkg, "INFO", "Post-Process-Slot erhalten", { + slotWaitMs + }); + } + } + let round = 0; do { round += 1; const hadRequeue = this.hybridExtractRequeue.has(packageId); @@ -8594,8 +8742,10 @@ export class DownloadManager extends EventEmitter { } } } while (this.hybridExtractRequeue.has(packageId)); - } finally { - this.releasePostProcessSlot(); + } finally { + if (slotAcquired) { + this.releasePostProcessSlot(); + } // Identity guard: only clear the map entries if they still point to THIS // task/controller. After an abort deletes our handle a new run can install // a fresh task+controller for the same packageId; a blind delete here would @@ -8981,6 +9131,16 @@ export class DownloadManager extends EventEmitter { } this.historyRecordedPackages.delete(packageId); this.abortPackagePostProcessing(packageId, "package_removed"); + this.packagePostProcessVersions.delete(packageId); + this.packageFileOpChain.delete(packageId); + if (reason === "deleted") { + this.pruneRemovedPackageResultState(packageId); + } + if (pkg && reason === "deleted") { + pkg.outputCount = 0; + pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION; + pkg.outputProvenance = []; + } for (const itemId of itemIds) { this.retryAfterByItem.delete(itemId); this.retryStateByItem.delete(itemId); @@ -8988,9 +9148,12 @@ export class DownloadManager extends EventEmitter { this.dropItemContribution(itemId); delete this.session.items[itemId]; this.itemCount = Math.max(0, this.itemCount - 1); - } + } delete this.session.packages[packageId]; this.session.packageOrder = this.session.packageOrder.filter((id) => id !== packageId); + if (reason === "deleted") { + this.runPackageIds.delete(packageId); + } this.runCompletedPackages.delete(packageId); this.resetSessionTotalsIfQueueEmpty(); } @@ -12288,6 +12451,7 @@ export class DownloadManager extends EventEmitter { pkg.archiveOperations = []; pkg.remuxOperations = []; pkg.outputCount = 0; + pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION; pkg.outputProvenance = []; pkg.cleanupErrorCategory = ""; } @@ -12312,6 +12476,30 @@ export class DownloadManager extends EventEmitter { } } + private pruneRemovedPackageResultState(packageId: string): void { + const prefix = `${packageId}:`; + for (const key of [...this.finalizedPackageResults.keys()]) { + if (key.startsWith(prefix)) { + this.finalizedPackageResults.delete(key); + } + } + for (const collection of [this.standalonePackageResults, this.suppressedPackageResults]) { + for (const key of [...collection]) { + if (key.startsWith(prefix)) { + collection.delete(key); + } + } + } + for (const key of [...this.successDigestResults.keys()]) { + if (key.startsWith(prefix)) { + this.successDigestResults.delete(key); + } + } + for (const context of this.runContexts.values()) { + context.packageGenerations.delete(packageId); + } + } + private createRunContext(packageIds: Iterable, startedAt: number, downloadsFinished: boolean): RunLifecycleContext { const packageGenerations = new Map(); for (const packageId of packageIds) { @@ -13091,13 +13279,16 @@ export class DownloadManager extends EventEmitter { } const completedAt = nowMs(); const durationMs = Math.max(0, Math.floor(Number(progress.elapsedMs) || 0)); - const itemIds = [...new Set(items.map((item) => item.id))]; - const itemProvenance = items - .map((item) => String(item.targetPath || item.id).replace(/\\/g, "/").toLocaleLowerCase("de-DE")) + const provenancedItems = items.filter((item) => String(item.targetPath || "").trim().length > 0); + const itemIds = [...new Set(provenancedItems.map((item) => item.id))]; + const itemProvenance = provenancedItems + .map((item) => this.normalizePackageProvenancePath(pkg, item.targetPath)) .sort(); - const archiveIdentity = itemProvenance.length > 0 - ? itemProvenance.join("|") - : `${progress.archiveName.toLocaleLowerCase("de-DE")}:${Math.max(0, Math.floor(progress.current))}`; + const archivePathProvenance = String(progress.archivePath || "").trim() + ? this.normalizePackageProvenancePath(pkg, progress.archivePath || "") + : ""; + const archiveIdentity = (itemProvenance.length > 0 ? itemProvenance.join("|") : archivePathProvenance) + || `${progress.archiveName.toLocaleLowerCase("de-DE")}:${Math.max(0, Math.floor(progress.current))}`; const operation: ArchiveOperationMetric = { id: `${pkg.id}:${createHash("sha256").update(archiveIdentity).digest("hex").slice(0, 24)}`, name: progress.archiveName, @@ -13244,8 +13435,8 @@ export class DownloadManager extends EventEmitter { return 0; } - const resolveArchiveItems = (archiveName: string): DownloadItem[] => - resolveArchiveItemsFromList(archiveName, items); + const resolveArchiveItems = (archiveName: string, archivePath = ""): DownloadItem[] => + resolveArchiveItemsFromList(archiveName, items, archivePath); const readyArchiveKeyByName = new Map(); const readyArchiveMarkers = new Map(); @@ -13291,9 +13482,9 @@ export class DownloadManager extends EventEmitter { return 0; } - const result = await this.runWithPackageOutputProvenance(pkg, () => extractPackageArchives({ - packageDir: pkg.outputDir, - targetDir: pkg.extractDir, + const result = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({ + packageDir: pkg.outputDir, + targetDir, cleanupMode: this.settings.cleanupMode, conflictMode: this.settings.extractConflictMode, removeLinks: false, @@ -13341,7 +13532,7 @@ export class DownloadManager extends EventEmitter { if (progress.archiveName) { if (!hybridResolvedItems.has(progress.archiveName)) { - const resolved = resolveArchiveItems(progress.archiveName); + const resolved = resolveArchiveItems(progress.archiveName, progress.archivePath); hybridResolvedItems.set(progress.archiveName, resolved); hybridStartTimes.set(progress.archiveName, nowMs()); if (resolved.length === 0) { @@ -13756,8 +13947,8 @@ export class DownloadManager extends EventEmitter { const extractionStartMs = nowMs(); const preExtractStatuses = new Map(); - const resolveArchiveItems = (archiveName: string): DownloadItem[] => - resolveArchiveItemsFromList(archiveName, completedItems); + const resolveArchiveItems = (archiveName: string, archivePath = ""): DownloadItem[] => + resolveArchiveItemsFromList(archiveName, completedItems, archivePath); let lastExtractEmitAt = 0; const emitExtractStatus = (text: string, force = false): void => { @@ -13866,9 +14057,9 @@ export class DownloadManager extends EventEmitter { entry.updatedAt = pendingAt; } this.emitState(); - const result = await this.runWithPackageOutputProvenance(pkg, () => extractPackageArchives({ - packageDir: pkg.outputDir, - targetDir: pkg.extractDir, + const result = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({ + packageDir: pkg.outputDir, + targetDir, cleanupMode: this.settings.cleanupMode, conflictMode: this.settings.extractConflictMode, removeLinks: this.settings.removeLinkFilesAfterExtract, @@ -13919,7 +14110,7 @@ export class DownloadManager extends EventEmitter { if (progress.archiveName) { if (!fullResolvedItems.has(progress.archiveName)) { - const resolved = resolveArchiveItems(progress.archiveName); + const resolved = resolveArchiveItems(progress.archiveName, progress.archivePath); fullResolvedItems.set(progress.archiveName, resolved); fullStartTimes.set(progress.archiveName, nowMs()); if (resolved.length === 0) { @@ -14214,9 +14405,9 @@ export class DownloadManager extends EventEmitter { }); const nestedFailureCategories = new Map(); const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[]; - const nestedResult = await this.runWithPackageOutputProvenance(pkg, () => extractPackageArchives({ - packageDir: pkg.extractDir, - targetDir: pkg.extractDir, + const nestedResult = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({ + packageDir: pkg.extractDir, + targetDir, cleanupMode: this.settings.cleanupMode, conflictMode: this.settings.extractConflictMode, removeLinks: false, @@ -14235,7 +14426,7 @@ export class DownloadManager extends EventEmitter { this.recordArchiveOperation( pkg, progress, - resolveArchiveItemsFromList(progress.archiveName, nestedItems), + resolveArchiveItemsFromList(progress.archiveName, nestedItems, progress.archivePath), nestedFailureCategories.get(progress.archiveName.toLowerCase()) || "" ); } diff --git a/src/main/extractor.ts b/src/main/extractor.ts index da640b1..1d01ff9 100644 --- a/src/main/extractor.ts +++ b/src/main/extractor.ts @@ -63,11 +63,12 @@ export interface ExtractOptions { onLog?: (level: "INFO" | "WARN" | "ERROR", message: string) => void; } -export interface ExtractProgressUpdate { +export interface ExtractProgressUpdate { current: number; total: number; percent: number; - archiveName: string; + archiveName: string; + archivePath?: string; archivePercent?: number; elapsedMs?: number; phase: "extracting" | "done" | "preparing"; @@ -2954,14 +2955,15 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ } }; - const emitProgress = ( + const emitProgress = ( current: number, archiveName: string, phase: "extracting" | "done", archivePercent?: number, elapsedMs?: number, pwInfo?: { passwordAttempt?: number; passwordTotal?: number; passwordFound?: boolean }, - archiveInfo?: { archiveDone?: boolean; archiveSuccess?: boolean } + archiveInfo?: { archiveDone?: boolean; archiveSuccess?: boolean }, + archivePath?: string ): void => { if (!options.onProgress) { return; @@ -2981,7 +2983,8 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ current, total, percent, - archiveName, + archiveName, + ...(archivePath ? { archivePath } : {}), archivePercent: normalizedArchivePercent, elapsedMs, phase, @@ -2997,7 +3000,7 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ for (const archivePath of candidates) { if (resumeCompleted.has(archiveNameKey(path.basename(archivePath)))) { - emitProgress(extracted, path.basename(archivePath), "extracting", 100, 0, undefined, { archiveDone: true, archiveSuccess: true }); + emitProgress(extracted, path.basename(archivePath), "extracting", 100, 0, undefined, { archiveDone: true, archiveSuccess: true }, archivePath); } } @@ -3022,9 +3025,9 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ let archivePercent = 0; let reached99At: number | null = null; let archiveOutcome: "success" | "failed" | "skipped" = "failed"; - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, 0); + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, 0, undefined, undefined, archivePath); const pulseTimer = setInterval(() => { - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt); + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, undefined, undefined, archivePath); }, 1100); const hybrid = Boolean(options.hybridMode); const filenamePasswords = archiveFilenamePasswords(archiveName); @@ -3041,7 +3044,7 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ reached99At = Date.now(); logger.info(`Extract-Trace 99%: archive=${archiveName}, elapsedMs=${reached99At - archiveStartedAt}`); } - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt); + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, undefined, undefined, archivePath); }; const isGenericSplit = /\.\d{3}$/i.test(archiveName) && !/\.(zip|7z)\.\d{3}$/i.test(archiveName); @@ -3069,11 +3072,11 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ options.onLog?.("INFO", `Archiv-Passwortliste: archive=${archiveName}, passwordCount=${archivePasswordCandidates.length}, redacted=true, emptyCandidates=${emptyArchivePasswordCount}`); const hasManyPasswords = archivePasswordCandidates.length > 1; if (hasManyPasswords) { - emitProgress(extracted + failed, archiveName, "extracting", 0, 0, { passwordAttempt: 0, passwordTotal: archivePasswordCandidates.length }); + emitProgress(extracted + failed, archiveName, "extracting", 0, 0, { passwordAttempt: 0, passwordTotal: archivePasswordCandidates.length }, undefined, archivePath); } const onPwAttempt = hasManyPasswords ? (attempt: number, total: number) => { - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, { passwordAttempt: attempt, passwordTotal: total }); + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, { passwordAttempt: attempt, passwordTotal: total }, undefined, archivePath); options.onLog?.("INFO", `Passwort-Versuch ${attempt}/${total}: archive=${archiveName}, password=`); } : undefined; @@ -3135,9 +3138,9 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ lastArchiveFinishedAt = successAt; archivePercent = 100; if (hasManyPasswords) { - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, { passwordFound: true }, { archiveDone: true, archiveSuccess: true }); - } else { - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, undefined, { archiveDone: true, archiveSuccess: true }); + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, { passwordFound: true }, { archiveDone: true, archiveSuccess: true }, archivePath); + } else { + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, undefined, { archiveDone: true, archiveSuccess: true }, archivePath); } } catch (error) { const errorText = String(error); @@ -3168,7 +3171,7 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ const tailAfter99Ms = reached99At ? (failedAt - reached99At) : -1; logger.warn(`Extract-Trace Archiv Fehler: archive=${archiveName}, totalMs=${failedAt - archiveStartedAt}, tailAfter99Ms=${tailAfter99Ms >= 0 ? tailAfter99Ms : "n/a"}, category=${errorCategory}`); lastArchiveFinishedAt = failedAt; - emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, undefined, { archiveDone: true, archiveSuccess: false }); + emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt, undefined, { archiveDone: true, archiveSuccess: false }, archivePath); if (isNoExtractorError(errorText)) { noExtractorEncountered = true; } @@ -3327,9 +3330,9 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ } const nestedStartedAt = Date.now(); let nestedPercent = 0; - emitProgress(extracted + failed, `nested: ${nestedName}`, "extracting", nestedPercent, 0); + emitProgress(extracted + failed, `nested: ${nestedName}`, "extracting", nestedPercent, 0, undefined, undefined, nestedArchive); const nestedPulse = setInterval(() => { - emitProgress(extracted + failed, `nested: ${nestedName}`, "extracting", nestedPercent, Date.now() - nestedStartedAt); + emitProgress(extracted + failed, `nested: ${nestedName}`, "extracting", nestedPercent, Date.now() - nestedStartedAt, undefined, undefined, nestedArchive); }, 1100); const hybrid = Boolean(options.hybridMode); logger.info(`Nested-Entpacke: ${nestedName} -> ${options.targetDir}${hybrid ? " (hybrid)" : ""}`); diff --git a/src/main/storage.ts b/src/main/storage.ts index 33d0e80..c658517 100644 --- a/src/main/storage.ts +++ b/src/main/storage.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { getDebridLinkApiKeyIds } from "../shared/debrid-link-keys"; import { getMegaDebridAccountIds, mergeMegaDebridCredentialPools, parseMegaDebridAccounts } from "../shared/mega-debrid-accounts"; -import { AppSettings, ArchiveOperationMetric, AudioStripSummary, BandwidthScheduleEntry, DailyStartOutcome, DebridAccountStatus, DebridFallbackProvider, DebridProvider, DownloadItem, DownloadStatus, FailurePhase, HistoryEntry, HistoryRetentionMode, LogStorageLocation, PackageEntry, PackagePriority, RemuxOperationMetric, SessionState } from "../shared/types"; +import { AppSettings, ArchiveOperationMetric, AudioStripSummary, BandwidthScheduleEntry, DailyStartOutcome, DebridAccountStatus, DebridFallbackProvider, DebridProvider, DownloadItem, DownloadStatus, FailurePhase, HistoryEntry, HistoryRetentionMode, LogStorageLocation, PACKAGE_OUTPUT_PROVENANCE_VERSION, PackageEntry, PackagePriority, RemuxOperationMetric, SessionState } from "../shared/types"; import { getProviderUsageDayKey } from "../shared/provider-daily-limits"; import { getRealDebridAccountIds, normalizeRealDebridWebAccountIds, parseRealDebridApiAccounts, serializeRealDebridApiAccounts } from "../shared/real-debrid-accounts"; import { defaultSettings } from "./constants"; @@ -971,10 +971,13 @@ export function normalizeLoadedSession(raw: unknown): SessionState { if (!id) { continue; } - const statusRaw = asText(pkg.status) as DownloadStatus; - const status: DownloadStatus = VALID_DOWNLOAD_STATUSES.has(statusRaw) ? statusRaw : "queued"; - const rawItemIds = Array.isArray(pkg.itemIds) ? pkg.itemIds : []; - packagesById[id] = { + const statusRaw = asText(pkg.status) as DownloadStatus; + const status: DownloadStatus = VALID_DOWNLOAD_STATUSES.has(statusRaw) ? statusRaw : "queued"; + const rawItemIds = Array.isArray(pkg.itemIds) ? pkg.itemIds : []; + const outputProvenance = Array.isArray(pkg.outputProvenance) + ? [...new Set(pkg.outputProvenance.map((value) => asText(value).toLowerCase()).filter((value) => /^[a-f0-9]{64}$/.test(value)))].slice(0, 1_000_000) + : []; + packagesById[id] = { id, name: asText(pkg.name) || "Paket", outputDir: asText(pkg.outputDir), @@ -1006,10 +1009,9 @@ export function normalizeLoadedSession(raw: unknown): SessionState { terminalAt: clampNumber(pkg.terminalAt, 0, 0, Number.MAX_SAFE_INTEGER), archiveOperations: normalizeArchiveOperations(pkg.archiveOperations), remuxOperations: normalizeRemuxOperations(pkg.remuxOperations), - outputCount: clampNumber(pkg.outputCount, 0, 0, 1_000_000), - outputProvenance: Array.isArray(pkg.outputProvenance) - ? [...new Set(pkg.outputProvenance.map((value) => asText(value).toLowerCase()).filter((value) => /^[a-f0-9]{64}$/.test(value)))].slice(0, 1_000_000) - : [], + outputCount: outputProvenance.length, + outputProvenanceVersion: PACKAGE_OUTPUT_PROVENANCE_VERSION, + outputProvenance, cleanupErrorCategory: asText(pkg.cleanupErrorCategory), resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER), createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER), diff --git a/src/shared/types.ts b/src/shared/types.ts index 028ce74..0c5931b 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -495,6 +495,7 @@ export interface AudioStripSummary { export type PackageResultStatus = "completed" | "partial" | "failed" | "cancelled"; export type FailurePhase = "download" | "extract" | "remux" | "cleanup" | null; +export const PACKAGE_OUTPUT_PROVENANCE_VERSION = 1; export interface ArchiveOperationMetric { id: string; @@ -588,6 +589,7 @@ export interface PackageEntry { archiveOperations?: ArchiveOperationMetric[]; remuxOperations?: RemuxOperationMetric[]; outputCount?: number; + outputProvenanceVersion?: number; outputProvenance?: string[]; cleanupErrorCategory?: string; resultGeneration?: number; diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index e4ea5d1..e6a51d1 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -15114,12 +15114,26 @@ describe("package priority ordering", () => { }); describe("package lifecycle telemetry boundaries", () => { - it("serializes provenance capture for packages sharing one extract directory", async () => { + it("captures shared-root provenance from package staging without scanning unrelated files", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-provenance-lock-")); tempDirs.push(root); const extractDir = path.join(root, "extract"); fs.mkdirSync(extractDir, { recursive: true }); - const manager = new DownloadManager(defaultSettings(), emptySession(), createStoragePaths(path.join(root, "state"))); + for (let index = 0; index < 2_000; index += 1) { + fs.writeFileSync(path.join(extractDir, `foreign-${index}.txt`), "foreign"); + } + const traversedDirectories: string[] = []; + const manager = new DownloadManager( + { ...defaultSettings(), extractConflictMode: "overwrite" }, + emptySession(), + createStoragePaths(path.join(root, "state")), + { + readOutputDirectory: async (directory: string) => { + traversedDirectories.push(path.resolve(directory)); + return fs.promises.readdir(directory, { withFileTypes: true }); + } + } as any + ); const createPackage = (id: string): PackageEntry => ({ id, name: id, @@ -15141,25 +15155,107 @@ describe("package lifecycle telemetry boundaries", () => { let enteredB = false; const state = manager as any; - const first = state.runWithPackageOutputProvenance(packageA, async () => { - fs.writeFileSync(path.join(extractDir, "package-a.mkv"), "a"); + const first = state.runWithPackageOutputProvenance(packageA, async (operationTarget = extractDir) => { + fs.writeFileSync(path.join(operationTarget, "package-a.mkv"), "a"); await gateA; }); - await vi.waitFor(() => expect(fs.existsSync(path.join(extractDir, "package-a.mkv"))).toBe(true)); - const second = state.runWithPackageOutputProvenance(packageB, async () => { + await vi.waitFor(() => expect(enteredB).toBe(false)); + const second = state.runWithPackageOutputProvenance(packageB, async (operationTarget = extractDir) => { enteredB = true; - fs.writeFileSync(path.join(extractDir, "package-b.mkv"), "b"); + fs.writeFileSync(path.join(operationTarget, "package-b.mkv"), "b"); }); - await Promise.resolve(); - - expect(enteredB).toBe(false); + await vi.waitFor(() => expect(enteredB).toBe(true)); releaseA(); await Promise.all([first, second]); expect(packageA.outputCount).toBe(1); expect(packageB.outputCount).toBe(1); + expect(traversedDirectories.length).toBeGreaterThan(0); + expect(traversedDirectories).not.toContain(path.resolve(extractDir)); + expect(traversedDirectories.length).toBeLessThanOrEqual(4); }); - it("uses item-path provenance for archive identity and leaves unknown part counts at zero", () => { + it.each([ + ["overwrite", "package", ["episode.mkv"]], + ["skip", "foreign", ["episode.mkv"]], + ["rename", "foreign", ["episode (1).mkv", "episode.mkv"]] + ] as const)("preserves %s conflicts while merging staged package outputs", async (conflictMode, expectedOriginal, expectedFiles) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `rd-output-${conflictMode}-`)); + tempDirs.push(root); + const extractDir = path.join(root, "extract"); + fs.mkdirSync(extractDir, { recursive: true }); + fs.writeFileSync(path.join(extractDir, "episode.mkv"), "foreign"); + const manager = new DownloadManager( + { ...defaultSettings(), extractConflictMode: conflictMode }, + emptySession(), + createStoragePaths(path.join(root, "state")) + ); + const pkg: PackageEntry = { + id: `conflict-${conflictMode}`, + name: `conflict-${conflictMode}`, + outputDir: path.join(root, "downloads"), + extractDir, + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + createdAt: 1_000, + updatedAt: 1_000 + }; + const state = manager as any; + + await state.runWithPackageOutputProvenance(pkg, async (operationTarget = extractDir) => { + fs.writeFileSync(path.join(operationTarget, "episode.mkv"), "package"); + }); + + expect(fs.readFileSync(path.join(extractDir, "episode.mkv"), "utf8")).toBe(expectedOriginal); + expect(fs.readdirSync(extractDir).filter((name) => name.endsWith(".mkv")).sort()).toEqual([...expectedFiles]); + expect(pkg.outputCount).toBe(conflictMode === "skip" ? 0 : 1); + }); + + it("retains partial staged outputs deterministically when extraction aborts", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-abort-")); + tempDirs.push(root); + const extractDir = path.join(root, "extract"); + fs.mkdirSync(extractDir, { recursive: true }); + const traversedDirectories: string[] = []; + const manager = new DownloadManager( + defaultSettings(), + emptySession(), + createStoragePaths(path.join(root, "state")), + { + readOutputDirectory: async (directory: string) => { + traversedDirectories.push(path.resolve(directory)); + return fs.promises.readdir(directory, { withFileTypes: true }); + } + } as any + ); + const pkg: PackageEntry = { + id: "aborted-output", + name: "aborted-output", + outputDir: path.join(root, "downloads"), + extractDir, + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + createdAt: 1_000, + updatedAt: 1_000 + }; + const state = manager as any; + + await expect(state.runWithPackageOutputProvenance(pkg, async (operationTarget = extractDir) => { + fs.writeFileSync(path.join(operationTarget, "partial.mkv"), "partial"); + throw new Error("aborted:extract"); + })).rejects.toThrow("aborted:extract"); + + expect(fs.readFileSync(path.join(extractDir, "partial.mkv"), "utf8")).toBe("partial"); + expect(pkg.outputCount).toBe(1); + expect(traversedDirectories.length).toBeGreaterThan(0); + expect(traversedDirectories).not.toContain(path.resolve(extractDir)); + expect(fs.readdirSync(extractDir).filter((name) => name.startsWith(".rd-output-"))).toEqual([]); + }); + + it("uses normalized nested item paths for archive identity and leaves empty item provenance at zero", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-archive-identity-")); tempDirs.push(root); const session = emptySession(); @@ -15196,28 +15292,104 @@ describe("package lifecycle telemetry boundaries", () => { createdAt: 1_000, updatedAt: 1_000 }); - const progress = (current: number) => ({ - current, + const progress = (archivePath: string) => ({ + current: 0, total: 3, percent: 100, archiveName: "episode.rar", + archivePath, archivePercent: 100, elapsedMs: 1_000, archiveDone: true, archiveSuccess: true }); const state = manager as any; + const itemA = item("item-a", "season-a"); + const itemB = item("item-b", "season-b"); + const archiveAPath = path.join(pkg.outputDir, "season-a", "episode.rar"); + const archiveBPath = path.join(pkg.outputDir, "season-b", "episode.rar"); + const archiveAItems = (resolveArchiveItemsFromList as any)("episode.rar", [itemA, itemB], archiveAPath); + const archiveBItems = (resolveArchiveItemsFromList as any)("episode.rar", [itemA, itemB], archiveBPath); + const unresolvedItem = { ...item("unresolved", "season-c"), targetPath: "" }; - state.recordArchiveOperation(pkg, progress(0), [item("item-a", "season-a")]); - state.recordArchiveOperation(pkg, progress(1), [item("item-b", "season-b")]); - state.recordArchiveOperation(pkg, { ...progress(2), archiveName: "unresolved.rar" }, []); + state.recordArchiveOperation(pkg, progress(archiveAPath), archiveAItems); + state.recordArchiveOperation(pkg, progress(archiveBPath), archiveBItems); + state.recordArchiveOperation(pkg, { ...progress(""), archiveName: "unresolved.rar" }, [unresolvedItem]); const operations = pkg.archiveOperations || []; expect(operations).toHaveLength(3); expect(new Set(operations.map((operation) => operation.id))).toHaveLength(3); + expect(operations.map((operation) => operation.itemIds)).toEqual([["item-a"], ["item-b"], []]); expect(operations.map((operation) => operation.partCount)).toEqual([1, 1, 0]); }); + it("keeps foreign post-process waiters reserved when stopping another run", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-run-owned-slots-")); + tempDirs.push(root); + const session = emptySession(); + const createPackage = (id: string): PackageEntry => ({ + id, + name: id, + outputDir: path.join(root, "downloads", id), + extractDir: path.join(root, "extract", id), + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + createdAt: 1_000, + updatedAt: 1_000 + }); + const packageA = createPackage("run-a-package"); + const packageB = createPackage("run-b-package"); + session.packages[packageA.id] = packageA; + session.packages[packageB.id] = packageB; + session.packageOrder = [packageA.id, packageB.id]; + const manager = new DownloadManager( + { ...defaultSettings(), maxParallelExtract: 1 }, + session, + createStoragePaths(path.join(root, "state")) + ); + const state = manager as any; + const runA = state.createRunContext([packageA.id], 1_000, false); + const runB = state.beginActiveRunContext([packageB.id], 2_000); + session.running = true; + session.runStartedAt = 2_000; + state.runPackageIds = new Set([packageB.id]); + state.runItemIds = new Set(["run-b-item"]); + let concurrent = 1; + let peak = concurrent; + let foreignResolved = false; + + await state.acquirePostProcessSlot("active-a", runA.id); + const foreignWaiter = state.acquirePostProcessSlot("waiting-a", runA.id).then((acquired: boolean | undefined) => { + foreignResolved = true; + if (acquired !== false) { + concurrent += 1; + peak = Math.max(peak, concurrent); + } + return acquired; + }); + const stoppedWaiter = state.acquirePostProcessSlot("waiting-b", runB.id); + manager.stop(); + const stoppedResult = await stoppedWaiter; + await Promise.resolve(); + + expect(stoppedResult).toBe(false); + expect(foreignResolved).toBe(false); + expect(state.packagePostProcessActive).toBe(1); + + concurrent -= 1; + state.releasePostProcessSlot(); + const foreignResult = await foreignWaiter; + expect(foreignResult).toBe(true); + expect(state.packagePostProcessActive).toBe(1); + expect(peak).toBe(1); + + concurrent -= 1; + state.releasePostProcessSlot(); + expect(state.packagePostProcessActive).toBe(0); + }); + it("records queued, slot start and terminal timestamps around real post-processing", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-lifecycle-boundaries-")); tempDirs.push(root); diff --git a/tests/notify-hooks.test.ts b/tests/notify-hooks.test.ts index fe517b1..7f131f5 100644 --- a/tests/notify-hooks.test.ts +++ b/tests/notify-hooks.test.ts @@ -622,6 +622,41 @@ describe("authoritative run completion", () => { expect([...state.finalizedPackageResults.keys()].filter((key) => key.startsWith(`${pkg.id}:`))).toEqual([`${pkg.id}:2`]); }); + it("prunes only the removed package result generations and provenance state", () => { + const { manager, session } = setup(); + const packageA = addPackage(session, ["completed"], "removed-package"); + const packageB = addPackage(session, ["completed"], "retained-package"); + packageA.outputCount = 1; + packageA.outputProvenance = ["a".repeat(64)]; + packageB.outputCount = 1; + packageB.outputProvenance = ["b".repeat(64)]; + const state = internal(manager); + const resultA = { packageId: packageA.id }; + const resultB = { packageId: packageB.id }; + state.finalizedPackageResults.set(`${packageA.id}:1`, resultA); + state.finalizedPackageResults.set(`${packageA.id}:2`, resultA); + state.finalizedPackageResults.set(`${packageB.id}:1`, resultB); + state.standalonePackageResults.add(`${packageA.id}:2`); + state.standalonePackageResults.add(`${packageB.id}:1`); + state.suppressedPackageResults.add(`${packageA.id}:1`); + state.suppressedPackageResults.add(`${packageB.id}:1`); + state.successDigestResults.set(`${packageA.id}:2`, { generation: 2, result: resultA }); + state.successDigestResults.set(`${packageB.id}:1`, { generation: 1, result: resultB }); + const context = state.createRunContext([packageA.id, packageB.id], 1_000, false); + + state.removePackageFromSession(packageA.id, [...packageA.itemIds]); + + expect([...state.finalizedPackageResults.keys()]).toEqual([`${packageB.id}:1`]); + expect([...state.standalonePackageResults]).toEqual([`${packageB.id}:1`]); + expect([...state.suppressedPackageResults]).toEqual([`${packageB.id}:1`]); + expect([...state.successDigestResults.keys()]).toEqual([`${packageB.id}:1`]); + expect(context.packageGenerations).toEqual(new Map([[packageB.id, 1]])); + expect(packageA.outputCount).toBe(0); + expect(packageA.outputProvenance).toEqual([]); + expect(packageB.outputCount).toBe(1); + expect(packageB.outputProvenance).toEqual(["b".repeat(64)]); + }); + it("tracks a main postprocess task created by triggerPendingExtractions after start begins", async () => { const { manager, session, events, history } = setup({ autoExtract: true }); const pkg = addPackage(session); diff --git a/tests/storage.test.ts b/tests/storage.test.ts index 0db7385..9110556 100644 --- a/tests/storage.test.ts +++ b/tests/storage.test.ts @@ -1200,6 +1200,7 @@ describe("settings storage", () => { }], remuxOperations: [], outputCount: 1, + outputProvenanceVersion: 1, outputProvenance: ["a".repeat(64), "invalid", "a".repeat(64)], cleanupErrorCategory: "", createdAt: 1_000, @@ -1227,11 +1228,49 @@ describe("settings storage", () => { archiveOperations: [expect.objectContaining({ id: "archive-1", durationMs: 4_000 })], remuxOperations: [], outputCount: 1, + outputProvenanceVersion: 1, outputProvenance: ["a".repeat(64)], cleanupErrorCategory: "" })); }); + it("invalidates an unversioned legacy package output count on load", () => { + const normalized = normalizeLoadedSession({ + version: 2, + packageOrder: ["legacy-output-count"], + packages: { + "legacy-output-count": { + id: "legacy-output-count", + name: "Legacy output count", + outputDir: "C:\\Downloads\\Legacy", + extractDir: "C:\\Downloads\\Shared", + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + outputCount: 48_000, + createdAt: 1_000, + updatedAt: 2_000 + } + }, + items: {}, + runStartedAt: 0, + totalDownloadedBytes: 0, + summaryText: "", + reconnectUntil: 0, + reconnectReason: "", + paused: false, + running: false, + updatedAt: 2_000 + }); + + expect(normalized.packages["legacy-output-count"]).toEqual(expect.objectContaining({ + outputCount: 0, + outputProvenanceVersion: 1, + outputProvenance: [] + })); + }); + it("skips adding persisted history entries when history retention is never", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rd-store-")); tempDirs.push(dir);