diff --git a/src/main/cleanup.ts b/src/main/cleanup.ts index 2ab0cef..a5928d2 100644 --- a/src/main/cleanup.ts +++ b/src/main/cleanup.ts @@ -2,11 +2,58 @@ import fs from "node:fs"; import path from "node:path"; import { ARCHIVE_TEMP_EXTENSIONS, LINK_ARTIFACT_EXTENSIONS, MAX_LINK_ARTIFACT_BYTES, RAR_SPLIT_RE, SAMPLE_DIR_NAMES, SAMPLE_TOKEN_RE, SAMPLE_VIDEO_EXTENSIONS } from "./constants"; -async function yieldToLoop(): Promise { +async function yieldToLoop(): Promise { await new Promise((resolve) => { setTimeout(resolve, 0); }); -} +} + +async function isDownloadLinkArtifact(filePath: string): Promise { + const fileName = path.basename(filePath); + const ext = path.extname(fileName).toLowerCase(); + const name = fileName.toLowerCase(); + if (LINK_ARTIFACT_EXTENSIONS.has(ext)) { + return true; + } + if (![".txt", ".html", ".htm", ".nfo"].includes(ext) + || !/[._\- ](links?|downloads?|urls?|dlc)([._\- ]|$)/i.test(name)) { + return false; + } + try { + const stat = await fs.promises.lstat(filePath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_LINK_ARTIFACT_BYTES) { + return false; + } + const text = await fs.promises.readFile(filePath, "utf8"); + return /https?:\/\//i.test(text); + } catch { + return false; + } +} + +async function removeEmptyParentChains(rootDir: string, parents: ReadonlySet): Promise { + const rootPath = path.resolve(rootDir); + const candidates = new Set(); + for (const parent of parents) { + let current = path.resolve(parent); + while (current !== rootPath && current.startsWith(`${rootPath}${path.sep}`)) { + candidates.add(current); + current = path.dirname(current); + } + } + let removed = 0; + for (const directory of [...candidates].sort((left, right) => right.length - left.length)) { + try { + const entries = await fs.promises.readdir(directory); + if (entries.length === 0) { + await fs.promises.rmdir(directory); + removed += 1; + } + } catch { + } + } + return removed; +} export function isArchiveOrTempFile(filePath: string): boolean { const lowerName = path.basename(filePath).toLowerCase(); @@ -126,22 +173,7 @@ export async function removeDownloadLinkArtifacts( continue; } - const ext = path.extname(entry.name).toLowerCase(); - const name = entry.name.toLowerCase(); - let shouldDelete = LINK_ARTIFACT_EXTENSIONS.has(ext); - if (!shouldDelete && [".txt", ".html", ".htm", ".nfo"].includes(ext)) { - if (/[._\- ](links?|downloads?|urls?|dlc)([._\- ]|$)/i.test(name)) { - try { - const stat = await fs.promises.stat(full); - if (stat.size <= MAX_LINK_ARTIFACT_BYTES) { - const text = await fs.promises.readFile(full, "utf8"); - shouldDelete = /https?:\/\//i.test(text); - } - } catch { - shouldDelete = false; - } - } - } + const shouldDelete = await isDownloadLinkArtifact(full); if (shouldDelete) { try { @@ -153,9 +185,35 @@ export async function removeDownloadLinkArtifacts( } } return removed; -} +} + +export async function removeDownloadLinkArtifactsFromScope( + outputFiles: readonly string[], + options: { shouldAbort?: () => boolean; rootDir?: string } = {} +): Promise { + let removed = 0; + const parents = new Set(); + for (const outputFile of outputFiles) { + if (options.shouldAbort?.()) { + return removed; + } + if (!await isDownloadLinkArtifact(outputFile)) { + continue; + } + try { + await fs.promises.rm(outputFile, { force: true }); + parents.add(path.dirname(outputFile)); + removed += 1; + } catch { + } + } + if (options.rootDir) { + await removeEmptyParentChains(options.rootDir, parents); + } + return removed; +} -export async function removeSampleArtifacts( +export async function removeSampleArtifacts( extractDir: string, options: { shouldAbort?: () => boolean } = {} ): Promise<{ files: number; dirs: number }> { @@ -263,4 +321,55 @@ export async function removeSampleArtifacts( } return { files: removedFiles, dirs: removedDirs }; -} +} + +export async function removeSampleArtifactsFromScope( + outputFiles: readonly string[], + options: { shouldAbort?: () => boolean; rootDir?: string } = {} +): Promise<{ files: number; dirs: number }> { + let removedFiles = 0; + const candidateParents = new Set(); + for (const outputFile of outputFiles) { + if (options.shouldAbort?.()) { + return { files: removedFiles, dirs: 0 }; + } + const fileName = path.basename(outputFile); + const stem = path.parse(fileName).name.toLowerCase(); + const ext = path.extname(fileName).toLowerCase(); + const parentDir = path.dirname(outputFile); + const inSampleDir = SAMPLE_DIR_NAMES.has(path.basename(parentDir).toLowerCase()); + if (!inSampleDir && !(SAMPLE_VIDEO_EXTENSIONS.has(ext) && SAMPLE_TOKEN_RE.test(stem))) { + continue; + } + try { + const stat = await fs.promises.lstat(outputFile); + if (!stat.isFile() || stat.isSymbolicLink()) { + continue; + } + await fs.promises.rm(outputFile, { force: true }); + removedFiles += 1; + if (inSampleDir) { + candidateParents.add(parentDir); + } + } catch { + } + } + let removedDirs = 0; + for (const parentDir of candidateParents) { + if (options.shouldAbort?.()) { + return { files: removedFiles, dirs: removedDirs }; + } + try { + const entries = await fs.promises.readdir(parentDir); + if (entries.length === 0) { + await fs.promises.rmdir(parentDir); + removedDirs += 1; + } + } catch { + } + } + if (options.rootDir) { + removedDirs += await removeEmptyParentChains(options.rootDir, candidateParents); + } + return { files: removedFiles, dirs: removedDirs }; +} diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 8fe052d..5785da6 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -61,7 +61,7 @@ function releaseTlsSkip(): void { delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; } } -import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup"; +import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifactsFromScope, removeSampleArtifactsFromScope } from "./cleanup"; import { planDownloadCompletion, reconcileFinalizedSize, validateDownloadedFileCompletion } from "./download-completion"; import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkDdownloadOnline, checkOneFichierLinks, checkRapidgatorOnline, fetchAllDebridHostInfo, filenameFromDdownloadUrlPath, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, isDdownloadLink, isOneFichierLink, isProviderDisabledForSelection, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown, type DdownloadCheckResult, type OneFichierCheckResult } from "./debrid"; import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, detectArchiveSignature, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, resetExtractorCachesForPasswordChange, type ExtractArchiveFailureInfo, type ExtractProgressUpdate } from "./extractor"; @@ -82,6 +82,7 @@ import { compactErrorText, ensureDirPath, filenameFromUrl, formatEta, humanSize, import { mergeKnownTotalBytes } from "./download-size"; import { DiskCapacityError, DiskReservationCoordinator, type DiskReservationLease } from "./disk-space"; import { createRendererState } from "./renderer-state"; +import { PackageOutputScope } from "./package-output-scope"; import { RollingAccountStatisticsAccumulator, addStatisticsActiveIntervalInPlace, @@ -477,7 +478,6 @@ type DownloadManagerOptions = { onHistoryEntry?: HistoryEntryCallback; enqueueNotification?: (event: NotificationEvent) => Promise; protectEmptyClobber?: boolean; - readOutputDirectory?: (directory: string) => Promise; }; type RunLifecycleContext = { @@ -1930,7 +1930,7 @@ export class DownloadManager extends EventEmitter { private cleanupQueue: Promise = Promise.resolve(); - private packageOutputProvenanceTails = new Map>(); + private packageOutputScopes = new Map(); private packagePostProcessQueue: Promise = Promise.resolve(); @@ -2069,8 +2069,6 @@ export class DownloadManager extends EventEmitter { private enqueueNotificationCallback?: (event: NotificationEvent) => Promise; - private readOutputDirectoryFn: (directory: string) => Promise; - public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) { super(); this.settings = settings; @@ -2100,7 +2098,6 @@ export class DownloadManager extends EventEmitter { this.invalidateMegaSessionFn = options.invalidateMegaSession; this.onHistoryEntryCallback = options.onHistoryEntry; this.enqueueNotificationCallback = options.enqueueNotification; - this.readOutputDirectoryFn = options.readOutputDirectory || ((directory) => fs.promises.readdir(directory, { withFileTypes: true })); logger.info(`DownloadManager Init: ${Object.keys(this.session.packages).length} Pakete, ${this.itemCount} Items, cleanupPolicy=${this.settings.completedCleanupPolicy}`); for (const pkg of Object.values(this.session.packages)) { this.ensurePackageLogForPackage(pkg); @@ -3512,10 +3509,10 @@ export class DownloadManager extends EventEmitter { continue; } - const extractDirKey = pathKey(pkg.extractDir); - const hasExtractedFiles = hasFilesByExtractDir.has(extractDirKey) - ? Boolean(hasFilesByExtractDir.get(extractDirKey)) - : await this.directoryHasAnyFiles(pkg.extractDir); + const extractDirKey = pathKey(pkg.extractDir); + const hasExtractedFiles = hasFilesByExtractDir.has(extractDirKey) + ? Boolean(hasFilesByExtractDir.get(extractDirKey)) + : this.getPackageOutputScope(pkg).completeFiles().some((filePath) => isPathInsideDir(filePath, pkg.extractDir)); if (!hasFilesByExtractDir.has(extractDirKey)) { hasFilesByExtractDir.set(extractDirKey, hasExtractedFiles); } @@ -4268,7 +4265,8 @@ export class DownloadManager extends EventEmitter { const hasExtractMarker = items.some((item) => isExtractedLabel(item.fullStatus)); const extractDirIsUnique = (extractDirUsage.get(pathKey(pkg.extractDir)) || 0) === 1; - const hasExtractedOutput = extractDirIsUnique && await this.directoryHasAnyFiles(pkg.extractDir); + const hasExtractedOutput = extractDirIsUnique + && this.getPackageOutputScope(pkg).completeFiles().some((filePath) => isPathInsideDir(filePath, pkg.extractDir)); if (!hasExtractMarker && !hasExtractedOutput) { continue; } @@ -4384,103 +4382,6 @@ export class DownloadManager extends EventEmitter { return false; } - private async listStagedOutputFiles(stagingDir: string): Promise { - const files: string[] = []; - const stack = [stagingDir]; - while (stack.length > 0) { - const current = stack.pop() as string; - let entries: fs.Dirent[] = []; - try { - entries = await this.readOutputDirectoryFn(current); - } catch { - continue; - } - for (const entry of entries) { - const fullPath = path.join(current, entry.name); - if (entry.isSymbolicLink()) { - continue; - } - if (entry.isDirectory()) { - stack.push(fullPath); - } else if (entry.isFile()) { - files.push(fullPath); - } - } - } - return files.sort((left, right) => path.relative(stagingDir, left).localeCompare(path.relative(stagingDir, right))); - } - - private async resolveStagedOutputDestination(targetDir: string, relativePath: string): Promise { - const targetRoot = path.resolve(targetDir); - const destination = path.resolve(targetRoot, relativePath); - if (destination !== targetRoot && !destination.startsWith(`${targetRoot}${path.sep}`)) { - throw new Error(`Ungültiger Staging-Ausgabepfad: ${relativePath}`); - } - let existing: fs.Stats | null = null; - try { - existing = await fs.promises.lstat(destination); - } catch { - } - if (!existing) { - return destination; - } - if (this.settings.extractConflictMode === "skip" || this.settings.extractConflictMode === "ask") { - return null; - } - if (this.settings.extractConflictMode === "overwrite") { - return existing.isFile() ? destination : null; - } - const parsed = path.parse(destination); - for (let index = 1; index <= 10_000; index += 1) { - const candidate = path.join(parsed.dir, `${parsed.name} (${index})${parsed.ext}`); - try { - await fs.promises.lstat(candidate); - } catch { - return candidate; - } - } - throw new Error(`Staging-Rename-Limit erreicht für ${relativePath}`); - } - - private async moveStagedOutputFile(sourcePath: string, destinationPath: string): Promise { - await fs.promises.mkdir(path.dirname(destinationPath), { recursive: true }); - try { - await fs.promises.rename(sourcePath, destinationPath); - return; - } catch (error) { - const code = String((error as NodeJS.ErrnoException)?.code || ""); - if (this.settings.extractConflictMode !== "overwrite" || (code !== "EEXIST" && code !== "EPERM" && code !== "EACCES")) { - throw error; - } - } - const displacedPath = path.join(path.dirname(destinationPath), `.rd-replace-${uuidv4()}`); - await fs.promises.rename(destinationPath, displacedPath); - try { - await fs.promises.rename(sourcePath, destinationPath); - } catch (error) { - await fs.promises.rename(displacedPath, destinationPath); - throw error; - } - await fs.promises.rm(displacedPath, { force: true }); - } - - private async mergeStagedPackageOutputs(stagingDir: string, targetDir: string): Promise { - const movedOutputs: string[] = []; - const stagedFiles = await this.listStagedOutputFiles(stagingDir); - for (const stagedFile of stagedFiles) { - const relativePath = path.relative(stagingDir, stagedFile); - const destination = await this.resolveStagedOutputDestination(targetDir, relativePath); - if (!destination) { - continue; - } - await this.moveStagedOutputFile(stagedFile, destination); - if (!isArchiveLikePath(destination) && !isIgnorableEmptyDirFileName(path.basename(destination))) { - movedOutputs.push(destination); - } - } - return movedOutputs; - } - private normalizePackageProvenancePath(pkg: PackageEntry, sourcePath: string): string { const absolutePath = path.resolve(sourcePath); for (const rootDir of [pkg.outputDir, pkg.extractDir]) { @@ -4490,81 +4391,128 @@ export class DownloadManager extends EventEmitter { } const relativePath = path.relative(path.resolve(root), absolutePath); if (!relativePath.startsWith(`..${path.sep}`) && relativePath !== ".." && !path.isAbsolute(relativePath)) { - const segments = relativePath.replace(/\\/g, "/").split("/"); - if (/^\.rd-output-[^/]+$/i.test(segments[0] || "")) { - segments.shift(); - } - return segments.join("/").toLocaleLowerCase("de-DE"); + return relativePath.replace(/\\/g, "/").toLocaleLowerCase("de-DE"); } } return absolutePath.replace(/\\/g, "/").toLocaleLowerCase("de-DE"); } - private recordPackageOutputFiles(pkg: PackageEntry, outputFiles: readonly string[]): void { - const provenance = new Set(pkg.outputProvenance || []); - for (const outputFile of outputFiles) { + private syncPackageOutputScope(pkg: PackageEntry, scope: PackageOutputScope): void { + scope.pruneMissing(); + const provenance = new Set(); + for (const outputFile of scope.files()) { const key = this.normalizePackageProvenancePath(pkg, outputFile); provenance.add(createHash("sha256").update(key).digest("hex")); } pkg.outputProvenance = [...provenance]; pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION; pkg.outputCount = provenance.size; + pkg.outputRecords = scope.records(); } - private async runWithPackageOutputProvenance(pkg: PackageEntry, operation: (targetDir: string) => Promise): Promise { - const key = pathKey(pkg.extractDir); + private getPackageOutputScope(pkg: PackageEntry): PackageOutputScope { + const current = this.packageOutputScopes.get(pkg.id); + if (current) { + return current; + } + const roots = [pkg.outputDir, pkg.extractDir, String(this.settings.mkvLibraryDir || "").trim()].filter(Boolean); + const scope = new PackageOutputScope(roots); + if (pkg.outputProvenanceVersion === PACKAGE_OUTPUT_PROVENANCE_VERSION) { + for (const record of pkg.outputRecords || []) { + try { + scope.add(record); + } catch { + } + } + } + this.packageOutputScopes.set(pkg.id, scope); + return scope; + } + + private async adoptLegacyPackageOutputsIfExclusive(pkg: PackageEntry, scope: PackageOutputScope): Promise { + if (pkg.outputScopeAdopted || scope.records().length > 0) { + pkg.outputScopeAdopted = true; + return; + } + pkg.outputScopeAdopted = true; + if (pkg.outputProvenanceVersion !== undefined + && pkg.outputProvenanceVersion !== PACKAGE_OUTPUT_PROVENANCE_VERSION) { + return; + } + const packageExclusive = (this.settings.createExtractSubfolder || this.isPackageSpecificExtractDir(pkg)) + && !this.isExtractDirSharedWithOtherPackages(pkg.id, pkg.extractDir); + if (!packageExclusive || !await this.existsAsync(pkg.extractDir)) { + return; + } + const candidates: string[] = []; + const stack = [pkg.extractDir]; + let inspected = 0; + let overflow = false; + while (stack.length > 0 && !overflow) { + const current = stack.pop() as string; + let entries: fs.Dirent[] = []; + try { + entries = await fs.promises.readdir(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + inspected += 1; + if (inspected > 100_000) { + overflow = true; + break; + } + const fullPath = path.join(current, entry.name); + if (entry.isSymbolicLink()) { + continue; + } + if (entry.isDirectory()) { + if (!/^\.rd-(?:output|replace)-/i.test(entry.name) && !/^\.rd-trash$/i.test(entry.name)) { + stack.push(fullPath); + } + } else if (entry.isFile() + && !/^\.rd-(?:output|replace)-/i.test(entry.name) + && !/^\.rd_extract_progress(?:_[^.]+)?\.json$/i.test(entry.name) + && !isIgnorableEmptyDirFileName(entry.name)) { + candidates.push(fullPath); + } + } + } + if (overflow) { + return; + } + const archivePath = path.resolve(pkg.outputDir || pkg.extractDir, ".legacy-output-scope"); + for (const outputPath of candidates) { + const entryPath = path.relative(pkg.extractDir, outputPath).replace(/\\/g, "/"); + try { + scope.add({ + version: 1, + archivePath, + entryPath, + outputPath, + state: "complete", + disposition: "written" + }); + } catch { + } + } + this.syncPackageOutputScope(pkg, scope); + } + + private async runWithPackageOutputProvenance( + pkg: PackageEntry, + operation: (targetDir: string, scope: PackageOutputScope) => Promise + ): Promise { const packageWasInSession = this.session.packages[pkg.id] === pkg; - const previous = this.packageOutputProvenanceTails.get(key) || Promise.resolve(); - let release!: () => void; - const current = new Promise((resolve) => { - release = resolve; - }); - this.packageOutputProvenanceTails.set(key, current); - let stagingDir = ""; - let result: T | undefined; - let operationError: unknown; - let mergeError: unknown; - let mergeTurnReached = false; + const scope = this.getPackageOutputScope(pkg); try { await fs.promises.mkdir(pkg.extractDir, { recursive: true }); - stagingDir = await fs.promises.mkdtemp(path.join(pkg.extractDir, ".rd-output-")); - try { - result = await operation(stagingDir); - } catch (error) { - operationError = error; - } - await previous; - mergeTurnReached = true; - try { - if (!packageWasInSession || this.session.packages[pkg.id] === pkg) { - const outputFiles = await this.mergeStagedPackageOutputs(stagingDir, pkg.extractDir); - this.recordPackageOutputFiles(pkg, outputFiles); - } - } catch (error) { - mergeError = error; - } + return await operation(pkg.extractDir, scope); } finally { - try { - if (stagingDir) { - await fs.promises.rm(stagingDir, { recursive: true, force: true }); - } - } finally { - if (!mergeTurnReached) { - await previous; - } - release(); - if (this.packageOutputProvenanceTails.get(key) === current) { - this.packageOutputProvenanceTails.delete(key); - } + if (!packageWasInSession || this.session.packages[pkg.id] === pkg) { + this.syncPackageOutputScope(pkg, scope); } } - if (operationError) { - throw operationError; - } - if (mergeError) { - throw mergeError; - } - return result as T; } private async removeEmptyDirectoryTree(rootDir: string): Promise { @@ -4622,15 +4570,10 @@ export class DownloadManager extends EventEmitter { return removed; } - private async collectFilesByExtensions(rootDir: string, extensions: Set): Promise { - if (!rootDir || extensions.size === 0) { - return []; - } - try { - await fs.promises.access(rootDir); - } catch { - return []; - } + private async collectFilesByExtensions(scope: PackageOutputScope, extensions: Set): Promise { + if (extensions.size === 0) { + return []; + } const normalizedExtensions = new Set(); for (const extension of extensions) { @@ -4643,48 +4586,26 @@ export class DownloadManager extends EventEmitter { return []; } - const files: string[] = []; - const stack = [rootDir]; - while (stack.length > 0) { - const current = stack.pop() as string; - let entries: fs.Dirent[] = []; - try { - entries = await fs.promises.readdir(current, { withFileTypes: true }); - } catch { - continue; - } - - for (const entry of entries) { - const fullPath = path.join(current, entry.name); - if (entry.isSymbolicLink()) { - continue; - } - if (entry.isDirectory()) { - stack.push(fullPath); - continue; - } - if (!entry.isFile()) { - continue; - } - // Never collect our own remux temp/orphan sidecars (~rd.): a - // partial file left by a crash mid-remux must not be swept into the library. - if (entry.name.startsWith("~rd")) { - continue; - } - const extension = path.extname(entry.name).toLowerCase(); - if (!normalizedExtensions.has(extension)) { - continue; - } - files.push(fullPath); - } - } - - return files; - } - - private async collectVideoFiles(rootDir: string): Promise { - return await this.collectFilesByExtensions(rootDir, SAMPLE_VIDEO_EXTENSIONS); - } + const files: string[] = []; + for (const filePath of scope.completeFiles()) { + const fileName = path.basename(filePath); + if (fileName.startsWith("~rd") || !normalizedExtensions.has(path.extname(fileName).toLowerCase())) { + continue; + } + try { + const stat = await fs.promises.lstat(filePath); + if (stat.isFile() && !stat.isSymbolicLink()) { + files.push(filePath); + } + } catch { + } + } + return files; + } + + private async collectVideoFiles(scope: PackageOutputScope): Promise { + return await this.collectFilesByExtensions(scope, SAMPLE_VIDEO_EXTENSIONS); + } private async existsAsync(filePath: string): Promise { try { @@ -4777,10 +4698,11 @@ export class DownloadManager extends EventEmitter { await fs.promises.rm(sourceFsPath, { force: true }); } - private async renameCompanionFiles( - sourceVideoPath: string, - targetVideoPath: string, - pkg?: PackageEntry + private async renameCompanionFiles( + sourceVideoPath: string, + targetVideoPath: string, + scope: PackageOutputScope, + pkg?: PackageEntry ): Promise { const COMPANION_EXTENSIONS = new Set([".srt", ".ass", ".ssa", ".sub", ".idx", ".vtt", ".smi", ".nfo"]); const sourceDir = path.dirname(sourceVideoPath); @@ -4790,17 +4712,9 @@ export class DownloadManager extends EventEmitter { if (!sourceVideoBase || !targetVideoBase || sourceVideoBase === targetVideoBase) { return; } - let entries: fs.Dirent[]; - try { - entries = await fs.promises.readdir(sourceDir, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - if (!entry.isFile() || entry.isSymbolicLink()) { - continue; - } - const entryName = entry.name; + const companionFiles = scope.completeFiles().filter((filePath) => pathKey(path.dirname(filePath)) === pathKey(sourceDir)); + for (const sourceCompanionPath of companionFiles) { + const entryName = path.basename(sourceCompanionPath); const entryExt = path.extname(entryName).toLowerCase(); if (!COMPANION_EXTENSIONS.has(entryExt)) { continue; @@ -4813,13 +4727,13 @@ export class DownloadManager extends EventEmitter { } const suffixAfterBase = isExactMatch ? "" : entryBase.slice(sourceVideoBase.length); const newCompanionName = `${targetVideoBase}${suffixAfterBase}${entryExt}`; - const sourceCompanionPath = path.join(sourceDir, entryName); - const targetCompanionPath = path.join(targetDir, newCompanionName); + const targetCompanionPath = path.join(targetDir, newCompanionName); if (sourceCompanionPath === targetCompanionPath) { continue; } try { - await this.renamePathWithExdevFallback(sourceCompanionPath, targetCompanionPath, { label: "companion" }); + await this.renamePathWithExdevFallback(sourceCompanionPath, targetCompanionPath, { label: "companion" }); + scope.replacePath(sourceCompanionPath, targetCompanionPath); logger.info(`Auto-Rename Companion: ${entryName} -> ${newCompanionName}`); if (pkg) { this.logPackageForPackage(pkg, "INFO", "Auto-Rename Companion umbenannt", { @@ -4833,10 +4747,11 @@ export class DownloadManager extends EventEmitter { } } - private async moveCompanionFiles( - sourceVideoPath: string, - targetVideoPath: string, - pkg?: PackageEntry + private async moveCompanionFiles( + sourceVideoPath: string, + targetVideoPath: string, + scope: PackageOutputScope, + pkg?: PackageEntry ): Promise { const COMPANION_EXTENSIONS = new Set([".srt", ".ass", ".ssa", ".sub", ".idx", ".vtt", ".smi"]); const sourceDir = path.dirname(sourceVideoPath); @@ -4846,17 +4761,9 @@ export class DownloadManager extends EventEmitter { if (!sourceVideoBase || !targetVideoBase) { return; } - let entries: fs.Dirent[]; - try { - entries = await fs.promises.readdir(sourceDir, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - if (!entry.isFile() || entry.isSymbolicLink()) { - continue; - } - const entryName = entry.name; + const companionFiles = scope.completeFiles().filter((filePath) => pathKey(path.dirname(filePath)) === pathKey(sourceDir)); + for (const sourceCompanionPath of companionFiles) { + const entryName = path.basename(sourceCompanionPath); const entryExt = path.extname(entryName).toLowerCase(); if (!COMPANION_EXTENSIONS.has(entryExt)) { continue; @@ -4869,13 +4776,13 @@ export class DownloadManager extends EventEmitter { } const suffixAfterBase = isExactMatch ? "" : entryBase.slice(sourceVideoBase.length); const newCompanionName = `${targetVideoBase}${suffixAfterBase}${entryExt}`; - const sourceCompanionPath = path.join(sourceDir, entryName); - const targetCompanionPath = path.join(targetDir, newCompanionName); + const targetCompanionPath = path.join(targetDir, newCompanionName); if (sourceCompanionPath === targetCompanionPath) { continue; } try { - await this.moveFileWithExdevFallback(sourceCompanionPath, targetCompanionPath); + await this.moveFileWithExdevFallback(sourceCompanionPath, targetCompanionPath); + scope.replacePath(sourceCompanionPath, targetCompanionPath); logger.info(`MKV-Move Companion: ${entryName} -> ${newCompanionName}`); if (pkg) { this.logPackageForPackage(pkg, "INFO", "Companion mit-verschoben", { @@ -5005,31 +4912,33 @@ export class DownloadManager extends EventEmitter { }); } - private async autoRenameExtractedVideoFiles( - extractDir: string, - pkg?: PackageEntry, + private async autoRenameExtractedVideoFiles( + extractDir: string, + scope: PackageOutputScope, + pkg?: PackageEntry, shouldAbort?: () => boolean, treatFilesAsStable = false ): Promise { if (!pkg) { - return this.autoRenameExtractedVideoFilesImpl(extractDir, undefined, shouldAbort, treatFilesAsStable); + return this.autoRenameExtractedVideoFilesImpl(extractDir, scope, undefined, shouldAbort, treatFilesAsStable); } return this.chainPackageFileOp(pkg.id, () => - this.autoRenameExtractedVideoFilesImpl(extractDir, pkg, shouldAbort, treatFilesAsStable) + this.autoRenameExtractedVideoFilesImpl(extractDir, scope, pkg, shouldAbort, treatFilesAsStable) ); } - private async keepGermanAudioOnly( - extractDir: string, - pkg?: PackageEntry, + private async keepGermanAudioOnly( + extractDir: string, + scope: PackageOutputScope, + pkg?: PackageEntry, shouldAbort?: () => boolean, signal?: AbortSignal ): Promise { if (!pkg) { - return this.keepGermanAudioOnlyImpl(extractDir, undefined, shouldAbort, signal); + return this.keepGermanAudioOnlyImpl(extractDir, scope, undefined, shouldAbort, signal); } return this.chainPackageFileOp(pkg.id, () => - this.keepGermanAudioOnlyImpl(extractDir, pkg, shouldAbort, signal) + this.keepGermanAudioOnlyImpl(extractDir, scope, pkg, shouldAbort, signal) ); } @@ -5037,9 +4946,10 @@ export class DownloadManager extends EventEmitter { // German audio track and strip the ".DL." marker from the filename. Operates // only inside pkg.extractDir, before MKV-collect. Best-effort per file; an // error never fails the package. Original is never lost (see video-processor). - private async keepGermanAudioOnlyImpl( - extractDir: string, - pkg?: PackageEntry, + private async keepGermanAudioOnlyImpl( + extractDir: string, + scope: PackageOutputScope, + pkg?: PackageEntry, shouldAbort?: () => boolean, signal?: AbortSignal ): Promise { @@ -5052,7 +4962,7 @@ export class DownloadManager extends EventEmitter { return 0; } - const videoFiles = await this.collectVideoFiles(extractDir); + const videoFiles = await this.collectVideoFiles(scope); const sampleTokenRe = /(^|[._\-\s])sample([._\-\s]|$)/i; const targets = videoFiles.filter((p) => { const name = path.basename(p); @@ -5208,7 +5118,7 @@ export class DownloadManager extends EventEmitter { // already single-track. Skips/errors leave the file fully untouched so the // unprocessed state stays visible. if (result.action === "remuxed" || result.action === "kept-single") { - await this.stripDualLangFromFileName(sourcePath, pkg); + await this.stripDualLangFromFileName(sourcePath, scope, pkg); } if (pkg) { const completedAt = nowMs(); @@ -5237,7 +5147,7 @@ export class DownloadManager extends EventEmitter { return processed; } - private async stripDualLangFromFileName(sourcePath: string, pkg?: PackageEntry): Promise { + private async stripDualLangFromFileName(sourcePath: string, scope: PackageOutputScope, pkg?: PackageEntry): Promise { const dir = path.dirname(sourcePath); const name = path.basename(sourcePath); const newName = stripDualLangMarker(name); @@ -5250,8 +5160,9 @@ export class DownloadManager extends EventEmitter { return; } try { - await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "audio-strip" }); - await this.renameCompanionFiles(sourcePath, targetPath, pkg); + await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "audio-strip" }); + scope.replacePath(sourcePath, targetPath); + await this.renameCompanionFiles(sourcePath, targetPath, scope, pkg); if (pkg) { const resolved = this.inferItemForMediaLog(pkg, targetPath, path.basename(targetPath)); this.logRenameProcess(pkg, "INFO", "audio-strip", ".DL. aus Dateiname entfernt", { sourcePath, targetPath }, resolved.item, resolved.matchedBy); @@ -5261,9 +5172,10 @@ export class DownloadManager extends EventEmitter { } } - private async autoRenameExtractedVideoFilesImpl( - extractDir: string, - pkg?: PackageEntry, + private async autoRenameExtractedVideoFilesImpl( + extractDir: string, + scope: PackageOutputScope, + pkg?: PackageEntry, shouldAbort?: () => boolean, treatFilesAsStable = false ): Promise { @@ -5287,7 +5199,7 @@ export class DownloadManager extends EventEmitter { } } - const videoFiles = await this.collectVideoFiles(extractDir); + const videoFiles = await this.collectVideoFiles(scope); logger.info(`Auto-Rename: ${videoFiles.length} Video-Dateien gefunden in ${extractDir}`); if (pkg) { this.logPackageForPackage(pkg, "INFO", "Auto-Rename Scan gestartet", { @@ -5531,7 +5443,8 @@ export class DownloadManager extends EventEmitter { } if (pathKey(targetPath) === pathKey(sourcePath) && targetPath !== sourcePath) { try { - await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "auto-rename (Schreibweise)" }); + await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "auto-rename (Schreibweise)" }); + scope.replacePath(sourcePath, targetPath); renamed += 1; if (pkg) { const resolved = resolveRenameItem(targetPath); @@ -5592,7 +5505,8 @@ export class DownloadManager extends EventEmitter { } try { - await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "auto-rename" }); + await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "auto-rename" }); + scope.replacePath(sourcePath, targetPath); if (pkg) { this.logPackageForPackage(pkg, "INFO", "Auto-Rename durchgeführt", { sourcePath, @@ -5610,7 +5524,7 @@ export class DownloadManager extends EventEmitter { } logger.info(`Auto-Rename: ${sourceName} -> ${path.basename(targetPath)}`); renamed += 1; - await this.renameCompanionFiles(sourcePath, targetPath, pkg); + await this.renameCompanionFiles(sourcePath, targetPath, scope, pkg); } catch (error) { if (this.isPathLengthRenameError(error)) { const fallbackCandidates = [ @@ -5627,7 +5541,8 @@ export class DownloadManager extends EventEmitter { continue; } try { - await this.renamePathWithExdevFallback(sourcePath, fallbackPath, { label: "auto-rename (Pfadlaenge-Fallback)" }); + await this.renamePathWithExdevFallback(sourcePath, fallbackPath, { label: "auto-rename (Pfadlaenge-Fallback)" }); + scope.replacePath(sourcePath, fallbackPath); logger.warn(`Auto-Rename Fallback wegen Pfadlänge: ${sourceName} -> ${path.basename(fallbackPath)}`); renamed += 1; if (pkg) { @@ -5687,48 +5602,60 @@ export class DownloadManager extends EventEmitter { await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "mkv-move" }); } - private async cleanupNonMkvResidualFiles(rootDir: string, targetDir: string): Promise { - if (!rootDir || !await this.existsAsync(rootDir)) { - return 0; - } - - let removed = 0; - const stack = [rootDir]; - while (stack.length > 0) { - const current = stack.pop() as string; - let entries: fs.Dirent[] = []; - try { - entries = await fs.promises.readdir(current, { withFileTypes: true }); - } catch { - continue; - } - - for (const entry of entries) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) { - if (isPathInsideDir(fullPath, targetDir)) { - continue; - } - stack.push(fullPath); - continue; - } - if (!entry.isFile()) { - continue; - } - const extension = path.extname(entry.name).toLowerCase(); - if (SAMPLE_VIDEO_EXTENSIONS.has(extension)) { - continue; - } - try { - await fs.promises.rm(toWindowsLongPathIfNeeded(fullPath), { force: true }); - removed += 1; - } catch { - } - } - } - - return removed; - } + private async cleanupNonMkvResidualFiles( + scope: PackageOutputScope, + targetDir: string, + touchedParents: Set + ): Promise { + let removed = 0; + for (const fullPath of scope.completeFiles()) { + if (isPathInsideDir(fullPath, targetDir)) { + continue; + } + if (SAMPLE_VIDEO_EXTENSIONS.has(path.extname(fullPath).toLowerCase())) { + continue; + } + try { + await fs.promises.rm(toWindowsLongPathIfNeeded(fullPath), { force: true }); + touchedParents.add(path.dirname(fullPath)); + scope.removePath(fullPath); + removed += 1; + } catch { + } + } + return removed; + } + + private async removeEmptyScopedParentChains(rootDir: string, parents: ReadonlySet): Promise { + const rootPath = path.resolve(rootDir); + const candidates = new Set(); + for (const parent of parents) { + let current = path.resolve(parent); + while (isPathInsideDir(current, rootPath) && pathKey(current) !== pathKey(rootPath)) { + candidates.add(current); + const next = path.dirname(current); + if (next === current) { + break; + } + current = next; + } + } + if (parents.size > 0) { + candidates.add(rootPath); + } + let removed = 0; + for (const directory of [...candidates].sort((left, right) => right.length - left.length)) { + try { + const entries = await fs.promises.readdir(directory); + if (entries.length === 0) { + await fs.promises.rmdir(directory); + removed += 1; + } + } catch { + } + } + return removed; + } private async cleanupRemainingArchiveArtifacts(packageDir: string, shouldAbort?: () => boolean): Promise { if (this.settings.cleanupMode === "none") { @@ -5964,7 +5891,27 @@ export class DownloadManager extends EventEmitter { logger.warn(`MKV-Sammelordner übersprungen: pkg=${pkg.name}, ungültiger Pfad`); return; } - const targetDir = path.resolve(targetDirRaw); + const targetDir = path.resolve(targetDirRaw); + const scope = this.getPackageOutputScope(pkg); + await this.adoptLegacyPackageOutputsIfExclusive(pkg, scope); + for (const itemId of pkg.itemIds) { + const item = this.session.items[itemId]; + const outputPath = String(item?.targetPath || "").trim(); + if (!item || item.status !== "completed" || !outputPath || !isPathInsideDir(outputPath, pkg.outputDir)) { + continue; + } + try { + scope.add({ + version: 1, + archivePath: path.resolve(outputPath), + entryPath: path.basename(outputPath), + outputPath: path.resolve(outputPath), + state: "complete", + disposition: "written" + }); + } catch { + } + } const sourceDirs: string[] = []; for (const dir of sourceDirsAll) { @@ -5998,16 +5945,18 @@ export class DownloadManager extends EventEmitter { return; } - const seenBasenames = new Set(); - const collected: { filePath: string; sourceRoot: string }[] = []; - for (const dir of sourceDirs) { - const filesInDir = await this.collectFilesByExtensions(dir, SAMPLE_VIDEO_EXTENSIONS); - for (const filePath of filesInDir) { - const baseLower = path.basename(filePath).toLowerCase(); - if (seenBasenames.has(baseLower)) continue; - seenBasenames.add(baseLower); - collected.push({ filePath, sourceRoot: dir }); - } + const seenBasenames = new Set(); + const collected: { filePath: string; sourceRoot: string }[] = []; + const scopedVideoFiles = await this.collectFilesByExtensions(scope, SAMPLE_VIDEO_EXTENSIONS); + for (const filePath of scopedVideoFiles) { + const sourceRoot = sourceDirs.find((dir) => isPathInsideDir(filePath, dir)); + if (!sourceRoot) { + continue; + } + const baseLower = path.basename(filePath).toLowerCase(); + if (seenBasenames.has(baseLower)) continue; + seenBasenames.add(baseLower); + collected.push({ filePath, sourceRoot }); } if (collected.length === 0) { logger.info(`MKV-Sammelordner: pkg=${pkg.name}, keine MKV gefunden`); @@ -6056,9 +6005,10 @@ export class DownloadManager extends EventEmitter { const reservedTargets = new Set(); let moved = 0; let skipped = 0; - let failed = 0; - let sourceArtifactsChanged = false; - let sourceCleanupRelevant = false; + let failed = 0; + let sourceArtifactsChanged = false; + let sourceCleanupRelevant = false; + const touchedParents = new Set(); for (const { filePath: sourcePath, sourceRoot } of mkvFiles) { if (shouldAbort?.()) { @@ -6130,8 +6080,10 @@ export class DownloadManager extends EventEmitter { sourceSize }, resolved.item, resolved.matchedBy); try { - await fs.promises.unlink(sourcePath); - sourceArtifactsChanged = true; + await fs.promises.unlink(sourcePath); + touchedParents.add(path.dirname(sourcePath)); + scope.removePath(sourcePath); + sourceArtifactsChanged = true; } catch { } sourceCleanupRelevant = true; @@ -6148,7 +6100,9 @@ export class DownloadManager extends EventEmitter { } try { - await this.moveFileWithExdevFallback(sourcePath, targetPath); + await this.moveFileWithExdevFallback(sourcePath, targetPath); + touchedParents.add(path.dirname(sourcePath)); + scope.replacePath(sourcePath, targetPath); moved += 1; sourceArtifactsChanged = true; sourceCleanupRelevant = true; @@ -6163,7 +6117,7 @@ export class DownloadManager extends EventEmitter { targetPath, sourceSize }, resolved.item, resolved.matchedBy); - await this.moveCompanionFiles(sourcePath, targetPath, pkg); + await this.moveCompanionFiles(sourcePath, targetPath, scope, pkg); } catch (error) { failed += 1; logger.warn(`MKV verschieben fehlgeschlagen: ${sourcePath} -> ${targetPath} (${compactErrorText(error)})`); @@ -6183,11 +6137,11 @@ export class DownloadManager extends EventEmitter { } if ((sourceArtifactsChanged || sourceCleanupRelevant) && cleanupDir && await this.existsAsync(cleanupDir)) { - const removedResidual = await this.cleanupNonMkvResidualFiles(cleanupDir, targetDir); + const removedResidual = await this.cleanupNonMkvResidualFiles(scope, targetDir, touchedParents); if (removedResidual > 0) { logger.info(`MKV-Sammelordner entfernte Restdateien: pkg=${pkg.name}, dir=${cleanupDir}, entfernt=${removedResidual}`); } - const removedDirs = await this.removeEmptyDirectoryTree(cleanupDir); + const removedDirs = await this.removeEmptyScopedParentChains(cleanupDir, touchedParents); if (removedDirs > 0) { logger.info(`MKV-Sammelordner entfernte leere Ordner: pkg=${pkg.name}, dir=${cleanupDir}, entfernt=${removedDirs}`); } @@ -9140,6 +9094,8 @@ export class DownloadManager extends EventEmitter { pkg.outputCount = 0; pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION; pkg.outputProvenance = []; + pkg.outputRecords = []; + pkg.outputScopeAdopted = false; } for (const itemId of itemIds) { this.retryAfterByItem.delete(itemId); @@ -12453,6 +12409,9 @@ export class DownloadManager extends EventEmitter { pkg.outputCount = 0; pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION; pkg.outputProvenance = []; + pkg.outputRecords = []; + pkg.outputScopeAdopted = false; + this.packageOutputScopes.delete(packageId); pkg.cleanupErrorCategory = ""; } this.pruneFinalizedPackageResults(); @@ -12477,6 +12436,7 @@ export class DownloadManager extends EventEmitter { } private pruneRemovedPackageResultState(packageId: string): void { + this.packageOutputScopes.delete(packageId); const prefix = `${packageId}:`; for (const key of [...this.finalizedPackageResults.keys()]) { if (key.startsWith(prefix)) { @@ -13482,7 +13442,7 @@ export class DownloadManager extends EventEmitter { return 0; } - const result = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({ + const result = await this.runWithPackageOutputProvenance(pkg, (targetDir, scope) => extractPackageArchives({ packageDir: pkg.outputDir, targetDir, cleanupMode: this.settings.cleanupMode, @@ -13496,8 +13456,9 @@ export class DownloadManager extends EventEmitter { packageId, hybridMode: true, maxParallel: this.settings.maxParallelExtract || 2, - extractCpuPriority: "high", - onLog: (level, message) => this.logExtractionForItems(pkg, items, "Hybrid-Extractor", level, message), + extractCpuPriority: "high", + onLog: (level, message) => this.logExtractionForItems(pkg, items, "Hybrid-Extractor", level, message), + onOutput: (event) => scope.add(event), onArchiveFailure: (failure) => { failedArchiveCategories.set(String(failure.archiveName || "").toLowerCase(), failure.category); const failedArchiveKey = readyArchiveKeyByName.get(String(failure.archiveName || "").toLowerCase()); @@ -13680,19 +13641,23 @@ export class DownloadManager extends EventEmitter { hybridSet = new Set(); this.packageHybridPostProcessControllers.set(packageId, hybridSet); } - hybridSet.add(hybridController); + hybridSet.add(hybridController); + const hybridOutputScope = this.getPackageOutputScope(pkg); const hybridShouldAbort = (): boolean => hybridController.signal.aborted || this.session.packages[packageId] !== pkg; const hybridHandle: { task?: Promise } = {}; const hybridTask = (async () => { try { - await this.chainPackageFileOp(pkg.id, async () => { - await this.autoRenameExtractedVideoFilesImpl(pkg.extractDir, pkg, hybridShouldAbort); - await this.keepGermanAudioOnlyImpl(pkg.extractDir, pkg, hybridShouldAbort, hybridController.signal); - await this.collectMkvFilesToLibrary(packageId, pkg, hybridShouldAbort, true); - }); + await this.chainPackageFileOp(pkg.id, async () => { + await this.autoRenameExtractedVideoFilesImpl(pkg.extractDir, hybridOutputScope, pkg, hybridShouldAbort); + await this.keepGermanAudioOnlyImpl(pkg.extractDir, hybridOutputScope, pkg, hybridShouldAbort, hybridController.signal); + await this.collectMkvFilesToLibrary(packageId, pkg, hybridShouldAbort, true); + }); } catch (err) { logger.warn(`Hybrid Post-Extract (Rename+Collect) Fehler: pkg=${pkg.name}, reason=${compactErrorText(err)}`); - } finally { + } finally { + if (this.session.packages[packageId] === pkg) { + this.syncPackageOutputScope(pkg, hybridOutputScope); + } const set = this.packageHybridPostProcessControllers.get(packageId); if (set) { set.delete(hybridController); @@ -13769,10 +13734,12 @@ export class DownloadManager extends EventEmitter { if (!pkg || pkg.cancelled) { return; } - if (signal?.aborted) { - return; - } - const items = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[]; + if (signal?.aborted) { + return; + } + const packageOutputScope = this.getPackageOutputScope(pkg); + await this.adoptLegacyPackageOutputsIfExclusive(pkg, packageOutputScope); + const items = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[]; const recoveryStart = nowMs(); for (const item of items) { @@ -14057,7 +14024,7 @@ export class DownloadManager extends EventEmitter { entry.updatedAt = pendingAt; } this.emitState(); - const result = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({ + const result = await this.runWithPackageOutputProvenance(pkg, (targetDir, scope) => extractPackageArchives({ packageDir: pkg.outputDir, targetDir, cleanupMode: this.settings.cleanupMode, @@ -14070,8 +14037,9 @@ export class DownloadManager extends EventEmitter { onlyArchives: fullArchiveSet, skipPostCleanup: true, maxParallel: this.settings.maxParallelExtract || 2, - extractCpuPriority: "high", - onLog: (level, message) => this.logExtractionForItems(pkg, completedItems, "Extractor", level, message), + extractCpuPriority: "high", + onLog: (level, message) => this.logExtractionForItems(pkg, completedItems, "Extractor", level, message), + onOutput: (event) => scope.add(event), onArchiveFailure: (failure) => { fullFailedArchiveCategories.set(failure.archiveName.toLowerCase(), failure.category); if (autoRecoveredArchives.has(failure.archiveName)) { @@ -14240,7 +14208,8 @@ export class DownloadManager extends EventEmitter { ); pkg.status = "failed"; } else { - const hasExtractedOutput = await this.directoryHasAnyFiles(pkg.extractDir); + const hasExtractedOutput = this.getPackageOutputScope(pkg).completeFiles() + .some((filePath) => isPathInsideDir(filePath, pkg.extractDir)); const sourceExists = await this.existsAsync(pkg.outputDir); let finalStatusText = ""; @@ -14384,17 +14353,19 @@ export class DownloadManager extends EventEmitter { const deferredVersion = this.getPackagePostProcessVersion(packageId); const shouldAbort = (): boolean => !this.isDeferredPostProcessStillCurrent(packageId, pkg, deferredVersion, deferredController.signal); const throwIfAborted = (): void => this.throwIfDeferredPostProcessAborted(packageId, pkg, deferredVersion, deferredController.signal); - const hasBlockingExtractError = pkg.itemIds.some((itemId) => { + const hasBlockingExtractError = pkg.itemIds.some((itemId) => { const item = this.session.items[itemId]; return Boolean(item && item.status === "completed" && isExtractErrorLabel(item.fullStatus || "")); - }); + }); + const outputScope = this.getPackageOutputScope(pkg); + await this.adoptLegacyPackageOutputsIfExclusive(pkg, outputScope); try { throwIfAborted(); if ((extractedCount > 0 || alreadyMarkedExtracted) && failed === 0 && this.settings.autoExtract) { const nestedBlacklist = /\.(iso|img|bin|dmg|vhd|vhdx|vmdk|wim)$/i; - const nestedCandidates = (await findArchiveCandidates(pkg.extractDir)) - .filter((p) => !nestedBlacklist.test(p)); + const nestedCandidates = outputScope.archiveFiles() + .filter((candidate) => isPathInsideDir(candidate, pkg.extractDir) && !nestedBlacklist.test(candidate)); if (nestedCandidates.length > 0) { pkg.postProcessLabel = "Nested Entpacken..."; this.emitState(); @@ -14405,7 +14376,7 @@ export class DownloadManager extends EventEmitter { }); const nestedFailureCategories = new Map(); const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[]; - const nestedResult = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({ + const nestedResult = await this.runWithPackageOutputProvenance(pkg, (targetDir, scope) => extractPackageArchives({ packageDir: pkg.extractDir, targetDir, cleanupMode: this.settings.cleanupMode, @@ -14419,6 +14390,7 @@ export class DownloadManager extends EventEmitter { maxParallel: this.settings.maxParallelExtract || 2, extractCpuPriority: this.settings.extractCpuPriority, onLog: (level, message) => this.logPackageForPackage(pkg, level, `Nested-Extractor: ${message}`), + onOutput: (event) => scope.add(event), onArchiveFailure: (failure) => { nestedFailureCategories.set(failure.archiveName.toLowerCase(), failure.category); }, @@ -14448,12 +14420,12 @@ export class DownloadManager extends EventEmitter { extractDir: pkg.extractDir }); throwIfAborted(); - await this.autoRenameExtractedVideoFiles(pkg.extractDir, pkg, shouldAbort, true); + await this.autoRenameExtractedVideoFiles(pkg.extractDir, outputScope, pkg, shouldAbort, true); if (this.settings.keepGermanAudioOnly) { pkg.postProcessLabel = "Tonspur..."; this.emitState(); throwIfAborted(); - await this.keepGermanAudioOnly(pkg.extractDir, pkg, shouldAbort, deferredController.signal); + await this.keepGermanAudioOnly(pkg.extractDir, outputScope, pkg, shouldAbort, deferredController.signal); } } @@ -14488,13 +14460,15 @@ export class DownloadManager extends EventEmitter { if (extractedCount > 0 || alreadyMarkedExtracted) { throwIfAborted(); if (this.settings.removeLinkFilesAfterExtract) { - const removedLinks = await removeDownloadLinkArtifacts(pkg.extractDir, { shouldAbort }); + const removedLinks = await removeDownloadLinkArtifactsFromScope(outputScope.completeFiles(), { shouldAbort, rootDir: pkg.extractDir }); + outputScope.pruneMissing(); if (removedLinks > 0) { logger.info(`Deferred Link-Cleanup: pkg=${pkg.name}, entfernt=${removedLinks}`); } } if (this.settings.removeSamplesAfterExtract) { - const removedSamples = await removeSampleArtifacts(pkg.extractDir, { shouldAbort }); + const removedSamples = await removeSampleArtifactsFromScope(outputScope.completeFiles(), { shouldAbort, rootDir: pkg.extractDir }); + outputScope.pruneMissing(); if (removedSamples.files > 0 || removedSamples.dirs > 0) { logger.info(`Deferred Sample-Cleanup: pkg=${pkg.name}, files=${removedSamples.files}, dirs=${removedSamples.dirs}`); } @@ -14553,8 +14527,9 @@ export class DownloadManager extends EventEmitter { if (this.packageDeferredPostProcessAbortControllers.get(packageId) === deferredController) { this.packageDeferredPostProcessAbortControllers.delete(packageId); } - if (this.session.packages[packageId] === pkg && this.getPackagePostProcessVersion(packageId) === deferredVersion) { - pkg.postProcessLabel = undefined; + if (this.session.packages[packageId] === pkg && this.getPackagePostProcessVersion(packageId) === deferredVersion) { + this.syncPackageOutputScope(pkg, outputScope); + pkg.postProcessLabel = undefined; pkg.updatedAt = nowMs(); this.persistSoon(); this.emitState(); diff --git a/src/main/extractor.ts b/src/main/extractor.ts index a97d0b4..31b6542 100644 --- a/src/main/extractor.ts +++ b/src/main/extractor.ts @@ -5,7 +5,7 @@ import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import AdmZip from "adm-zip"; import { CleanupMode, ConflictMode } from "../shared/types"; import { logger } from "./logger"; -import { removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup"; +import { removeDownloadLinkArtifactsFromScope, removeSampleArtifactsFromScope } from "./cleanup"; import { PackageOutputScope, type ExtractOutputEvent } from "./package-output-scope"; export type { ExtractOutputEvent } from "./package-output-scope"; @@ -170,7 +170,7 @@ type ExtractResumeMember = { type ExtractResumeOutput = { entryPath: string; path: string; - disposition: ExtractOutputEvent["disposition"]; + disposition: Exclude; }; type ExtractResumeArchive = { @@ -369,7 +369,7 @@ export async function findArchiveCandidates(packageDir: string): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + return ["x", "-y", "-bb1", "-sccUTF-8", overwrite, pass, archivePath, `-o${targetDir}`]; +} + +export function parseNativeExtractOutput( + command: string, + line: string, + archivePath: string, + targetDir: string, + conflictMode: ConflictMode +): ExtractOutputEvent[] { + const trimmed = String(line || "").trim(); + let reportedPath = ""; + if (extractorCommandKind(command) === "seven_zip") { + const match = trimmed.match(/^[-+]\s+(.+)$/); + reportedPath = match?.[1]?.trim() || ""; + } else if (isRarNativeCommand(command)) { + const match = trimmed.match(/^Extracting\s+(.+?)(?:\s+OK)?$/i); + reportedPath = match?.[1]?.trim() || ""; + } + if (!reportedPath) { + return []; + } + const targetRoot = path.resolve(targetDir); + const rawPath = reportedPath.replace(/^"|"$/g, ""); + const outputPath = path.isAbsolute(rawPath) ? path.resolve(rawPath) : path.resolve(targetRoot, rawPath); + const relativePath = path.relative(targetRoot, outputPath); + if (!relativePath + || relativePath === ".." + || relativePath.startsWith(`..${path.sep}`) + || path.isAbsolute(relativePath)) { + return []; + } + const entryPath = relativePath.replace(/\\/g, "/"); + if (entryPath.split("/").some((segment) => !segment || segment === "..")) { + return []; + } + const mode = effectiveConflictMode(conflictMode); + if (mode === "rename" && !/ \(\d+\)(?=\.[^./]+$|$)/.test(path.basename(outputPath))) { + return []; + } + const event: ExtractOutputEvent = { + version: 1, + archivePath: path.resolve(archivePath), + entryPath, + outputPath, + state: "complete", + disposition: mode === "rename" ? "renamed" : mode === "overwrite" ? "overwritten" : "written" + }; + try { + const scope = new PackageOutputScope([targetRoot]); + scope.add(event); + return [event]; + } catch { + return []; + } +} + +function createNativeOutputCollector( + command: string, + archivePath: string, + targetDir: string, + conflictMode: ConflictMode, + onOutput?: (event: ExtractOutputEvent) => void +): { push: (chunk: string) => void; finish: (state: ExtractOutputEvent["state"]) => void } { + let buffer = ""; + const lines = new Set(); + const collectLine = (value: string): void => { + const trimmed = value.trim(); + if ((extractorCommandKind(command) === "seven_zip" && /^[-+]\s+/.test(trimmed)) + || (isRarNativeCommand(command) && /^Extracting\s+/i.test(trimmed))) { + lines.add(trimmed); + } + }; + return { + push: (chunk) => { + buffer += chunk; + const parts = buffer.split(/[\r\n]+/); + buffer = parts.pop() || ""; + for (const part of parts) { + collectLine(part); + } + }, + finish: (state) => { + collectLine(buffer); + buffer = ""; + for (const outputLine of lines) { + for (const event of parseNativeExtractOutput(command, outputLine, archivePath, targetDir, conflictMode)) { + onOutput?.({ ...event, state }); + } + } + } + }; +} + +const extractRetryDelay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); async function runExternalExtractInner( command: string, @@ -2188,9 +2279,26 @@ async function runExternalExtractInner( let bestPercent = 0; let passwordAttempt = 0; let usePerformanceFlags = externalExtractorSupportsPerfFlags && shouldUseExtractorPerformanceFlags(); - const summarizeResultError = (errorText: string): string => cleanErrorText(errorText); - let createErrorText = ""; - let createErrorPassword = ""; + const summarizeResultError = (errorText: string): string => cleanErrorText(errorText); + let createErrorText = ""; + let createErrorPassword = ""; + const runNativeAttempt = async (args: string[]): Promise => { + const outputs = createNativeOutputCollector(command, archivePath, targetDir, conflictMode, onOutput); + const result = await runExtractCommand(command, args, (chunk) => { + outputs.push(chunk); + const parsed = parseProgressPercent(chunk); + if (parsed === null) { + return; + } + const next = nextArchivePercent(bestPercent, parsed); + if (next !== bestPercent) { + bestPercent = next; + onArchiveProgress?.(bestPercent); + } + }, signal, timeoutMs); + outputs.finish(result.ok ? "complete" : "partial"); + return result; + }; if (forceFlatMode) { logger.info(`Flat-Modus direkt (gespeichert vom vorherigen Archiv): ${path.basename(archivePath)}`); @@ -2201,12 +2309,7 @@ async function runExternalExtractInner( onLog?.("INFO", `Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length}: archive=${path.basename(archivePath)}, password=`); logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length} für ${path.basename(archivePath)} (password=)`); const args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, usePerformanceFlags, hybridMode, true); - const result = await runExtractCommand(command, args, (chunk) => { - const parsed = parseProgressPercent(chunk); - if (parsed === null) return; - const next = nextArchivePercent(bestPercent, parsed); - if (next !== bestPercent) { bestPercent = next; onArchiveProgress?.(bestPercent); } - }, signal, timeoutMs); + const result = await runNativeAttempt(args); logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length}: ok=${result.ok}, bestPercent=${bestPercent}`); onLog?.("INFO", `Flach-Extraktion Ergebnis ${passwordAttempt}/${passwords.length}: archive=${path.basename(archivePath)}, ok=${result.ok}, timedOut=${result.timedOut}, missingCommand=${result.missingCommand}, bestPercent=${bestPercent}`); if (result.ok) { if (flatModeResult) flatModeResult.needed = true; onArchiveProgress?.(100); return password; } @@ -2233,17 +2336,7 @@ async function runExternalExtractInner( onPasswordAttempt?.(passwordAttempt, passwords.length); } let args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, usePerformanceFlags, hybridMode); - let result = await runExtractCommand(command, args, (chunk) => { - const parsed = parseProgressPercent(chunk); - if (parsed === null) { - return; - } - const next = nextArchivePercent(bestPercent, parsed); - if (next !== bestPercent) { - bestPercent = next; - onArchiveProgress?.(bestPercent); - } - }, signal, timeoutMs); + let result = await runNativeAttempt(args); if (!result.ok && usePerformanceFlags && isUnsupportedExtractorSwitchError(result.errorText)) { usePerformanceFlags = false; @@ -2251,17 +2344,7 @@ async function runExternalExtractInner( onLog?.("WARN", `Entpacker ohne Performance-Flags fortgesetzt: ${path.basename(archivePath)}`); logger.warn(`Entpacker ohne Performance-Flags fortgesetzt: ${path.basename(archivePath)}`); args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, false, hybridMode); - result = await runExtractCommand(command, args, (chunk) => { - const parsed = parseProgressPercent(chunk); - if (parsed === null) { - return; - } - const next = nextArchivePercent(bestPercent, parsed); - if (next !== bestPercent) { - bestPercent = next; - onArchiveProgress?.(bestPercent); - } - }, signal, timeoutMs); + result = await runNativeAttempt(args); } logger.info( @@ -2324,12 +2407,7 @@ async function runExternalExtractInner( logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length} für ${path.basename(archivePath)} (password=)`); onLog?.("INFO", `Flach-Extraktion Versuch ${passwordAttempt}/${flatPasswords.length}: archive=${path.basename(archivePath)}, password=`); const args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, usePerformanceFlags, hybridMode, true); - const result = await runExtractCommand(command, args, (chunk) => { - const parsed = parseProgressPercent(chunk); - if (parsed === null) return; - const next = nextArchivePercent(bestPercent, parsed); - if (next !== bestPercent) { bestPercent = next; onArchiveProgress?.(bestPercent); } - }, signal, timeoutMs); + const result = await runNativeAttempt(args); logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length}: ok=${result.ok}, bestPercent=${bestPercent}`); onLog?.("INFO", `Flach-Extraktion Ergebnis ${passwordAttempt}/${flatPasswords.length}: archive=${path.basename(archivePath)}, ok=${result.ok}, timedOut=${result.timedOut}, missingCommand=${result.missingCommand}, bestPercent=${bestPercent}`); if (result.ok) { if (flatModeResult) flatModeResult.needed = true; onArchiveProgress?.(100); return password; } @@ -2436,9 +2514,15 @@ async function runExternalExtract( const effectiveTargetDir = subst ? `${subst.drive}:\\` : targetDir; if (subst) { onLog?.("INFO", `Legacy-Zielpfad verkuerzt via subst: archive=${archiveName}, originalTargetDir=${targetDir}, effectiveTargetDir=${effectiveTargetDir}`); - } else { - onLog?.("INFO", `Legacy-Zielpfad unveraendert: archive=${archiveName}, effectiveTargetDir=${effectiveTargetDir}`); - } + } else { + onLog?.("INFO", `Legacy-Zielpfad unveraendert: archive=${archiveName}, effectiveTargetDir=${effectiveTargetDir}`); + } + const legacyOnOutput = subst && onOutput + ? (event: ExtractOutputEvent): void => onOutput({ + ...event, + outputPath: path.resolve(targetDir, ...event.entryPath.split("/")) + }) + : onOutput; const command = await resolveExtractorCommand(archivePath); const legacyStartedAt = Date.now(); @@ -2449,7 +2533,7 @@ async function runExternalExtract( password = await runExternalExtractInner( command, archivePath, effectiveTargetDir, conflictMode, passwordCandidates, onArchiveProgress, signal, timeoutMs, hybridMode, onPasswordAttempt, - forceFlatMode, flatModeResult, onLog + forceFlatMode, flatModeResult, onLog, legacyOnOutput ); } catch (primaryError) { const isRar = /\.rar$/i.test(archiveName) || /\.r\d{2,3}$/i.test(archiveName); @@ -2465,7 +2549,7 @@ async function runExternalExtract( password = await runExternalExtractInner( alt, archivePath, effectiveTargetDir, conflictMode, passwordCandidates, onArchiveProgress, signal, timeoutMs, hybridMode, onPasswordAttempt, - forceFlatMode, flatModeResult, onLog + forceFlatMode, flatModeResult, onLog, legacyOnOutput ); } else { throw primaryError; @@ -2507,7 +2591,8 @@ async function runExternalExtract( onPasswordAttempt, forceFlatMode, flatModeResult, - onLog + onLog, + legacyOnOutput ); logger.info(`Legacy-Retry erfolgreich: ${archiveName}`); onLog?.("INFO", `Legacy-Retry erfolgreich: ${archiveName}`); @@ -3660,7 +3745,7 @@ export async function extractPackageArchives(options: ExtractOptions): Promise 0) { - const hasOutputAfter = await hasAnyFilesRecursive(options.targetDir); + const hasOutputAfter = outputScope.completeFiles().length > 0; const hadResumeProgress = resumeCompletedAtStart > 0; if (!hasOutputAfter && conflictMode !== "skip" && !hadResumeProgress) { lastError = "Keine entpackten Dateien erkannt"; @@ -3681,12 +3766,20 @@ export async function extractPackageArchives(options: ExtractOptions): Promise options.signal?.aborted === true, + rootDir: options.targetDir + }); + outputScope.pruneMissing(); + logger.info(`Link-Artefakt-Cleanup: ${removedLinks} Datei(en) entfernt`); + } + if (options.removeSamples) { + const removedSamples = await removeSampleArtifactsFromScope(outputScope.completeFiles(), { + shouldAbort: () => options.signal?.aborted === true, + rootDir: options.targetDir + }); + outputScope.pruneMissing(); logger.info(`Sample-Cleanup: ${removedSamples.files} Datei(en), ${removedSamples.dirs} Ordner entfernt`); } } diff --git a/src/main/package-output-scope.ts b/src/main/package-output-scope.ts index cec6ed5..daab09f 100644 --- a/src/main/package-output-scope.ts +++ b/src/main/package-output-scope.ts @@ -172,7 +172,7 @@ export class PackageOutputScope { } public archiveFiles(): string[] { - return this.completeFiles().filter((filePath) => /\.(?:7z|rar|zip|tar|gz|bz2|xz|001)$/i.test(filePath)); + return this.completeFiles().filter((filePath) => /\.(?:7z|rar|zip|tar|gz|bz2|xz|tgz|tbz2|txz|001)$/i.test(filePath)); } public replacePath(sourcePath: string, targetPath: string, state?: ExtractOutputState): boolean { diff --git a/src/main/storage.ts b/src/main/storage.ts index c658517..c94c391 100644 --- a/src/main/storage.ts +++ b/src/main/storage.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { getDebridLinkApiKeyIds } from "../shared/debrid-link-keys"; import { getMegaDebridAccountIds, mergeMegaDebridCredentialPools, parseMegaDebridAccounts } from "../shared/mega-debrid-accounts"; -import { AppSettings, ArchiveOperationMetric, AudioStripSummary, BandwidthScheduleEntry, DailyStartOutcome, DebridAccountStatus, DebridFallbackProvider, DebridProvider, DownloadItem, DownloadStatus, FailurePhase, HistoryEntry, HistoryRetentionMode, LogStorageLocation, PACKAGE_OUTPUT_PROVENANCE_VERSION, PackageEntry, PackagePriority, RemuxOperationMetric, SessionState } from "../shared/types"; +import { AppSettings, ArchiveOperationMetric, AudioStripSummary, BandwidthScheduleEntry, DailyStartOutcome, DebridAccountStatus, DebridFallbackProvider, DebridProvider, DownloadItem, DownloadStatus, FailurePhase, HistoryEntry, HistoryRetentionMode, LogStorageLocation, PACKAGE_OUTPUT_PROVENANCE_VERSION, PackageEntry, PackageOutputRecord, PackagePriority, RemuxOperationMetric, SessionState } from "../shared/types"; import { getProviderUsageDayKey } from "../shared/provider-daily-limits"; import { getRealDebridAccountIds, normalizeRealDebridWebAccountIds, parseRealDebridApiAccounts, serializeRealDebridApiAccounts } from "../shared/real-debrid-accounts"; import { defaultSettings } from "./constants"; @@ -838,6 +838,44 @@ function normalizeRemuxOperations(raw: unknown): RemuxOperationMetric[] { }); } +function normalizePackageOutputRecords(raw: unknown): PackageOutputRecord[] { + if (!Array.isArray(raw)) { + return []; + } + const records = new Map(); + for (const value of raw.slice(0, 1_000_000)) { + const record = asRecord(value); + if (!record || Number(record.version) !== 1) { + continue; + } + const archivePath = asText(record.archivePath); + const outputPath = asText(record.outputPath); + const entryPath = asText(record.entryPath).replace(/\\/g, "/"); + const state = asText(record.state); + const disposition = asText(record.disposition); + if (!path.isAbsolute(archivePath) + || !path.isAbsolute(outputPath) + || !entryPath + || entryPath.startsWith("/") + || /^[a-zA-Z]:/.test(entryPath) + || entryPath.split("/").some((segment) => !segment || segment === "..") + || (state !== "complete" && state !== "partial") + || !["written", "overwritten", "renamed", "skipped"].includes(disposition)) { + continue; + } + const key = path.resolve(outputPath).toLocaleLowerCase("en-US"); + records.set(key, { + version: 1, + archivePath: path.resolve(archivePath), + entryPath, + outputPath: path.resolve(outputPath), + state, + disposition: disposition as PackageOutputRecord["disposition"] + }); + } + return [...records.values()]; +} + function optionalClampedNumber(record: Record, key: string, max = Number.MAX_SAFE_INTEGER): number | undefined { return Object.prototype.hasOwnProperty.call(record, key) ? clampNumber(record[key], 0, 0, max) @@ -974,9 +1012,18 @@ export function normalizeLoadedSession(raw: unknown): SessionState { const statusRaw = asText(pkg.status) as DownloadStatus; const status: DownloadStatus = VALID_DOWNLOAD_STATUSES.has(statusRaw) ? statusRaw : "queued"; const rawItemIds = Array.isArray(pkg.itemIds) ? pkg.itemIds : []; - const outputProvenance = Array.isArray(pkg.outputProvenance) + const normalizedOutputProvenance = Array.isArray(pkg.outputProvenance) ? [...new Set(pkg.outputProvenance.map((value) => asText(value).toLowerCase()).filter((value) => /^[a-f0-9]{64}$/.test(value)))].slice(0, 1_000_000) : []; + const hasOutputProvenanceVersion = pkg.outputProvenanceVersion !== undefined && pkg.outputProvenanceVersion !== null; + const rawOutputProvenanceVersion = Number(pkg.outputProvenanceVersion); + const unknownOutputProvenanceVersion = hasOutputProvenanceVersion + && rawOutputProvenanceVersion !== PACKAGE_OUTPUT_PROVENANCE_VERSION; + const outputProvenance = unknownOutputProvenanceVersion ? [] : normalizedOutputProvenance; + const outputRecords = unknownOutputProvenanceVersion + || (!hasOutputProvenanceVersion && outputProvenance.length === 0) + ? [] + : normalizePackageOutputRecords(pkg.outputRecords); packagesById[id] = { id, name: asText(pkg.name) || "Paket", @@ -1010,8 +1057,12 @@ export function normalizeLoadedSession(raw: unknown): SessionState { archiveOperations: normalizeArchiveOperations(pkg.archiveOperations), remuxOperations: normalizeRemuxOperations(pkg.remuxOperations), outputCount: outputProvenance.length, - outputProvenanceVersion: PACKAGE_OUTPUT_PROVENANCE_VERSION, + outputProvenanceVersion: unknownOutputProvenanceVersion + ? rawOutputProvenanceVersion + : PACKAGE_OUTPUT_PROVENANCE_VERSION, outputProvenance, + outputRecords, + outputScopeAdopted: Boolean(pkg.outputScopeAdopted), cleanupErrorCategory: asText(pkg.cleanupErrorCategory), resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER), createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER), diff --git a/src/shared/types.ts b/src/shared/types.ts index 0c5931b..be4f4a1 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -497,6 +497,15 @@ export type PackageResultStatus = "completed" | "partial" | "failed" | "cancelle export type FailurePhase = "download" | "extract" | "remux" | "cleanup" | null; export const PACKAGE_OUTPUT_PROVENANCE_VERSION = 1; +export interface PackageOutputRecord { + version: 1; + archivePath: string; + entryPath: string; + outputPath: string; + state: "complete" | "partial"; + disposition: "written" | "overwritten" | "renamed" | "skipped"; +} + export interface ArchiveOperationMetric { id: string; name: string; @@ -591,6 +600,8 @@ export interface PackageEntry { outputCount?: number; outputProvenanceVersion?: number; outputProvenance?: string[]; + outputRecords?: PackageOutputRecord[]; + outputScopeAdopted?: boolean; cleanupErrorCategory?: string; resultGeneration?: number; createdAt: number; diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index 49d3573..c1237b0 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -2,7 +2,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { cleanupCancelledPackageArtifacts, removeDownloadLinkArtifacts, removeSampleArtifacts } from "../src/main/cleanup"; +import { + cleanupCancelledPackageArtifacts, + removeDownloadLinkArtifacts, + removeDownloadLinkArtifactsFromScope, + removeSampleArtifacts, + removeSampleArtifactsFromScope +} from "../src/main/cleanup"; const tempDirs: string[] = []; @@ -81,7 +87,7 @@ describe("cleanup", () => { expect(fs.existsSync(path.join(dir, "readme.txt"))).toBe(true); }); - it("does not recurse into sample symlink or junction targets", async () => { + it("does not recurse into sample symlink or junction targets", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rd-clean-")); const external = fs.mkdtempSync(path.join(os.tmpdir(), "rd-clean-ext-")); tempDirs.push(dir, external); @@ -95,6 +101,32 @@ describe("cleanup", () => { const result = await removeSampleArtifacts(dir); expect(result.files).toBe(0); - expect(fs.existsSync(outsideFile)).toBe(true); - }); -}); + expect(fs.existsSync(outsideFile)).toBe(true); + }); + + it("removes only scoped link and sample outputs from a shared root", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rd-clean-scope-")); + tempDirs.push(dir); + const sampleDir = path.join(dir, "Samples"); + fs.mkdirSync(sampleDir, { recursive: true }); + const ownedLink = path.join(dir, "owned.url"); + const foreignLink = path.join(dir, "foreign.url"); + const ownedSample = path.join(sampleDir, "owned-sample.mkv"); + const foreignSample = path.join(sampleDir, "foreign-sample.mkv"); + fs.writeFileSync(ownedLink, "owned"); + fs.writeFileSync(foreignLink, "foreign"); + fs.writeFileSync(ownedSample, "owned"); + fs.writeFileSync(foreignSample, "foreign"); + + const removedLinks = await removeDownloadLinkArtifactsFromScope([ownedLink]); + const removedSamples = await removeSampleArtifactsFromScope([ownedSample]); + + expect(removedLinks).toBe(1); + expect(removedSamples).toEqual({ files: 1, dirs: 0 }); + expect(fs.existsSync(ownedLink)).toBe(false); + expect(fs.existsSync(ownedSample)).toBe(false); + expect(fs.existsSync(foreignLink)).toBe(true); + expect(fs.existsSync(foreignSample)).toBe(true); + expect(fs.existsSync(sampleDir)).toBe(true); + }); +}); diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index e6a51d1..7281774 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -587,7 +587,16 @@ describe("disk write recovery", () => { }) }); - const processed = await (manager as any).keepGermanAudioOnlyImpl(extractDir, pkg); + const outputScope = (manager as any).getPackageOutputScope(pkg); + outputScope.add({ + version: 1, + archivePath: path.join(pkg.outputDir, "archive.rar"), + entryPath: path.basename(sourcePath), + outputPath: sourcePath, + state: "complete", + disposition: "written" + }); + const processed = await (manager as any).keepGermanAudioOnlyImpl(extractDir, outputScope, pkg); expect(processed).toBe(0); expect(fs.existsSync(sourcePath)).toBe(true); @@ -12758,7 +12767,7 @@ describe("download manager", () => { void manager; }, 20000); - it("collect cleans a raw file sitting OUTSIDE extractDir (Downloader-Unfertig case) AND its .srt follows the rename", async () => { + it("does not collect unscoped raw files outside the package extract directory", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root); @@ -12808,10 +12817,9 @@ describe("download manager", () => { await (manager as any).collectMkvFilesToLibrary(packageId, session.packages[packageId], undefined, false); - const cleanBase = "Fritzie.-.Der.Himmel.muss.warten.S04E01.GERMAN.720p.WEB.AVC-4SF"; - expect(fs.existsSync(path.join(mkvLibraryDir, `${cleanBase}.mkv`))).toBe(true); - expect(fs.existsSync(path.join(mkvLibraryDir, rawName))).toBe(false); - expect(fs.existsSync(path.join(mkvLibraryDir, `${cleanBase}.de.srt`))).toBe(true); + expect(fs.existsSync(path.join(outputDir, rawName))).toBe(true); + expect(fs.existsSync(path.join(outputDir, rawSrt))).toBe(true); + expect(fs.existsSync(mkvLibraryDir) ? fs.readdirSync(mkvLibraryDir) : []).toEqual([]); void manager; }, 20000); @@ -13246,7 +13254,7 @@ describe("download manager", () => { void manager; }, 20000); - it("does NOT delete pending RAR archive sets in outputDir when collecting MKVs from extractDir", async () => { + it("does NOT delete pending RAR archive sets in outputDir when collecting MKVs from extractDir", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root); @@ -13318,8 +13326,53 @@ describe("download manager", () => { expect(fs.existsSync(path.join(outputDir, part))).toBe(true); } - void manager; - }, 20000); + void manager; + }, 20000); + + it("does not adopt unscoped files from an extraction root shared by packages", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-shared-adoption-")); + tempDirs.push(root); + const sharedExtractDir = path.join(root, "shared"); + const libraryDir = path.join(root, "library"); + fs.mkdirSync(sharedExtractDir, { recursive: true }); + const foreignPath = path.join(sharedExtractDir, "foreign.mkv"); + fs.writeFileSync(foreignPath, "foreign"); + const session = emptySession(); + for (const id of ["package-a", "package-b"]) { + session.packages[id] = { + id, + name: id, + outputDir: path.join(root, "downloads", id), + extractDir: sharedExtractDir, + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + outputProvenanceVersion: 1, + outputRecords: [], + createdAt: 1_000, + updatedAt: 1_000 + }; + } + session.packageOrder = ["package-a", "package-b"]; + const manager = new DownloadManager( + { + ...defaultSettings(), + autoExtract: true, + createExtractSubfolder: true, + collectMkvToLibrary: true, + mkvLibraryDir: libraryDir + }, + session, + createStoragePaths(path.join(root, "state")) + ); + + await (manager as any).collectMkvFilesToLibrary("package-a", session.packages["package-a"]); + + expect(fs.existsSync(foreignPath)).toBe(true); + expect(fs.existsSync(path.join(libraryDir, "foreign.mkv"))).toBe(false); + expect(session.packages["package-a"].outputRecords).toEqual([]); + }); it("does NOT move bonus files from Extras subdirectory to flat library", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); @@ -15114,7 +15167,7 @@ describe("package priority ordering", () => { }); describe("package lifecycle telemetry boundaries", () => { - it("captures shared-root provenance from package staging without scanning unrelated files", async () => { + it("captures direct package output scopes concurrently without scanning a shared root", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-provenance-lock-")); tempDirs.push(root); const extractDir = path.join(root, "extract"); @@ -15122,17 +15175,10 @@ describe("package lifecycle telemetry boundaries", () => { for (let index = 0; index < 2_000; index += 1) { fs.writeFileSync(path.join(extractDir, `foreign-${index}.txt`), "foreign"); } - const traversedDirectories: string[] = []; const manager = new DownloadManager( { ...defaultSettings(), extractConflictMode: "overwrite" }, emptySession(), - createStoragePaths(path.join(root, "state")), - { - readOutputDirectory: async (directory: string) => { - traversedDirectories.push(path.resolve(directory)); - return fs.promises.readdir(directory, { withFileTypes: true }); - } - } as any + createStoragePaths(path.join(root, "state")) ); const createPackage = (id: string): PackageEntry => ({ id, @@ -15155,79 +15201,51 @@ describe("package lifecycle telemetry boundaries", () => { let enteredB = false; const state = manager as any; - const first = state.runWithPackageOutputProvenance(packageA, async (operationTarget = extractDir) => { - fs.writeFileSync(path.join(operationTarget, "package-a.mkv"), "a"); + const first = state.runWithPackageOutputProvenance(packageA, async (operationTarget: string, scope: any) => { + const outputPath = path.join(operationTarget, "package-a.mkv"); + fs.writeFileSync(outputPath, "a"); + scope.add({ + version: 1, + archivePath: path.join(packageA.outputDir, "archive.rar"), + entryPath: "package-a.mkv", + outputPath, + state: "complete", + disposition: "written" + }); await gateA; }); await vi.waitFor(() => expect(enteredB).toBe(false)); - const second = state.runWithPackageOutputProvenance(packageB, async (operationTarget = extractDir) => { + const second = state.runWithPackageOutputProvenance(packageB, async (operationTarget: string, scope: any) => { enteredB = true; - fs.writeFileSync(path.join(operationTarget, "package-b.mkv"), "b"); + const outputPath = path.join(operationTarget, "package-b.mkv"); + fs.writeFileSync(outputPath, "b"); + scope.add({ + version: 1, + archivePath: path.join(packageB.outputDir, "archive.rar"), + entryPath: "package-b.mkv", + outputPath, + state: "complete", + disposition: "written" + }); }); await vi.waitFor(() => expect(enteredB).toBe(true)); releaseA(); await Promise.all([first, second]); expect(packageA.outputCount).toBe(1); expect(packageB.outputCount).toBe(1); - expect(traversedDirectories.length).toBeGreaterThan(0); - expect(traversedDirectories).not.toContain(path.resolve(extractDir)); - expect(traversedDirectories.length).toBeLessThanOrEqual(4); + expect(packageA.outputRecords).toEqual([expect.objectContaining({ outputPath: path.join(extractDir, "package-a.mkv") })]); + expect(packageB.outputRecords).toEqual([expect.objectContaining({ outputPath: path.join(extractDir, "package-b.mkv") })]); }); - it.each([ - ["overwrite", "package", ["episode.mkv"]], - ["skip", "foreign", ["episode.mkv"]], - ["rename", "foreign", ["episode (1).mkv", "episode.mkv"]] - ] as const)("preserves %s conflicts while merging staged package outputs", async (conflictMode, expectedOriginal, expectedFiles) => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), `rd-output-${conflictMode}-`)); - tempDirs.push(root); - const extractDir = path.join(root, "extract"); - fs.mkdirSync(extractDir, { recursive: true }); - fs.writeFileSync(path.join(extractDir, "episode.mkv"), "foreign"); - const manager = new DownloadManager( - { ...defaultSettings(), extractConflictMode: conflictMode }, - emptySession(), - createStoragePaths(path.join(root, "state")) - ); - const pkg: PackageEntry = { - id: `conflict-${conflictMode}`, - name: `conflict-${conflictMode}`, - outputDir: path.join(root, "downloads"), - extractDir, - status: "completed", - itemIds: [], - cancelled: false, - enabled: true, - createdAt: 1_000, - updatedAt: 1_000 - }; - const state = manager as any; - - await state.runWithPackageOutputProvenance(pkg, async (operationTarget = extractDir) => { - fs.writeFileSync(path.join(operationTarget, "episode.mkv"), "package"); - }); - - expect(fs.readFileSync(path.join(extractDir, "episode.mkv"), "utf8")).toBe(expectedOriginal); - expect(fs.readdirSync(extractDir).filter((name) => name.endsWith(".mkv")).sort()).toEqual([...expectedFiles]); - expect(pkg.outputCount).toBe(conflictMode === "skip" ? 0 : 1); - }); - - it("retains partial staged outputs deterministically when extraction aborts", async () => { + it("retains directly reported partial outputs when extraction aborts", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-abort-")); tempDirs.push(root); const extractDir = path.join(root, "extract"); fs.mkdirSync(extractDir, { recursive: true }); - const traversedDirectories: string[] = []; const manager = new DownloadManager( defaultSettings(), emptySession(), - createStoragePaths(path.join(root, "state")), - { - readOutputDirectory: async (directory: string) => { - traversedDirectories.push(path.resolve(directory)); - return fs.promises.readdir(directory, { withFileTypes: true }); - } - } as any + createStoragePaths(path.join(root, "state")) ); const pkg: PackageEntry = { id: "aborted-output", @@ -15243,16 +15261,23 @@ describe("package lifecycle telemetry boundaries", () => { }; const state = manager as any; - await expect(state.runWithPackageOutputProvenance(pkg, async (operationTarget = extractDir) => { - fs.writeFileSync(path.join(operationTarget, "partial.mkv"), "partial"); + await expect(state.runWithPackageOutputProvenance(pkg, async (operationTarget: string, scope: any) => { + const outputPath = path.join(operationTarget, "partial.mkv"); + fs.writeFileSync(outputPath, "partial"); + scope.add({ + version: 1, + archivePath: path.join(pkg.outputDir, "archive.rar"), + entryPath: "partial.mkv", + outputPath, + state: "partial", + disposition: "written" + }); throw new Error("aborted:extract"); })).rejects.toThrow("aborted:extract"); expect(fs.readFileSync(path.join(extractDir, "partial.mkv"), "utf8")).toBe("partial"); expect(pkg.outputCount).toBe(1); - expect(traversedDirectories.length).toBeGreaterThan(0); - expect(traversedDirectories).not.toContain(path.resolve(extractDir)); - expect(fs.readdirSync(extractDir).filter((name) => name.startsWith(".rd-output-"))).toEqual([]); + expect(pkg.outputRecords).toEqual([expect.objectContaining({ state: "partial", outputPath: path.join(extractDir, "partial.mkv") })]); }); it("uses normalized nested item paths for archive identity and leaves empty item provenance at zero", () => { diff --git a/tests/extractor.test.ts b/tests/extractor.test.ts index 4989aff..0ee428f 100644 --- a/tests/extractor.test.ts +++ b/tests/extractor.test.ts @@ -17,6 +17,7 @@ import { shouldSerialRetryParallelFailures, findArchiveCandidates, orderExtractorCandidatesForArchive, + parseNativeExtractOutput, resolveExtractorBackendModeForArchive, resolveExtractorBackendMode, shouldFallbackLegacyRarToJvm, @@ -1467,5 +1468,32 @@ describe("extractor", () => { expect(fs.existsSync(path.join(targetDir, "second.txt"))).toBe(false); }); + it("strictly parses native output paths and fails closed for ambiguous rename output", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-native-output-")); + tempDirs.push(root); + const targetDir = path.join(root, "out"); + fs.mkdirSync(targetDir, { recursive: true }); + const archivePath = path.join(root, "archive.7z"); + const exactPath = path.join(targetDir, "folder", "episode.mkv"); + fs.mkdirSync(path.dirname(exactPath), { recursive: true }); + fs.writeFileSync(exactPath, "video"); + + expect(parseNativeExtractOutput("7z.exe", "- folder\\episode.mkv", archivePath, targetDir, "overwrite")).toEqual([ + expect.objectContaining({ entryPath: "folder/episode.mkv", outputPath: exactPath, disposition: "overwritten" }) + ]); + expect(parseNativeExtractOutput("UnRAR.exe", `Extracting ${exactPath} OK`, archivePath, targetDir, "overwrite")).toEqual([ + expect.objectContaining({ entryPath: "folder/episode.mkv", outputPath: exactPath }) + ]); + expect(parseNativeExtractOutput("7z.exe", "- ..\\foreign.mkv", archivePath, targetDir, "overwrite")).toEqual([]); + + const renamedPath = path.join(targetDir, "episode (1).mkv"); + fs.writeFileSync(path.join(targetDir, "episode.mkv"), "foreign"); + fs.writeFileSync(renamedPath, "owned"); + expect(parseNativeExtractOutput("7z.exe", "- episode.mkv", archivePath, targetDir, "rename")).toEqual([]); + expect(parseNativeExtractOutput("7z.exe", "- episode (1).mkv", archivePath, targetDir, "rename")).toEqual([ + expect.objectContaining({ outputPath: renamedPath, disposition: "renamed" }) + ]); + }); + }); }); diff --git a/tests/storage.test.ts b/tests/storage.test.ts index 9110556..321c205 100644 --- a/tests/storage.test.ts +++ b/tests/storage.test.ts @@ -1271,6 +1271,101 @@ describe("settings storage", () => { })); }); + it("fails closed for unknown package output provenance versions without relabeling", () => { + const normalized = normalizeLoadedSession({ + version: 2, + packageOrder: ["future-output"], + packages: { + "future-output": { + id: "future-output", + name: "Future output", + outputDir: "C:\\Downloads\\Future", + extractDir: "C:\\Downloads\\Shared", + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + outputCount: 2, + outputProvenanceVersion: 99, + outputProvenance: ["a".repeat(64), "b".repeat(64)], + outputRecords: [{ + version: 1, + archivePath: "C:\\Downloads\\Future\\archive.rar", + entryPath: "episode.mkv", + outputPath: "C:\\Downloads\\Shared\\episode.mkv", + state: "complete", + disposition: "written" + }], + createdAt: 1_000, + updatedAt: 2_000 + } + }, + items: {}, + runStartedAt: 0, + totalDownloadedBytes: 0, + summaryText: "", + reconnectUntil: 0, + reconnectReason: "", + paused: false, + running: false, + updatedAt: 2_000 + }); + + expect(normalized.packages["future-output"]).toEqual(expect.objectContaining({ + outputCount: 0, + outputProvenanceVersion: 99, + outputProvenance: [], + outputRecords: [] + })); + }); + + it("migrates unversioned valid provenance hashes and concrete output records to v1", () => { + const normalized = normalizeLoadedSession({ + version: 2, + packageOrder: ["legacy-hashes"], + packages: { + "legacy-hashes": { + id: "legacy-hashes", + name: "Legacy hashes", + outputDir: "C:\\Downloads\\Legacy", + extractDir: "C:\\Downloads\\Shared", + status: "completed", + itemIds: [], + cancelled: false, + enabled: true, + outputCount: 40_000, + outputProvenance: ["a".repeat(64)], + outputRecords: [{ + version: 1, + archivePath: "C:\\Downloads\\Legacy\\archive.rar", + entryPath: "episode.mkv", + outputPath: "C:\\Downloads\\Shared\\episode.mkv", + state: "complete", + disposition: "written" + }], + createdAt: 1_000, + updatedAt: 2_000 + } + }, + items: {}, + runStartedAt: 0, + totalDownloadedBytes: 0, + summaryText: "", + reconnectUntil: 0, + reconnectReason: "", + paused: false, + running: false, + updatedAt: 2_000 + }); + + expect(normalized.packages["legacy-hashes"]).toEqual(expect.objectContaining({ + outputCount: 1, + outputProvenanceVersion: 1, + outputProvenance: ["a".repeat(64)], + outputRecords: [expect.objectContaining({ entryPath: "episode.mkv", state: "complete" })] + })); + }); + it("skips adding persisted history entries when history retention is never", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rd-store-")); tempDirs.push(dir);