From 5c29355e9a65eb94d9b27221575218603e83bdc7 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Sat, 7 Mar 2026 22:13:51 +0100 Subject: [PATCH] Prevent repeated hybrid extraction retries --- src/main/download-manager.ts | 190 ++++++++++++++++++++++++++++----- tests/download-manager.test.ts | 76 +++++++++++++ 2 files changed, 242 insertions(+), 24 deletions(-) diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 351a4cc..60a12d9 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -73,6 +73,12 @@ type PackageItemDiskState = { reason: "ok" | "missing_path" | "missing_file" | "too_small" | "persisted_shortfall"; }; +type HybridFailedArchiveState = { + marker: string; + lastError: string; + updatedAt: number; +}; + const DEFAULT_DOWNLOAD_STALL_TIMEOUT_MS = 10000; const DEFAULT_DOWNLOAD_CONNECT_TIMEOUT_MS = 25000; @@ -436,6 +442,15 @@ function isExtractedLabel(statusText: string): boolean { return /^entpackt\b/i.test(String(statusText || "").trim()); } +function isExtractErrorLabel(statusText: string): boolean { + const text = String(statusText || "").trim(); + return /^entpacken\b/i.test(text) && /\berror\b/i.test(text); +} + +function shouldAutoRetryExtraction(statusText: string): boolean { + return !isExtractedLabel(statusText) && !isExtractErrorLabel(statusText); +} + function formatExtractDone(elapsedMs: number): string { if (elapsedMs < 1000) return "Entpackt - Done (<1s)"; const secs = elapsedMs / 1000; @@ -1082,10 +1097,14 @@ export class DownloadManager extends EventEmitter { private hybridExtractRequeue = new Set(); - // Tracks archive paths already attempted per package in the current post-processing session. - // Prevents infinite re-extraction of disk-fallback archives that have no session items. + // Tracks archive paths already attempted per package until the package/archive state changes + // or the user explicitly retries extraction. private hybridExtractedPaths = new Map>(); + // Tracks failed hybrid archives together with a lightweight state marker so unchanged + // archives are not retried on every subsequent post-processing wake-up. + private hybridFailedArchives = new Map>(); + private reservedTargetPaths = new Map(); private claimedTargetPathByItem = new Map(); @@ -1186,6 +1205,13 @@ export class DownloadManager extends EventEmitter { } } + const previousArchivePasswords = String(previous.archivePasswordList || "").replace(/\r\n|\r/g, "\n"); + const nextArchivePasswords = String(next.archivePasswordList || "").replace(/\r\n|\r/g, "\n"); + if (previousArchivePasswords !== nextArchivePasswords) { + this.hybridExtractedPaths.clear(); + this.hybridFailedArchives.clear(); + } + this.resolveExistingQueuedOpaqueFilenames(); void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (setSettings): ${compactErrorText(err)}`)); if (next.completedCleanupPolicy !== "never") { @@ -1550,6 +1576,7 @@ export class DownloadManager extends EventEmitter { this.packagePostProcessAbortControllers.clear(); this.hybridExtractRequeue.clear(); this.hybridExtractedPaths.clear(); + this.hybridFailedArchives.clear(); this.providerFailures.clear(); this.packagePostProcessQueue = Promise.resolve(); this.packagePostProcessActive = 0; @@ -1741,7 +1768,7 @@ export class DownloadManager extends EventEmitter { this.packagePostProcessAbortControllers.delete(packageId); this.packagePostProcessTasks.delete(packageId); this.hybridExtractRequeue.delete(packageId); - this.hybridExtractedPaths.delete(packageId); + this.clearHybridArchiveState(packageId); this.runPackageIds.delete(packageId); this.runCompletedPackages.delete(packageId); @@ -2927,7 +2954,7 @@ export class DownloadManager extends EventEmitter { this.packagePostProcessAbortControllers.delete(packageId); this.packagePostProcessTasks.delete(packageId); this.hybridExtractRequeue.delete(packageId); - this.hybridExtractedPaths.delete(packageId); + this.clearHybridArchiveState(packageId); this.runCompletedPackages.delete(packageId); // 3. Clean up extraction progress manifest (.rd_extract_progress.json) @@ -3017,7 +3044,7 @@ export class DownloadManager extends EventEmitter { this.packagePostProcessAbortControllers.delete(pkgId); this.packagePostProcessTasks.delete(pkgId); this.hybridExtractRequeue.delete(pkgId); - this.hybridExtractedPaths.delete(pkgId); + this.clearHybridArchiveState(pkgId); this.runCompletedPackages.delete(pkgId); this.historyRecordedPackages.delete(pkgId); @@ -3098,10 +3125,10 @@ export class DownloadManager extends EventEmitter { const pkgItems = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[]; const hasPending = pkgItems.some((i) => i.status !== "completed" && i.status !== "failed" && i.status !== "cancelled"); const hasFailed = pkgItems.some((i) => i.status === "failed"); - const hasUnextracted = pkgItems.some((i) => i.status === "completed" && !isExtractedLabel(i.fullStatus || "")); + const hasUnextracted = pkgItems.some((i) => i.status === "completed" && shouldAutoRetryExtraction(i.fullStatus || "")); if (!hasPending && !hasFailed && hasUnextracted) { for (const it of pkgItems) { - if (it.status === "completed" && !isExtractedLabel(it.fullStatus || "")) { + if (it.status === "completed" && shouldAutoRetryExtraction(it.fullStatus || "")) { it.fullStatus = "Entpacken - Ausstehend"; it.updatedAt = nowMs(); } @@ -4128,6 +4155,60 @@ export class DownloadManager extends EventEmitter { } } + private clearHybridArchiveState(packageId: string, archiveKey?: string): void { + if (!archiveKey) { + this.hybridExtractedPaths.delete(packageId); + this.hybridFailedArchives.delete(packageId); + return; + } + + const normalizedKey = pathKey(archiveKey); + const attempted = this.hybridExtractedPaths.get(packageId); + if (attempted) { + attempted.delete(normalizedKey); + if (attempted.size === 0) { + this.hybridExtractedPaths.delete(packageId); + } + } + + const failed = this.hybridFailedArchives.get(packageId); + if (failed) { + failed.delete(normalizedKey); + if (failed.size === 0) { + this.hybridFailedArchives.delete(packageId); + } + } + } + + private buildHybridArchiveRetryMarker(pkg: PackageEntry, items: DownloadItem[], archiveKey: string): string { + const archiveName = path.basename(archiveKey); + const archiveItems = resolveArchiveItemsFromList(archiveName, items) + .slice() + .sort((left, right) => { + const leftName = (left.fileName || left.targetPath || left.id || "").toLowerCase(); + const rightName = (right.fileName || right.targetPath || right.id || "").toLowerCase(); + return leftName.localeCompare(rightName); + }); + + const itemStates = archiveItems.map((item) => { + const diskState = inspectPackageItemDiskState(pkg, item); + return [ + (item.fileName || item.id || "").toLowerCase(), + item.status, + item.downloadedBytes || 0, + item.totalBytes || 0, + diskState.reason, + diskState.size + ].join("|"); + }); + + return JSON.stringify({ + archiveName: archiveName.toLowerCase(), + passwordList: String(this.settings.archivePasswordList || "").replace(/\r\n|\r/g, "\n").trim(), + itemStates + }); + } + private autoRecoverArchiveCrcFailure( pkg: PackageEntry, items: DownloadItem[], @@ -4180,6 +4261,7 @@ export class DownloadManager extends EventEmitter { } if (changed > 0) { + this.clearHybridArchiveState(pkg.id); pkg.status = (pkg.enabled && this.session.running && !this.session.paused) ? "downloading" : "queued"; pkg.updatedAt = queuedAt; const evidence = corruptArchiveItems @@ -4346,9 +4428,6 @@ export class DownloadManager extends EventEmitter { return existing; } - // Fresh session: reset the set of already-tried archives so new downloads can be retried. - this.hybridExtractedPaths.delete(packageId); - const abortController = new AbortController(); this.packagePostProcessAbortControllers.set(packageId, abortController); @@ -4424,12 +4503,12 @@ export class DownloadManager extends EventEmitter { // with pending extraction status → re-label and trigger post-processing // so extraction picks up where it left off. if (!allDone && this.settings.autoExtract && this.settings.hybridExtract && success > 0 && failed === 0) { - const needsExtraction = items.some((item) => item.status === "completed" && !isExtractedLabel(item.fullStatus)); + const needsExtraction = items.some((item) => item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)); if (needsExtraction) { pkg.status = "queued"; pkg.updatedAt = nowMs(); for (const item of items) { - if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) { + if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) { item.fullStatus = "Entpacken - Ausstehend"; item.updatedAt = nowMs(); } @@ -4445,12 +4524,12 @@ export class DownloadManager extends EventEmitter { } if (this.settings.autoExtract && failed === 0 && success > 0) { - const needsExtraction = items.some((item) => item.status === "completed" && !isExtractedLabel(item.fullStatus)); + const needsExtraction = items.some((item) => item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)); if (needsExtraction) { pkg.status = "queued"; pkg.updatedAt = nowMs(); for (const item of items) { - if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) { + if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) { item.fullStatus = "Entpacken - Ausstehend"; item.updatedAt = nowMs(); } @@ -4507,13 +4586,13 @@ export class DownloadManager extends EventEmitter { // Full extraction: all items done, no failures if (allDone && failed === 0 && success > 0) { const needsExtraction = items.some((item) => - item.status === "completed" && !isExtractedLabel(item.fullStatus) + item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus) ); if (needsExtraction) { pkg.status = "queued"; pkg.updatedAt = nowMs(); for (const item of items) { - if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) { + if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) { item.fullStatus = "Entpacken - Ausstehend"; item.updatedAt = nowMs(); } @@ -4527,13 +4606,13 @@ export class DownloadManager extends EventEmitter { // Hybrid extraction: not all items done, but some completed and no failures if (!allDone && this.settings.hybridExtract && success > 0 && failed === 0) { const needsExtraction = items.some((item) => - item.status === "completed" && !isExtractedLabel(item.fullStatus) + item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus) ); if (needsExtraction) { pkg.status = "queued"; pkg.updatedAt = nowMs(); for (const item of items) { - if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) { + if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) { item.fullStatus = "Entpacken - Ausstehend"; item.updatedAt = nowMs(); } @@ -4549,6 +4628,7 @@ export class DownloadManager extends EventEmitter { const pkg = this.session.packages[packageId]; if (!pkg) return; if (this.packagePostProcessTasks.has(packageId)) return; + this.clearHybridArchiveState(packageId); const items = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[]; const completedItems = items.filter((item) => item.status === "completed"); if (completedItems.length === 0) return; @@ -4570,6 +4650,7 @@ export class DownloadManager extends EventEmitter { const pkg = this.session.packages[packageId]; if (!pkg || pkg.cancelled) return; if (this.packagePostProcessTasks.has(packageId)) return; + this.clearHybridArchiveState(packageId); if (!pkg.enabled) { pkg.enabled = true; } @@ -4669,7 +4750,7 @@ export class DownloadManager extends EventEmitter { // causing "Start Selected" to continue with ALL packages after cleanup. this.runCompletedPackages.delete(packageId); this.hybridExtractRequeue.delete(packageId); - this.hybridExtractedPaths.delete(packageId); + this.clearHybridArchiveState(packageId); this.resetSessionTotalsIfQueueEmpty(); } @@ -7280,8 +7361,10 @@ export class DownloadManager extends EventEmitter { logger.info(`findReadyArchiveSets dauerte ${(findReadyMs / 1000).toFixed(1)}s: pkg=${pkg.name}, found=${readyArchives.size}`); } - // Skip archives already attempted in this post-processing session to prevent - // infinite re-extraction of disk-fallback archives with no session items. + const completedItems = items.filter((item) => item.status === "completed"); + + // Skip archives already attempted in the current package/archive state to prevent + // infinite re-extraction of disk-fallback archives or repeated unchanged failures. const alreadyTried = this.hybridExtractedPaths.get(packageId); if (alreadyTried) { for (const key of [...readyArchives]) { @@ -7291,6 +7374,29 @@ export class DownloadManager extends EventEmitter { } } + const failedArchiveStates = this.hybridFailedArchives.get(packageId); + if (failedArchiveStates) { + for (const archiveKey of [...readyArchives]) { + const previousFailure = failedArchiveStates.get(archiveKey); + if (!previousFailure) { + continue; + } + + const archiveItems = resolveArchiveItemsFromList(path.basename(archiveKey), completedItems); + const allItemsStillInError = archiveItems.length > 0 && archiveItems.every((item) => isExtractErrorLabel(item.fullStatus)); + const retryMarker = this.buildHybridArchiveRetryMarker(pkg, items, archiveKey); + if (!allItemsStillInError || previousFailure.marker !== retryMarker) { + continue; + } + + logger.info( + `Hybrid-Extract Skip: ${path.basename(archiveKey)} unveraendert seit letztem Fehler ` + + `(${compactErrorText(previousFailure.lastError)})` + ); + readyArchives.delete(archiveKey); + } + } + if (readyArchives.size === 0) { logger.info(`Hybrid-Extract: pkg=${pkg.name}, keine fertigen Archive-Sets`); return 0; @@ -7301,8 +7407,6 @@ export class DownloadManager extends EventEmitter { this.emitState(); const hybridExtractStartMs = nowMs(); - const completedItems = items.filter((item) => item.status === "completed"); - // Build set of file names belonging to ready archives (for matching items) const hybridFileNames = new Set(); let dirFiles: string[] | undefined; @@ -7365,8 +7469,16 @@ export class DownloadManager extends EventEmitter { const resolveArchiveItems = (archiveName: string): DownloadItem[] => resolveArchiveItemsFromList(archiveName, items); + const readyArchiveKeyByName = new Map(); + const readyArchiveMarkers = new Map(); + for (const archiveKey of readyArchives) { + readyArchiveKeyByName.set(path.basename(archiveKey).toLowerCase(), archiveKey); + readyArchiveMarkers.set(archiveKey, this.buildHybridArchiveRetryMarker(pkg, items, archiveKey)); + } + // Track archives for parallel hybrid extraction progress const autoRecoveredArchives = new Set(); + const failedArchiveErrors = new Map(); const hybridResolvedItems = new Map(); const hybridStartTimes = new Map(); let hybridLastEmitAt = 0; @@ -7381,6 +7493,9 @@ export class DownloadManager extends EventEmitter { if (isExtractedLabel(entry.fullStatus)) { continue; } + if (isExtractErrorLabel(entry.fullStatus)) { + continue; + } const belongsToReady = allDownloaded || hybridFileNames.has((entry.fileName || "").toLowerCase()) || (entry.targetPath && hybridFileNames.has(path.basename(entry.targetPath).toLowerCase())); @@ -7412,6 +7527,10 @@ export class DownloadManager extends EventEmitter { maxParallel: this.settings.maxParallelExtract || 2, extractCpuPriority: "high", onArchiveFailure: (failure) => { + const failedArchiveKey = readyArchiveKeyByName.get(String(failure.archiveName || "").toLowerCase()); + if (failedArchiveKey) { + failedArchiveErrors.set(failedArchiveKey, failure.errorText || failure.jvmFailureReason || "Entpacken fehlgeschlagen"); + } if (autoRecoveredArchives.has(failure.archiveName)) { return; } @@ -7473,6 +7592,10 @@ export class DownloadManager extends EventEmitter { const doneLabel = progress.archiveSuccess === false ? "Entpacken - Error" : formatExtractDone(doneAt - startedAt); + const archiveKey = readyArchiveKeyByName.get(progress.archiveName.toLowerCase()); + if (archiveKey && progress.archiveSuccess !== false) { + this.clearHybridArchiveState(packageId, archiveKey); + } for (const entry of archItems) { if (entry.status !== "completed" || isExtractedLabel(entry.fullStatus)) continue; entry.fullStatus = doneLabel; @@ -7548,6 +7671,25 @@ export class DownloadManager extends EventEmitter { if (!tried) { tried = new Set(); this.hybridExtractedPaths.set(packageId, tried); } for (const key of readyArchives) { tried.add(key); } } + if (failedArchiveErrors.size > 0) { + let failed = this.hybridFailedArchives.get(packageId); + if (!failed) { + failed = new Map(); + this.hybridFailedArchives.set(packageId, failed); + } + const failedAt = nowMs(); + for (const [archiveKey, errorText] of failedArchiveErrors.entries()) { + const marker = readyArchiveMarkers.get(archiveKey); + if (!marker) { + continue; + } + failed.set(archiveKey, { + marker, + lastError: errorText, + updatedAt: failedAt + }); + } + } if (result.extracted > 0) { // Fire-and-forget: rename then collect MKVs in background so the // slot is not blocked and the next archive set can start immediately. @@ -7565,7 +7707,7 @@ export class DownloadManager extends EventEmitter { })(); } if (result.failed > 0) { - logger.warn(`Hybrid-Extract: ${result.failed} Archive fehlgeschlagen, wird beim finalen Durchlauf erneut versucht`); + logger.warn(`Hybrid-Extract: ${result.failed} Archive fehlgeschlagen, werden erst nach echter Aenderung oder manuellem Retry erneut versucht`); } // Mark hybrid items with final status — only items whose archives were diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index 95eac36..d5cc427 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -2302,6 +2302,82 @@ describe("download manager", () => { expect(Array.from(ready)).toEqual([part1Path.toLowerCase()]); }); + it("skips unchanged hybrid archives after a previous extraction failure", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + + const { + session, + packageId, + itemId, + outputDir, + extractDir + } = createCompletedArchiveSession(root, "hybrid-failure-skip", "episode.mkv"); + const item = session.items[itemId]!; + const archiveKey = item.targetPath.toLowerCase(); + item.fullStatus = "Entpacken - Error"; + session.packages[packageId]!.status = "queued"; + + const manager = new DownloadManager( + { + ...defaultSettings(), + token: "rd-token", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: true, + hybridExtract: true + }, + session, + createStoragePaths(path.join(root, "state")) + ); + + const pkg = (manager as any).session.packages[packageId]; + const items = [((manager as any).session.items[itemId])]; + const marker = (manager as any).buildHybridArchiveRetryMarker(pkg, items, archiveKey); + (manager as any).hybridFailedArchives.set(packageId, new Map([ + [archiveKey, { marker, lastError: "Checksum error in the encrypted file", updatedAt: Date.now() }] + ])); + + const extracted = await (manager as any).runHybridExtraction(packageId, pkg, items); + + expect(extracted).toBe(0); + expect(fs.existsSync(path.join(extractDir, "episode.mkv"))).toBe(false); + expect(((manager as any).session.items[itemId]).fullStatus).toBe("Entpacken - Error"); + expect(((manager as any).session.packages[packageId]).status).not.toBe("extracting"); + expect(fs.existsSync(path.join(outputDir, "episode.zip"))).toBe(true); + }); + + it("does not auto-reschedule extraction for completed items already marked as extract error", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + + const { + session, + packageId, + itemId + } = createCompletedArchiveSession(root, "hybrid-error-hold", "episode.mkv"); + session.items[itemId]!.fullStatus = "Entpacken - Error"; + session.packages[packageId]!.status = "queued"; + + const manager = new DownloadManager( + { + ...defaultSettings(), + token: "rd-token", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: true, + hybridExtract: true + }, + session, + createStoragePaths(path.join(root, "state")) + ); + + (manager as any).triggerPendingExtractions(); + + expect((manager as any).packagePostProcessTasks.has(packageId)).toBe(false); + expect((manager as any).session.items[itemId].fullStatus).toBe("Entpacken - Error"); + }); + it("detects start conflicts when extract output already exists", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root);