Administrator/beta-real-debrid-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Prevent repeated hybrid extraction retries

Browse Source
This commit is contained in:
Sucukdeluxe 2026-03-07 22:13:51 +01:00
parent 960b1fa046
commit 5c29355e9a
2 changed files with 242 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
src/main/download-manager.ts
View File

@ -73,6 +73,12 @@ type PackageItemDiskState = {
reason: "ok" | "missing_path" | "missing_file" | "too_small" | "persisted_shortfall";
};
type HybridFailedArchiveState = {
marker: string;
lastError: string;
updatedAt: number;
};
const DEFAULT_DOWNLOAD_STALL_TIMEOUT_MS = 10000;
const DEFAULT_DOWNLOAD_CONNECT_TIMEOUT_MS = 25000;
@ -436,6 +442,15 @@ function isExtractedLabel(statusText: string): boolean {
return /^entpackt\b/i.test(String(statusText || "").trim());
}
function isExtractErrorLabel(statusText: string): boolean {
const text = String(statusText || "").trim();
return /^entpacken\b/i.test(text) && /\berror\b/i.test(text);
}
function shouldAutoRetryExtraction(statusText: string): boolean {
return !isExtractedLabel(statusText) && !isExtractErrorLabel(statusText);
}
function formatExtractDone(elapsedMs: number): string {
if (elapsedMs < 1000) return "Entpackt - Done (<1s)";
const secs = elapsedMs / 1000;
@ -1082,10 +1097,14 @@ export class DownloadManager extends EventEmitter {
private hybridExtractRequeue = new Set<string>();
// Tracks archive paths already attempted per package in the current post-processing session.
// Prevents infinite re-extraction of disk-fallback archives that have no session items.
// Tracks archive paths already attempted per package until the package/archive state changes
// or the user explicitly retries extraction.
private hybridExtractedPaths = new Map<string, Set<string>>();
// Tracks failed hybrid archives together with a lightweight state marker so unchanged
// archives are not retried on every subsequent post-processing wake-up.
private hybridFailedArchives = new Map<string, Map<string, HybridFailedArchiveState>>();
private reservedTargetPaths = new Map<string, string>();
private claimedTargetPathByItem = new Map<string, string>();
@ -1186,6 +1205,13 @@ export class DownloadManager extends EventEmitter {
}
}
const previousArchivePasswords = String(previous.archivePasswordList || "").replace(/\r\n|\r/g, "\n");
const nextArchivePasswords = String(next.archivePasswordList || "").replace(/\r\n|\r/g, "\n");
if (previousArchivePasswords !== nextArchivePasswords) {
this.hybridExtractedPaths.clear();
this.hybridFailedArchives.clear();
}
this.resolveExistingQueuedOpaqueFilenames();
void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (setSettings): ${compactErrorText(err)}`));
if (next.completedCleanupPolicy !== "never") {
@ -1550,6 +1576,7 @@ export class DownloadManager extends EventEmitter {
this.packagePostProcessAbortControllers.clear();
this.hybridExtractRequeue.clear();
this.hybridExtractedPaths.clear();
this.hybridFailedArchives.clear();
this.providerFailures.clear();
this.packagePostProcessQueue = Promise.resolve();
this.packagePostProcessActive = 0;
@ -1741,7 +1768,7 @@ export class DownloadManager extends EventEmitter {
this.packagePostProcessAbortControllers.delete(packageId);
this.packagePostProcessTasks.delete(packageId);
this.hybridExtractRequeue.delete(packageId);
this.hybridExtractedPaths.delete(packageId);
this.clearHybridArchiveState(packageId);
this.runPackageIds.delete(packageId);
this.runCompletedPackages.delete(packageId);
@ -2927,7 +2954,7 @@ export class DownloadManager extends EventEmitter {
this.packagePostProcessAbortControllers.delete(packageId);
this.packagePostProcessTasks.delete(packageId);
this.hybridExtractRequeue.delete(packageId);
this.hybridExtractedPaths.delete(packageId);
this.clearHybridArchiveState(packageId);
this.runCompletedPackages.delete(packageId);
// 3. Clean up extraction progress manifest (.rd_extract_progress.json)
@ -3017,7 +3044,7 @@ export class DownloadManager extends EventEmitter {
this.packagePostProcessAbortControllers.delete(pkgId);
this.packagePostProcessTasks.delete(pkgId);
this.hybridExtractRequeue.delete(pkgId);
this.hybridExtractedPaths.delete(pkgId);
this.clearHybridArchiveState(pkgId);
this.runCompletedPackages.delete(pkgId);
this.historyRecordedPackages.delete(pkgId);
@ -3098,10 +3125,10 @@ export class DownloadManager extends EventEmitter {
const pkgItems = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[];
const hasPending = pkgItems.some((i) => i.status !== "completed" && i.status !== "failed" && i.status !== "cancelled");
const hasFailed = pkgItems.some((i) => i.status === "failed");
const hasUnextracted = pkgItems.some((i) => i.status === "completed" && !isExtractedLabel(i.fullStatus || ""));
const hasUnextracted = pkgItems.some((i) => i.status === "completed" && shouldAutoRetryExtraction(i.fullStatus || ""));
if (!hasPending && !hasFailed && hasUnextracted) {
for (const it of pkgItems) {
if (it.status === "completed" && !isExtractedLabel(it.fullStatus || "")) {
if (it.status === "completed" && shouldAutoRetryExtraction(it.fullStatus || "")) {
it.fullStatus = "Entpacken - Ausstehend";
it.updatedAt = nowMs();
}
@ -4128,6 +4155,60 @@ export class DownloadManager extends EventEmitter {
}
}
private clearHybridArchiveState(packageId: string, archiveKey?: string): void {
if (!archiveKey) {
this.hybridExtractedPaths.delete(packageId);
this.hybridFailedArchives.delete(packageId);
return;
}
const normalizedKey = pathKey(archiveKey);
const attempted = this.hybridExtractedPaths.get(packageId);
if (attempted) {
attempted.delete(normalizedKey);
if (attempted.size === 0) {
this.hybridExtractedPaths.delete(packageId);
}
}
const failed = this.hybridFailedArchives.get(packageId);
if (failed) {
failed.delete(normalizedKey);
if (failed.size === 0) {
this.hybridFailedArchives.delete(packageId);
}
}
}
private buildHybridArchiveRetryMarker(pkg: PackageEntry, items: DownloadItem[], archiveKey: string): string {
const archiveName = path.basename(archiveKey);
const archiveItems = resolveArchiveItemsFromList(archiveName, items)
.slice()
.sort((left, right) => {
const leftName = (left.fileName || left.targetPath || left.id || "").toLowerCase();
const rightName = (right.fileName || right.targetPath || right.id || "").toLowerCase();
return leftName.localeCompare(rightName);
});
const itemStates = archiveItems.map((item) => {
const diskState = inspectPackageItemDiskState(pkg, item);
return [
(item.fileName || item.id || "").toLowerCase(),
item.status,
item.downloadedBytes || 0,
item.totalBytes || 0,
diskState.reason,
diskState.size
].join("|");
});
return JSON.stringify({
archiveName: archiveName.toLowerCase(),
passwordList: String(this.settings.archivePasswordList || "").replace(/\r\n|\r/g, "\n").trim(),
itemStates
});
}
private autoRecoverArchiveCrcFailure(
pkg: PackageEntry,
items: DownloadItem[],
@ -4180,6 +4261,7 @@ export class DownloadManager extends EventEmitter {
}
if (changed > 0) {
this.clearHybridArchiveState(pkg.id);
pkg.status = (pkg.enabled && this.session.running && !this.session.paused) ? "downloading" : "queued";
pkg.updatedAt = queuedAt;
const evidence = corruptArchiveItems
@ -4346,9 +4428,6 @@ export class DownloadManager extends EventEmitter {
return existing;
}
// Fresh session: reset the set of already-tried archives so new downloads can be retried.
this.hybridExtractedPaths.delete(packageId);
const abortController = new AbortController();
this.packagePostProcessAbortControllers.set(packageId, abortController);
@ -4424,12 +4503,12 @@ export class DownloadManager extends EventEmitter {
// with pending extraction status → re-label and trigger post-processing
// so extraction picks up where it left off.
if (!allDone && this.settings.autoExtract && this.settings.hybridExtract && success > 0 && failed === 0) {
const needsExtraction = items.some((item) => item.status === "completed" && !isExtractedLabel(item.fullStatus));
const needsExtraction = items.some((item) => item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus));
if (needsExtraction) {
pkg.status = "queued";
pkg.updatedAt = nowMs();
for (const item of items) {
if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) {
if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) {
item.fullStatus = "Entpacken - Ausstehend";
item.updatedAt = nowMs();
}
@ -4445,12 +4524,12 @@ export class DownloadManager extends EventEmitter {
}
if (this.settings.autoExtract && failed === 0 && success > 0) {
const needsExtraction = items.some((item) => item.status === "completed" && !isExtractedLabel(item.fullStatus));
const needsExtraction = items.some((item) => item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus));
if (needsExtraction) {
pkg.status = "queued";
pkg.updatedAt = nowMs();
for (const item of items) {
if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) {
if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) {
item.fullStatus = "Entpacken - Ausstehend";
item.updatedAt = nowMs();
}
@ -4507,13 +4586,13 @@ export class DownloadManager extends EventEmitter {
// Full extraction: all items done, no failures
if (allDone && failed === 0 && success > 0) {
const needsExtraction = items.some((item) =>
item.status === "completed" && !isExtractedLabel(item.fullStatus)
item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)
);
if (needsExtraction) {
pkg.status = "queued";
pkg.updatedAt = nowMs();
for (const item of items) {
if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) {
if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) {
item.fullStatus = "Entpacken - Ausstehend";
item.updatedAt = nowMs();
}
@ -4527,13 +4606,13 @@ export class DownloadManager extends EventEmitter {
// Hybrid extraction: not all items done, but some completed and no failures
if (!allDone && this.settings.hybridExtract && success > 0 && failed === 0) {
const needsExtraction = items.some((item) =>
item.status === "completed" && !isExtractedLabel(item.fullStatus)
item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)
);
if (needsExtraction) {
pkg.status = "queued";
pkg.updatedAt = nowMs();
for (const item of items) {
if (item.status === "completed" && !isExtractedLabel(item.fullStatus)) {
if (item.status === "completed" && shouldAutoRetryExtraction(item.fullStatus)) {
item.fullStatus = "Entpacken - Ausstehend";
item.updatedAt = nowMs();
}
@ -4549,6 +4628,7 @@ export class DownloadManager extends EventEmitter {
const pkg = this.session.packages[packageId];
if (!pkg) return;
if (this.packagePostProcessTasks.has(packageId)) return;
this.clearHybridArchiveState(packageId);
const items = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[];
const completedItems = items.filter((item) => item.status === "completed");
if (completedItems.length === 0) return;
@ -4570,6 +4650,7 @@ export class DownloadManager extends EventEmitter {
const pkg = this.session.packages[packageId];
if (!pkg || pkg.cancelled) return;
if (this.packagePostProcessTasks.has(packageId)) return;
this.clearHybridArchiveState(packageId);
if (!pkg.enabled) {
pkg.enabled = true;
}
@ -4669,7 +4750,7 @@ export class DownloadManager extends EventEmitter {
// causing "Start Selected" to continue with ALL packages after cleanup.
this.runCompletedPackages.delete(packageId);
this.hybridExtractRequeue.delete(packageId);
this.hybridExtractedPaths.delete(packageId);
this.clearHybridArchiveState(packageId);
this.resetSessionTotalsIfQueueEmpty();
}
@ -7280,8 +7361,10 @@ export class DownloadManager extends EventEmitter {
logger.info(`findReadyArchiveSets dauerte ${(findReadyMs / 1000).toFixed(1)}s: pkg=${pkg.name}, found=${readyArchives.size}`);
}
// Skip archives already attempted in this post-processing session to prevent
// infinite re-extraction of disk-fallback archives with no session items.
const completedItems = items.filter((item) => item.status === "completed");
// Skip archives already attempted in the current package/archive state to prevent
// infinite re-extraction of disk-fallback archives or repeated unchanged failures.
const alreadyTried = this.hybridExtractedPaths.get(packageId);
if (alreadyTried) {
for (const key of [...readyArchives]) {
@ -7291,6 +7374,29 @@ export class DownloadManager extends EventEmitter {
}
}
const failedArchiveStates = this.hybridFailedArchives.get(packageId);
if (failedArchiveStates) {
for (const archiveKey of [...readyArchives]) {
const previousFailure = failedArchiveStates.get(archiveKey);
if (!previousFailure) {
continue;
}
const archiveItems = resolveArchiveItemsFromList(path.basename(archiveKey), completedItems);
const allItemsStillInError = archiveItems.length > 0 && archiveItems.every((item) => isExtractErrorLabel(item.fullStatus));
const retryMarker = this.buildHybridArchiveRetryMarker(pkg, items, archiveKey);
if (!allItemsStillInError || previousFailure.marker !== retryMarker) {
continue;
}
logger.info(
`Hybrid-Extract Skip: ${path.basename(archiveKey)} unveraendert seit letztem Fehler ` +
`(${compactErrorText(previousFailure.lastError)})`
);
readyArchives.delete(archiveKey);
}
}
if (readyArchives.size === 0) {
logger.info(`Hybrid-Extract: pkg=${pkg.name}, keine fertigen Archive-Sets`);
return 0;
@ -7301,8 +7407,6 @@ export class DownloadManager extends EventEmitter {
this.emitState();
const hybridExtractStartMs = nowMs();
const completedItems = items.filter((item) => item.status === "completed");
// Build set of file names belonging to ready archives (for matching items)
const hybridFileNames = new Set<string>();
let dirFiles: string[] | undefined;
@ -7365,8 +7469,16 @@ export class DownloadManager extends EventEmitter {
const resolveArchiveItems = (archiveName: string): DownloadItem[] =>
resolveArchiveItemsFromList(archiveName, items);
const readyArchiveKeyByName = new Map<string, string>();
const readyArchiveMarkers = new Map<string, string>();
for (const archiveKey of readyArchives) {
readyArchiveKeyByName.set(path.basename(archiveKey).toLowerCase(), archiveKey);
readyArchiveMarkers.set(archiveKey, this.buildHybridArchiveRetryMarker(pkg, items, archiveKey));
}
// Track archives for parallel hybrid extraction progress
const autoRecoveredArchives = new Set<string>();
const failedArchiveErrors = new Map<string, string>();
const hybridResolvedItems = new Map<string, DownloadItem[]>();
const hybridStartTimes = new Map<string, number>();
let hybridLastEmitAt = 0;
@ -7381,6 +7493,9 @@ export class DownloadManager extends EventEmitter {
if (isExtractedLabel(entry.fullStatus)) {
continue;
}
if (isExtractErrorLabel(entry.fullStatus)) {
continue;
}
const belongsToReady = allDownloaded
|| hybridFileNames.has((entry.fileName || "").toLowerCase())
|| (entry.targetPath && hybridFileNames.has(path.basename(entry.targetPath).toLowerCase()));
@ -7412,6 +7527,10 @@ export class DownloadManager extends EventEmitter {
maxParallel: this.settings.maxParallelExtract || 2,
extractCpuPriority: "high",
onArchiveFailure: (failure) => {
const failedArchiveKey = readyArchiveKeyByName.get(String(failure.archiveName || "").toLowerCase());
if (failedArchiveKey) {
failedArchiveErrors.set(failedArchiveKey, failure.errorText || failure.jvmFailureReason || "Entpacken fehlgeschlagen");
}
if (autoRecoveredArchives.has(failure.archiveName)) {
return;
}
@ -7473,6 +7592,10 @@ export class DownloadManager extends EventEmitter {
const doneLabel = progress.archiveSuccess === false
? "Entpacken - Error"
: formatExtractDone(doneAt - startedAt);
const archiveKey = readyArchiveKeyByName.get(progress.archiveName.toLowerCase());
if (archiveKey && progress.archiveSuccess !== false) {
this.clearHybridArchiveState(packageId, archiveKey);
}
for (const entry of archItems) {
if (entry.status !== "completed" || isExtractedLabel(entry.fullStatus)) continue;
entry.fullStatus = doneLabel;
@ -7548,6 +7671,25 @@ export class DownloadManager extends EventEmitter {
if (!tried) { tried = new Set(); this.hybridExtractedPaths.set(packageId, tried); }
for (const key of readyArchives) { tried.add(key); }
}
if (failedArchiveErrors.size > 0) {
let failed = this.hybridFailedArchives.get(packageId);
if (!failed) {
failed = new Map();
this.hybridFailedArchives.set(packageId, failed);
}
const failedAt = nowMs();
for (const [archiveKey, errorText] of failedArchiveErrors.entries()) {
const marker = readyArchiveMarkers.get(archiveKey);
if (!marker) {
continue;
}
failed.set(archiveKey, {
marker,
lastError: errorText,
updatedAt: failedAt
});
}
}
if (result.extracted > 0) {
// Fire-and-forget: rename then collect MKVs in background so the
// slot is not blocked and the next archive set can start immediately.
@ -7565,7 +7707,7 @@ export class DownloadManager extends EventEmitter {
})();
}
if (result.failed > 0) {
logger.warn(`Hybrid-Extract: ${result.failed} Archive fehlgeschlagen, wird beim finalen Durchlauf erneut versucht`);
logger.warn(`Hybrid-Extract: ${result.failed} Archive fehlgeschlagen, werden erst nach echter Aenderung oder manuellem Retry erneut versucht`);
}
// Mark hybrid items with final status — only items whose archives were

76
tests/download-manager.test.ts
View File

@ -2302,6 +2302,82 @@ describe("download manager", () => {
expect(Array.from(ready)).toEqual([part1Path.toLowerCase()]);
});
it("skips unchanged hybrid archives after a previous extraction failure", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);
const {
session,
packageId,
itemId,
outputDir,
extractDir
} = createCompletedArchiveSession(root, "hybrid-failure-skip", "episode.mkv");
const item = session.items[itemId]!;
const archiveKey = item.targetPath.toLowerCase();
item.fullStatus = "Entpacken - Error";
session.packages[packageId]!.status = "queued";
const manager = new DownloadManager(
{
...defaultSettings(),
token: "rd-token",
outputDir: path.join(root, "downloads"),
extractDir: path.join(root, "extract"),
autoExtract: true,
hybridExtract: true
},
session,
createStoragePaths(path.join(root, "state"))
);
const pkg = (manager as any).session.packages[packageId];
const items = [((manager as any).session.items[itemId])];
const marker = (manager as any).buildHybridArchiveRetryMarker(pkg, items, archiveKey);
(manager as any).hybridFailedArchives.set(packageId, new Map([
[archiveKey, { marker, lastError: "Checksum error in the encrypted file", updatedAt: Date.now() }]
]));
const extracted = await (manager as any).runHybridExtraction(packageId, pkg, items);
expect(extracted).toBe(0);
expect(fs.existsSync(path.join(extractDir, "episode.mkv"))).toBe(false);
expect(((manager as any).session.items[itemId]).fullStatus).toBe("Entpacken - Error");
expect(((manager as any).session.packages[packageId]).status).not.toBe("extracting");
expect(fs.existsSync(path.join(outputDir, "episode.zip"))).toBe(true);
});
it("does not auto-reschedule extraction for completed items already marked as extract error", () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);
const {
session,
packageId,
itemId
} = createCompletedArchiveSession(root, "hybrid-error-hold", "episode.mkv");
session.items[itemId]!.fullStatus = "Entpacken - Error";
session.packages[packageId]!.status = "queued";
const manager = new DownloadManager(
{
...defaultSettings(),
token: "rd-token",
outputDir: path.join(root, "downloads"),
extractDir: path.join(root, "extract"),
autoExtract: true,
hybridExtract: true
},
session,
createStoragePaths(path.join(root, "state"))
);
(manager as any).triggerPendingExtractions();
expect((manager as any).packagePostProcessTasks.has(packageId)).toBe(false);
expect((manager as any).session.items[itemId].fullStatus).toBe("Entpacken - Error");
});
it("detects start conflicts when extract output already exists", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);