Administrator/real-debrid-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 467 Wiki Activity

Fix hybrid auto recovery loops

Browse Source
This commit is contained in:
Sucukdeluxe 2026-03-07 21:53:10 +01:00
parent 1222cb08b5
commit 9bc9c984cb
2 changed files with 377 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
src/main/download-manager.ts
View File

@ -63,6 +63,16 @@ type ActiveTask = {
blockedOnDiskSince?: number;
};
type PackageItemDiskState = {
diskPath: string | null;
exists: boolean;
size: number;
minBytes: number;
fullOnDisk: boolean;
persistedBytesReady: boolean;
reason: "ok" | "missing_path" | "missing_file" | "too_small" | "persisted_shortfall";
};
const DEFAULT_DOWNLOAD_STALL_TIMEOUT_MS = 10000;
const DEFAULT_DOWNLOAD_CONNECT_TIMEOUT_MS = 25000;
@ -87,6 +97,67 @@ const ALLDEBRID_START_STAGGER_MS = 2500;
const LARGE_BINARY_FILE_RE = /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|7z(?:\.\d+)?|tar|gz|bz2|xz|iso|mkv|mp4|avi|mov|wmv|m4v|ts|m2ts|webm|mp3|flac|aac|wav)$/i;
function itemExpectedMinBytes(item: DownloadItem): number {
return item.totalBytes && item.totalBytes > 0
? Math.max(10240, item.totalBytes - ALLOCATION_UNIT_SIZE)
: 10240;
}
function resolvePackageItemDiskPath(pkg: PackageEntry, item: DownloadItem): string | null {
if (item.targetPath) {
return item.targetPath;
}
if (item.fileName && pkg.outputDir) {
return path.join(pkg.outputDir, item.fileName);
}
return null;
}
function inspectPackageItemDiskState(pkg: PackageEntry, item: DownloadItem): PackageItemDiskState {
const minBytes = itemExpectedMinBytes(item);
const diskPath = resolvePackageItemDiskPath(pkg, item);
if (!diskPath) {
return {
diskPath: null,
exists: false,
size: 0,
minBytes,
fullOnDisk: false,
persistedBytesReady: false,
reason: "missing_path"
};
}
try {
const stat = fs.statSync(diskPath);
const fullOnDisk = stat.size >= minBytes;
const persistedBytesReady = item.downloadedBytes >= minBytes;
return {
diskPath,
exists: true,
size: stat.size,
minBytes,
fullOnDisk,
persistedBytesReady,
reason: !fullOnDisk
? "too_small"
: !persistedBytesReady
? "persisted_shortfall"
: "ok"
};
} catch {
return {
diskPath,
exists: false,
size: 0,
minBytes,
fullOnDisk: false,
persistedBytesReady: false,
reason: "missing_file"
};
}
}
function getDownloadStallTimeoutMs(): number {
const fromEnv = Number(process.env.RD_STALL_TIMEOUT_MS ?? NaN);
if (Number.isFinite(fromEnv) && fromEnv >= 2000 && fromEnv <= 600000) {
@ -4074,10 +4145,18 @@ export class DownloadManager extends EventEmitter {
return 0;
}
const corruptArchiveItems = archiveItems
.map((item) => ({ item, state: inspectPackageItemDiskState(pkg, item) }))
.filter(({ state }) => state.reason !== "ok");
if (corruptArchiveItems.length === 0) {
logger.warn(`Auto-Recovery (${scope}): ${failure.archiveName} uebersprungen - kein lokaler Dateifehler nachweisbar`);
return 0;
}
const queuedAt = nowMs();
const reason = "Wartet (Auto-Recovery: Archiv beschädigt/unvollständig)";
let changed = 0;
for (const item of archiveItems) {
for (const { item } of corruptArchiveItems) {
const claimedTargetPath = String(item.targetPath || "").trim();
if (claimedTargetPath) {
try {
@ -4103,9 +4182,14 @@ export class DownloadManager extends EventEmitter {
if (changed > 0) {
pkg.status = (pkg.enabled && this.session.running && !this.session.paused) ? "downloading" : "queued";
pkg.updatedAt = queuedAt;
const evidence = corruptArchiveItems
.slice(0, 3)
.map(({ item, state }) => `${item.fileName}:${state.reason}`)
.join(", ");
const suffix = corruptArchiveItems.length > 3 ? ` (+${corruptArchiveItems.length - 3} weitere)` : "";
logger.warn(
`Auto-Recovery (${scope}): ${failure.archiveName} auf queued gesetzt (${changed} Items), ` +
`reason=${compactErrorText(failure.jvmFailureReason || failure.errorText)}`
`evidence=${evidence}${suffix}, cause=${compactErrorText(failure.jvmFailureReason || failure.errorText)}`
);
this.persistSoon();
this.emitState();
@ -7037,10 +7121,14 @@ export class DownloadManager extends EventEmitter {
// Build lookup: pathKey → item status for pending items.
// Also map by filename (resolved against outputDir) so items without
// targetPath (never started) are still found by the disk-fallback check.
const packageItems = pkg.itemIds
.map((itemId) => this.session.items[itemId])
.filter(Boolean) as DownloadItem[];
const pendingItemStatus = new Map<string, string>();
for (const itemId of pkg.itemIds) {
const item = this.session.items[itemId];
if (!item || item.status === "completed") continue;
for (const item of packageItems) {
if (item.status === "completed") {
continue;
}
if (item.targetPath) {
pendingItemStatus.set(pathKey(item.targetPath), item.status);
}
@ -7065,6 +7153,30 @@ export class DownloadManager extends EventEmitter {
continue;
}
// Safe disk-fallback: only allow extraction when every tracked archive item
// already exists on disk at full size and the persisted byte counters
// also indicate a finished download. This recovers stale status after a
// crash without letting unrelated .rev files or freshly re-queued items
// look "ready".
const archiveItems = resolveArchiveItemsFromList(path.basename(candidate), packageItems);
if (archiveItems.length === 0) {
continue;
}
const hasActiveArchiveItem = archiveItems.some((item) =>
item.status === "downloading" || item.status === "validating" || item.status === "integrity_check"
);
if (hasActiveArchiveItem) {
continue;
}
const allArchiveItemsReadyOnDisk = archiveItems.every((item) => inspectPackageItemDiskState(pkg, item).reason === "ok");
if (!allArchiveItemsReadyOnDisk) {
continue;
}
const nonCompletedCount = archiveItems.filter((item) => item.status !== "completed").length;
logger.info(`Hybrid-Extract Disk-Fallback: ${path.basename(candidate)} (${nonCompletedCount} Part(s) laut Session ohne completed-Status)`);
ready.add(pathKey(candidate));
continue;
// Disk-fallback: if all parts exist on disk at their full expected size but some
// items lack "completed" status, allow extraction. This handles items that finished
// downloading but whose status was not updated (crash between write and persist).

260
tests/download-manager.test.ts
View File

@ -2042,6 +2042,266 @@ describe("download manager", () => {
expect(session.packages[packageId]?.status).toBe("queued");
});
it("does not requeue completed archive parts without local file evidence", () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);
const session = emptySession();
const packageId = "crc-clean-pkg";
const createdAt = Date.now() - 10_000;
const outputDir = path.join(root, "downloads", "crc-clean");
const extractDir = path.join(root, "extract", "crc-clean");
fs.mkdirSync(outputDir, { recursive: true });
const archiveNames = ["show.s01e01.part1.rar", "show.s01e01.part2.rar"];
const itemIds = archiveNames.map((_, index) => `crc-clean-item-${index}`);
const archiveSize = 64 * 1024;
session.packageOrder = [packageId];
session.packages[packageId] = {
id: packageId,
name: "crc-clean",
outputDir,
extractDir,
status: "extracting",
itemIds,
cancelled: false,
enabled: true,
createdAt,
updatedAt: createdAt
};
for (const [index, archiveName] of archiveNames.entries()) {
const targetPath = path.join(outputDir, archiveName);
fs.writeFileSync(targetPath, Buffer.alloc(archiveSize, index + 1));
session.items[itemIds[index]!] = {
id: itemIds[index]!,
packageId,
url: `https://dummy/${archiveName}`,
provider: "realdebrid",
status: "completed",
retries: 0,
speedBps: 0,
downloadedBytes: archiveSize,
totalBytes: archiveSize,
progressPercent: 100,
fileName: archiveName,
targetPath,
resumable: true,
attempts: 1,
lastError: "",
fullStatus: "Entpacken - Ausstehend",
createdAt,
updatedAt: createdAt
};
}
const manager = new DownloadManager(
{
...defaultSettings(),
token: "rd-token",
outputDir: path.join(root, "downloads"),
extractDir: path.join(root, "extract"),
autoExtract: true
},
session,
createStoragePaths(path.join(root, "state"))
);
const changed = (manager as any).autoRecoverArchiveCrcFailure(
session.packages[packageId],
itemIds.map((itemId) => session.items[itemId]!),
{
archiveName: "show.s01e01.part1.rar",
errorText: "Checksum error in the encrypted file",
category: "crc_error",
suggestRedownload: true,
jvmFailureReason: "Can not open the file as archive"
},
"hybrid"
);
expect(changed).toBe(0);
for (const itemId of itemIds) {
const item = session.items[itemId]!;
expect(item.status).toBe("completed");
expect(item.targetPath).toContain(".rar");
expect(item.downloadedBytes).toBe(archiveSize);
}
});
it("does not treat rev files as ready archive parts during disk fallback", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);
const session = emptySession();
const packageId = "disk-fallback-rev-pkg";
const itemIds = ["disk-fallback-rev-1", "disk-fallback-rev-2"];
const createdAt = Date.now() - 10_000;
const outputDir = path.join(root, "downloads", "disk-fallback-rev");
const extractDir = path.join(root, "extract", "disk-fallback-rev");
const part1Path = path.join(outputDir, "show.s01e01.part1.rar");
fs.mkdirSync(outputDir, { recursive: true });
fs.writeFileSync(part1Path, Buffer.alloc(64 * 1024, 1));
fs.writeFileSync(path.join(outputDir, "show.s01e01.rev"), Buffer.alloc(32 * 1024, 2));
session.packageOrder = [packageId];
session.packages[packageId] = {
id: packageId,
name: "disk-fallback-rev",
outputDir,
extractDir,
status: "downloading",
itemIds,
cancelled: false,
enabled: true,
createdAt,
updatedAt: createdAt
};
session.items[itemIds[0]] = {
id: itemIds[0],
packageId,
url: "https://dummy/show.s01e01.part1.rar",
provider: "realdebrid",
status: "completed",
retries: 0,
speedBps: 0,
downloadedBytes: 64 * 1024,
totalBytes: 64 * 1024,
progressPercent: 100,
fileName: "show.s01e01.part1.rar",
targetPath: part1Path,
resumable: true,
attempts: 1,
lastError: "",
fullStatus: "Entpacken - Ausstehend",
createdAt,
updatedAt: createdAt
};
session.items[itemIds[1]] = {
id: itemIds[1],
packageId,
url: "https://dummy/show.s01e01.part2.rar",
provider: "realdebrid",
status: "queued",
retries: 0,
speedBps: 0,
downloadedBytes: 0,
totalBytes: 64 * 1024,
progressPercent: 0,
fileName: "show.s01e01.part2.rar",
targetPath: "",
resumable: true,
attempts: 0,
lastError: "",
fullStatus: "Wartet",
createdAt,
updatedAt: createdAt
};
const manager = new DownloadManager(
{
...defaultSettings(),
token: "rd-token",
outputDir: path.join(root, "downloads"),
extractDir: path.join(root, "extract"),
autoExtract: true
},
session,
createStoragePaths(path.join(root, "state"))
);
const ready = await (manager as any).findReadyArchiveSets(session.packages[packageId]);
expect(Array.from(ready)).toHaveLength(0);
});
it("allows disk fallback when queued archive parts are fully present on disk", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);
const session = emptySession();
const packageId = "disk-fallback-ready-pkg";
const itemIds = ["disk-fallback-ready-1", "disk-fallback-ready-2"];
const createdAt = Date.now() - 10_000;
const outputDir = path.join(root, "downloads", "disk-fallback-ready");
const extractDir = path.join(root, "extract", "disk-fallback-ready");
const part1Path = path.join(outputDir, "show.s01e01.part1.rar");
const part2Path = path.join(outputDir, "show.s01e01.part2.rar");
const archiveSize = 64 * 1024;
fs.mkdirSync(outputDir, { recursive: true });
fs.writeFileSync(part1Path, Buffer.alloc(archiveSize, 1));
fs.writeFileSync(part2Path, Buffer.alloc(archiveSize, 2));
session.packageOrder = [packageId];
session.packages[packageId] = {
id: packageId,
name: "disk-fallback-ready",
outputDir,
extractDir,
status: "downloading",
itemIds,
cancelled: false,
enabled: true,
createdAt,
updatedAt: createdAt
};
session.items[itemIds[0]] = {
id: itemIds[0],
packageId,
url: "https://dummy/show.s01e01.part1.rar",
provider: "realdebrid",
status: "completed",
retries: 0,
speedBps: 0,
downloadedBytes: archiveSize,
totalBytes: archiveSize,
progressPercent: 100,
fileName: "show.s01e01.part1.rar",
targetPath: part1Path,
resumable: true,
attempts: 1,
lastError: "",
fullStatus: "Entpacken - Ausstehend",
createdAt,
updatedAt: createdAt
};
session.items[itemIds[1]] = {
id: itemIds[1],
packageId,
url: "https://dummy/show.s01e01.part2.rar",
provider: "realdebrid",
status: "queued",
retries: 0,
speedBps: 0,
downloadedBytes: archiveSize,
totalBytes: archiveSize,
progressPercent: 100,
fileName: "show.s01e01.part2.rar",
targetPath: "",
resumable: true,
attempts: 0,
lastError: "",
fullStatus: "Wartet",
createdAt,
updatedAt: createdAt
};
const manager = new DownloadManager(
{
...defaultSettings(),
token: "rd-token",
outputDir: path.join(root, "downloads"),
extractDir: path.join(root, "extract"),
autoExtract: true
},
session,
createStoragePaths(path.join(root, "state"))
);
const ready = await (manager as any).findReadyArchiveSets(session.packages[packageId]);
expect(Array.from(ready)).toEqual([part1Path.toLowerCase()]);
});
it("detects start conflicts when extract output already exists", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);