diff --git a/src/main/download-completion.ts b/src/main/download-completion.ts new file mode 100644 index 0000000..a11430f --- /dev/null +++ b/src/main/download-completion.ts @@ -0,0 +1,132 @@ +import { ALLOCATION_UNIT_SIZE } from "./constants"; + +export type DownloadCompletionSource = + | "content-range" + | "content-length" + | "provider-metadata" + | "stream-end"; + +export type DownloadCompletionPlan = { + expectedTotal: number | null; + source: DownloadCompletionSource; + canFinishEarly: boolean; +}; + +export function planDownloadCompletion(args: { + existingBytes: number; + responseStatus: number; + contentLength: number; + totalFromRange: number | null; + knownTotal: number | null; + correctedTotal: number | null; +}): DownloadCompletionPlan { + const existingBytes = Math.max(0, Math.floor(Number(args.existingBytes) || 0)); + const responseStatus = Math.floor(Number(args.responseStatus) || 0); + const contentLength = Math.max(0, Math.floor(Number(args.contentLength) || 0)); + const totalFromRange = Number.isFinite(args.totalFromRange || NaN) + ? Math.max(0, Math.floor(args.totalFromRange || 0)) + : 0; + const correctedTotal = Number.isFinite(args.correctedTotal || NaN) + ? Math.max(0, Math.floor(args.correctedTotal || 0)) + : 0; + const knownTotal = Number.isFinite(args.knownTotal || NaN) + ? Math.max(0, Math.floor(args.knownTotal || 0)) + : 0; + + if (correctedTotal > 0) { + return { + expectedTotal: correctedTotal, + source: totalFromRange > 0 ? "content-range" : "content-length", + canFinishEarly: true + }; + } + + if (totalFromRange > 0) { + return { + expectedTotal: totalFromRange, + source: "content-range", + canFinishEarly: true + }; + } + + if (contentLength > 0) { + return { + expectedTotal: responseStatus === 206 ? existingBytes + contentLength : contentLength, + source: "content-length", + canFinishEarly: true + }; + } + + if (knownTotal > 0) { + return { + expectedTotal: knownTotal, + source: "provider-metadata", + canFinishEarly: false + }; + } + + return { + expectedTotal: null, + source: "stream-end", + canFinishEarly: false + }; +} + +export function validateDownloadedFileCompletion(args: { + actualBytes: number; + plan: DownloadCompletionPlan; +}): { + ok: boolean; + totalBytes: number; + acceptedMetadataMismatch: boolean; + error?: string; +} { + const actualBytes = Math.max(0, Math.floor(Number(args.actualBytes) || 0)); + const expectedTotal = Number.isFinite(args.plan.expectedTotal || NaN) + ? Math.max(0, Math.floor(args.plan.expectedTotal || 0)) + : 0; + + if ( + expectedTotal > 0 && + (args.plan.source === "content-range" || args.plan.source === "content-length") && + actualBytes + ALLOCATION_UNIT_SIZE < expectedTotal + ) { + return { + ok: false, + totalBytes: expectedTotal, + acceptedMetadataMismatch: false, + error: `download_underflow:${actualBytes}/${expectedTotal}` + }; + } + + if (actualBytes <= 0 && expectedTotal > 0) { + return { + ok: false, + totalBytes: expectedTotal, + acceptedMetadataMismatch: false, + error: `download_underflow:${actualBytes}/${expectedTotal}` + }; + } + + if (args.plan.source === "provider-metadata") { + return { + ok: true, + totalBytes: actualBytes, + acceptedMetadataMismatch: expectedTotal > 0 && Math.abs(actualBytes - expectedTotal) > ALLOCATION_UNIT_SIZE + }; + } + + if (args.plan.source === "stream-end") { + return { + ok: true, + totalBytes: actualBytes, + acceptedMetadataMismatch: false + }; + } + + return { + ok: true, + totalBytes: Math.max(actualBytes, expectedTotal), + acceptedMetadataMismatch: false + }; +} diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index be29d26..36b2843 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -50,6 +50,7 @@ function releaseTlsSkip(): void { } } import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup"; +import { planDownloadCompletion, validateDownloadedFileCompletion } from "./download-completion"; import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkRapidgatorOnline, fetchAllDebridHostInfo, getAvailableDebridLinkApiKeys } from "./debrid"; import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, type ExtractArchiveFailureInfo } from "./extractor"; import { validateFileAgainstManifest } from "./integrity"; @@ -8363,6 +8364,14 @@ export class DownloadManager extends EventEmitter { // Only add existingBytes for 206 responses; for 200 the Content-Length is the full file item.totalBytes = response.status === 206 ? existingBytes + contentLength : contentLength; } + const completionPlan = planDownloadCompletion({ + existingBytes, + responseStatus: response.status, + contentLength, + totalFromRange, + knownTotal, + correctedTotal: correctedRealDebridTotal?.totalBytes || null + }); const writeMode = existingBytes > 0 && response.status === 206 ? "a" : "w"; logAttemptEvent("INFO", "HTTP-Antwort akzeptiert", { @@ -8723,12 +8732,7 @@ export class DownloadManager extends EventEmitter { // Use totalBytes (from unrestrict or Content-Length header) as // primary check, fall back to raw contentLength for providers // that don't report fileSize (e.g. Mega-Debrid Web). - const expectedTotal = (item.totalBytes && item.totalBytes > 0) ? item.totalBytes : 0; - const expectedFromResponse = contentLength > 0 ? contentLength : 0; - if (expectedTotal > 0 && existingBytes + written >= expectedTotal) { - break; - } - if (expectedTotal === 0 && expectedFromResponse > 0 && written >= expectedFromResponse) { + if (completionPlan.canFinishEarly && completionPlan.expectedTotal && written >= completionPlan.expectedTotal) { break; } @@ -8870,6 +8874,20 @@ export class DownloadManager extends EventEmitter { } } + try { + const finalizedStat = await fs.promises.stat(effectiveTargetPath); + if (Number.isFinite(finalizedStat.size) && finalizedStat.size >= 0 && finalizedStat.size !== written) { + logAttemptEvent("WARN", "Dateigroesse nach Stream-Abschluss korrigiert", { + attempt, + previousWritten: written, + statSize: finalizedStat.size + }); + written = finalizedStat.size; + } + } catch { + // ignore stat race; validation below will handle empty/missing files + } + // Detect tiny error-response files (e.g. hoster returning "Forbidden" with HTTP 200). // No legitimate file-hoster download is < 512 bytes. if (written > 0 && written < 512) { @@ -8900,21 +8918,37 @@ export class DownloadManager extends EventEmitter { } } - const exactLengthRequired = isLargeBinaryLikePath(item.fileName || effectiveTargetPath); - if (item.totalBytes && item.totalBytes > 0 && written < item.totalBytes) { - const shortfall = item.totalBytes - written; + const completionValidation = validateDownloadedFileCompletion({ + actualBytes: written, + plan: completionPlan + }); + if (!completionValidation.ok) { + const shortfall = Math.max(0, completionValidation.totalBytes - written); if (preAllocated) { try { await fs.promises.truncate(effectiveTargetPath, written); } catch { /* best-effort */ } } - logger.warn(`Download-Underflow: erwartet=${item.totalBytes}, erhalten=${written}, shortfall=${shortfall} fuer ${item.fileName}`); - if (exactLengthRequired || shortfall > ALLOCATION_UNIT_SIZE) { - item.downloadedBytes = written; - item.progressPercent = Math.max(0, Math.min(99, Math.floor((written / item.totalBytes) * 100))); - item.speedBps = 0; - throw new Error(`download_underflow:${written}/${item.totalBytes}`); - } + logger.warn(`Download-Underflow: erwartet=${completionValidation.totalBytes}, erhalten=${written}, shortfall=${shortfall} fuer ${item.fileName}`); + item.downloadedBytes = written; + item.progressPercent = completionValidation.totalBytes > 0 + ? Math.max(0, Math.min(99, Math.floor((written / completionValidation.totalBytes) * 100))) + : 0; + item.speedBps = 0; + throw new Error(completionValidation.error || `download_underflow:${written}/${completionValidation.totalBytes}`); + } + + if (completionValidation.acceptedMetadataMismatch) { + logger.warn( + `Provider-Groesseninfo verworfen, HTTP-EOF als vollstaendig akzeptiert: ` + + `${item.fileName} erwartet=${completionPlan.expectedTotal}, erhalten=${written}` + ); + logAttemptEvent("WARN", "Provider-Groesseninfo weicht von finaler Dateigroesse ab", { + attempt, + expectedTotal: completionPlan.expectedTotal, + actualBytes: written, + source: completionPlan.source + }); } // Truncate pre-allocated files to actual bytes written to prevent zero-padded tail @@ -8926,6 +8960,7 @@ export class DownloadManager extends EventEmitter { } item.downloadedBytes = written; + item.totalBytes = completionValidation.totalBytes > 0 ? completionValidation.totalBytes : item.totalBytes; item.progressPercent = item.totalBytes ? Math.max(0, Math.min(100, Math.floor((written / item.totalBytes) * 100))) : 100; item.speedBps = 0; item.fullStatus = "Finalisierend..."; diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index 8a3636f..29ad48f 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -7,6 +7,7 @@ import { EventEmitter, once } from "node:events"; import AdmZip from "adm-zip"; import { afterEach, describe, expect, it, vi } from "vitest"; import { DownloadManager, extractArchiveNameFromExtractorLogMessage, getAuthoritativeRealDebridTotal, resolveArchiveItemsFromList } from "../src/main/download-manager"; +import { planDownloadCompletion, validateDownloadedFileCompletion } from "../src/main/download-completion"; import { defaultSettings } from "../src/main/constants"; import { parseDebridLinkApiKeys } from "../src/shared/debrid-link-keys"; import { getProviderUsageDayKey } from "../src/shared/provider-daily-limits"; @@ -45,6 +46,38 @@ describe("resolveArchiveItemsFromList", () => { }); }); +describe("download completion planning", () => { + it("does not allow early finish on provider metadata alone", () => { + expect(planDownloadCompletion({ + existingBytes: 0, + responseStatus: 200, + contentLength: 0, + totalFromRange: null, + knownTotal: 192 * 1024, + correctedTotal: null + })).toEqual({ + expectedTotal: 192 * 1024, + source: "provider-metadata", + canFinishEarly: false + }); + }); + + it("accepts provider metadata mismatches after a clean stream end", () => { + expect(validateDownloadedFileCompletion({ + actualBytes: 256 * 1024, + plan: { + expectedTotal: 192 * 1024, + source: "provider-metadata", + canFinishEarly: false + } + })).toEqual({ + ok: true, + totalBytes: 256 * 1024, + acceptedMetadataMismatch: true + }); + }); +}); + async function waitFor(predicate: () => boolean, timeoutMs = 15000): Promise { const started = Date.now(); while (!predicate()) { @@ -1863,6 +1896,215 @@ describe("download manager", () => { } }); + it("does not stop early on provider-only totals when the HTTP response is longer", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-provider-total-mismatch-")); + tempDirs.push(root); + const actual = Buffer.alloc(256 * 1024, 77); + const advertised = 192 * 1024; + let unrestrictCalls = 0; + + const server = http.createServer((_req, res) => { + res.statusCode = 200; + res.setHeader("Transfer-Encoding", "chunked"); + res.write(actual.subarray(0, 96 * 1024)); + setTimeout(() => { + res.end(actual.subarray(96 * 1024)); + }, 40); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("server address unavailable"); + } + const directUrl = `http://127.0.0.1:${address.port}/provider-mismatch`; + + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes("/unrestrict/link")) { + unrestrictCalls += 1; + return new Response( + JSON.stringify({ + download: directUrl, + filename: "provider-mismatch.rar", + filesize: advertised + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + } + return originalFetch(input, init); + }; + + try { + const manager = new DownloadManager( + { + ...defaultSettings(), + token: "rd-token", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + retryLimit: 1, + autoExtract: false, + autoReconnect: false + }, + emptySession(), + createStoragePaths(path.join(root, "state")) + ); + + manager.addPackages([{ name: "provider-mismatch", links: ["https://dummy/provider-mismatch"] }]); + await manager.start(); + await waitFor(() => !manager.getSnapshot().session.running, 25000); + + const item = Object.values(manager.getSnapshot().session.items)[0]; + expect(item?.status).toBe("completed"); + expect(item?.downloadedBytes).toBe(actual.length); + expect(item?.totalBytes).toBe(actual.length); + expect(unrestrictCalls).toBe(1); + expect(fs.readFileSync(item.targetPath).equals(actual)).toBe(true); + } finally { + server.close(); + await once(server, "close"); + } + }); + + it("does not double-count resumed bytes when deciding early completion", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-resume-early-finish-")); + tempDirs.push(root); + const actual = Buffer.alloc(256 * 1024, 91); + const partialSize = 96 * 1024; + const pkgDir = path.join(root, "downloads", "resume-early-finish"); + fs.mkdirSync(pkgDir, { recursive: true }); + const targetPath = path.join(pkgDir, "resume-early-finish.mkv"); + fs.writeFileSync(targetPath, actual.subarray(0, partialSize)); + let unrestrictCalls = 0; + const starts: number[] = []; + + const server = http.createServer((req, res) => { + const range = String(req.headers.range || ""); + const match = range.match(/bytes=(\d+)-/i); + const start = match ? Number(match[1]) : 0; + starts.push(start); + + if (start <= 0) { + res.statusCode = 500; + res.end("expected resume"); + return; + } + + const remaining = actual.subarray(start); + const firstChunkBytes = 64 * 1024; + res.statusCode = 206; + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Range", `bytes ${start}-${actual.length - 1}/${actual.length}`); + res.setHeader("Content-Length", String(remaining.length)); + res.write(remaining.subarray(0, firstChunkBytes)); + setTimeout(() => { + res.end(remaining.subarray(firstChunkBytes)); + }, 50); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("server address unavailable"); + } + const directUrl = `http://127.0.0.1:${address.port}/resume-early-finish`; + + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes("/unrestrict/link")) { + unrestrictCalls += 1; + return new Response( + JSON.stringify({ + download: directUrl, + filename: "resume-early-finish.mkv", + filesize: actual.length + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + } + return originalFetch(input, init); + }; + + try { + const session = emptySession(); + const packageId = "resume-early-finish-pkg"; + const itemId = "resume-early-finish-item"; + const createdAt = Date.now() - 10_000; + + session.packageOrder = [packageId]; + session.packages[packageId] = { + id: packageId, + name: "resume-early-finish", + outputDir: pkgDir, + extractDir: path.join(root, "extract", "resume-early-finish"), + status: "queued", + itemIds: [itemId], + cancelled: false, + enabled: true, + createdAt, + updatedAt: createdAt + }; + session.items[itemId] = { + id: itemId, + packageId, + url: "https://dummy/resume-early-finish", + provider: null, + status: "queued", + retries: 0, + speedBps: 0, + downloadedBytes: partialSize, + totalBytes: actual.length, + progressPercent: Math.floor((partialSize / actual.length) * 100), + fileName: "resume-early-finish.mkv", + targetPath, + resumable: true, + attempts: 0, + lastError: "", + fullStatus: "Wartet", + createdAt, + updatedAt: createdAt + }; + + const manager = new DownloadManager( + { + ...defaultSettings(), + token: "rd-token", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + retryLimit: 1, + autoExtract: false, + autoReconnect: false + }, + session, + createStoragePaths(path.join(root, "state")) + ); + + await manager.start(); + await waitFor(() => !manager.getSnapshot().session.running, 25000); + + const item = manager.getSnapshot().session.items[itemId]; + expect(item?.status).toBe("completed"); + expect(item?.downloadedBytes).toBe(actual.length); + expect(item?.totalBytes).toBe(actual.length); + expect(unrestrictCalls).toBe(1); + expect(starts).toEqual([partialSize]); + expect(fs.readFileSync(targetPath).equals(actual)).toBe(true); + } finally { + server.close(); + await once(server, "close"); + } + }); + it("assigns unique target paths for same filenames in parallel", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root);