From 9d611bd7491fe5770ca61df3651cf16d74de0513 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Sat, 4 Apr 2026 20:04:15 +0200 Subject: [PATCH] Accept small metadata files (.sfv, .nfo, .nzb) without retry loops SFV checksum verification files are legitimately tiny (~128 bytes) but were rejected by the "suspicious small download" detection, causing infinite "Direktlink erneuern" retry loops that blocked package extraction. - Add KNOWN_SMALL_FILE_RE for .sfv, .nfo, .nzb, .md5, .sha1, .sha256, .crc, .txt, .url, .lnk, .srr file extensions - Skip suspicious-small-download rejection for known small files when they match their expected size (or have no size expectation) - Skip tiny-download error detection for known small metadata files - Add test: verifies .sfv file downloads without retries and completes Co-Authored-By: Claude Opus 4.6 (1M context) --- src/main/download-manager.ts | 814 +++++++++++++++++---------------- tests/download-manager.test.ts | 59 +++ 2 files changed, 474 insertions(+), 399 deletions(-) diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 053a032..1014ce2 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -119,35 +119,38 @@ const ARCHIVE_SETTLE_MIN_DELAY_MS = 1500; const ARCHIVE_SETTLE_POLL_MS = 250; -const ARCHIVE_SETTLE_MAX_WAIT_MS = 5000; - -const MAX_SAME_DIRECT_URL_ATTEMPTS = 3; - -const RESUME_REWIND_BYTES = 256 * 1024; - -const REALDEBRID_TOTAL_MISMATCH_TOLERANCE_BYTES = 64 * 1024; - -const PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES = 1024 * 1024; - -const LARGE_BINARY_FILE_RE = /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|7z(?:\.\d+)?|tar|gz|bz2|xz|iso|mkv|mp4|avi|mov|wmv|m4v|ts|m2ts|webm|mp3|flac|aac|wav)$/i; - -function expectedMinBytes(totalBytes: number | null | undefined, strict: boolean): number { - if (!totalBytes || totalBytes <= 0) { - return 10240; - } - return strict ? totalBytes : Math.max(10240, totalBytes - ALLOCATION_UNIT_SIZE); -} - -function itemExpectedMinBytes(item: DownloadItem): number { - const strict = isLargeBinaryLikePath(item.targetPath || item.fileName || ""); - return expectedMinBytes(item.totalBytes, strict); -} - -function resolvePreallocResumeMismatchThreshold(pathHint: string): number { - return isLargeBinaryLikePath(pathHint) - ? 0 - : PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES; -} +const ARCHIVE_SETTLE_MAX_WAIT_MS = 5000; + +const MAX_SAME_DIRECT_URL_ATTEMPTS = 3; + +const RESUME_REWIND_BYTES = 256 * 1024; + +const REALDEBRID_TOTAL_MISMATCH_TOLERANCE_BYTES = 64 * 1024; + +const PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES = 1024 * 1024; + +const LARGE_BINARY_FILE_RE = /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|7z(?:\.\d+)?|tar|gz|bz2|xz|iso|mkv|mp4|avi|mov|wmv|m4v|ts|m2ts|webm|mp3|flac|aac|wav)$/i; + +/** Files that are legitimately tiny (< 5 KB) and should NOT be rejected as suspicious. */ +const KNOWN_SMALL_FILE_RE = /\.(?:sfv|nfo|nzb|md5|sha1|sha256|crc|txt|url|lnk|srr)$/i; + +function expectedMinBytes(totalBytes: number | null | undefined, strict: boolean): number { + if (!totalBytes || totalBytes <= 0) { + return 10240; + } + return strict ? totalBytes : Math.max(10240, totalBytes - ALLOCATION_UNIT_SIZE); +} + +function itemExpectedMinBytes(item: DownloadItem): number { + const strict = isLargeBinaryLikePath(item.targetPath || item.fileName || ""); + return expectedMinBytes(item.totalBytes, strict); +} + +function resolvePreallocResumeMismatchThreshold(pathHint: string): number { + return isLargeBinaryLikePath(pathHint) + ? 0 + : PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES; +} function resolvePackageItemDiskPath(pkg: PackageEntry, item: DownloadItem): string | null { if (item.targetPath) { @@ -349,35 +352,35 @@ function cloneSettings(settings: AppSettings): AppSettings { }; } -type ParsedContentRange = { - start: number; - end: number; - total: number | null; -}; - -function parseContentRange(contentRange: string | null): ParsedContentRange | null { - if (!contentRange) { - return null; - } - const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i); - if (!match) { - return null; - } - const start = Number(match[1]); - const end = Number(match[2]); - const total = match[3] === "*" ? null : Number(match[3]); - if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start) { - return null; - } - if (total !== null && (!Number.isFinite(total) || total <= 0 || end >= total)) { - return null; - } - return { start, end, total }; -} - -function parseContentRangeTotal(contentRange: string | null): number | null { - return parseContentRange(contentRange)?.total ?? null; -} +type ParsedContentRange = { + start: number; + end: number; + total: number | null; +}; + +function parseContentRange(contentRange: string | null): ParsedContentRange | null { + if (!contentRange) { + return null; + } + const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i); + if (!match) { + return null; + } + const start = Number(match[1]); + const end = Number(match[2]); + const total = match[3] === "*" ? null : Number(match[3]); + if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start) { + return null; + } + if (total !== null && (!Number.isFinite(total) || total <= 0 || end >= total)) { + return null; + } + return { start, end, total }; +} + +function parseContentRangeTotal(contentRange: string | null): number | null { + return parseContentRange(contentRange)?.total ?? null; +} function parseContentDispositionFilename(contentDisposition: string | null): string { if (!contentDisposition) { @@ -437,6 +440,13 @@ function shouldRejectSuspiciousSmallDownload( const size = Math.max(0, Math.floor(Number(fileSizeOnDisk) || 0)); const expected = Number.isFinite(expectedTotal || NaN) ? Math.max(0, Math.floor(expectedTotal || 0)) : 0; const binaryLike = isLargeBinaryLikePath(filePath || fileName); + const name = path.basename(String(filePath || fileName || "")); + + // Known small files (e.g. .sfv, .nfo) are legitimately tiny — never reject them + // as long as they received the expected number of bytes (or we have no expectation). + if (KNOWN_SMALL_FILE_RE.test(name) && (expected <= 0 || size >= expected)) { + return false; + } if (size <= 0) { return expected > 0 || binaryLike; @@ -456,29 +466,29 @@ function shouldRejectSuspiciousSmallDownload( return binaryLike; } -function isFetchFailure(errorText: string): boolean { - const text = String(errorText || "").toLowerCase(); - return text.includes("fetch failed") || text.includes("socket hang up") || text.includes("econnreset") || text.includes("network error"); -} - -function shouldRewindResumeTail(errorText: string): boolean { - const text = String(errorText || "").toLowerCase(); - if (!text) { - return false; - } - return text.includes("terminated") - || text.includes("stall_timeout") - || text.includes("slow_throughput") - || text.includes("write_drain_timeout") - || text.includes("premature close") - || text.includes("unexpected eof") - || text.includes("download_underflow") - || isFetchFailure(text); -} - -function isHttp416Text(errorText: string): boolean { - return /(^|\D)416(\D|$)/.test(String(errorText || "")); -} +function isFetchFailure(errorText: string): boolean { + const text = String(errorText || "").toLowerCase(); + return text.includes("fetch failed") || text.includes("socket hang up") || text.includes("econnreset") || text.includes("network error"); +} + +function shouldRewindResumeTail(errorText: string): boolean { + const text = String(errorText || "").toLowerCase(); + if (!text) { + return false; + } + return text.includes("terminated") + || text.includes("stall_timeout") + || text.includes("slow_throughput") + || text.includes("write_drain_timeout") + || text.includes("premature close") + || text.includes("unexpected eof") + || text.includes("download_underflow") + || isFetchFailure(text); +} + +function isHttp416Text(errorText: string): boolean { + return /(^|\D)416(\D|$)/.test(String(errorText || "")); +} function shouldPreflightFinalizeItemFromDisk(item: DownloadItem): boolean { const text = `${item.fullStatus || ""} ${item.lastError || ""}`.toLowerCase(); @@ -5258,35 +5268,35 @@ export class DownloadManager extends EventEmitter { * knows which files belong to which items. Without this, after restart all paths are * unclaimed and a new download with the same filename would create a "(1)" copy * instead of reusing its own partial file — or worse, overwrite another item's file. */ - private restoreTargetPathReservations(): void { - let restored = 0; - let droppedUnsafe = 0; - for (const item of Object.values(this.session.items)) { - const pkg = this.session.packages[item.packageId]; - if (!pkg) { - continue; - } - const tp = String(item.targetPath || "").trim(); - if (!tp) continue; - if (!isPathInsideDir(tp, pkg.outputDir)) { - droppedUnsafe += 1; - item.targetPath = ""; - continue; - } - const key = pathKey(tp); - if (!this.reservedTargetPaths.has(key)) { - this.reservedTargetPaths.set(key, item.id); - this.claimedTargetPathByItem.set(item.id, tp); - restored += 1; + private restoreTargetPathReservations(): void { + let restored = 0; + let droppedUnsafe = 0; + for (const item of Object.values(this.session.items)) { + const pkg = this.session.packages[item.packageId]; + if (!pkg) { + continue; + } + const tp = String(item.targetPath || "").trim(); + if (!tp) continue; + if (!isPathInsideDir(tp, pkg.outputDir)) { + droppedUnsafe += 1; + item.targetPath = ""; + continue; + } + const key = pathKey(tp); + if (!this.reservedTargetPaths.has(key)) { + this.reservedTargetPaths.set(key, item.id); + this.claimedTargetPathByItem.set(item.id, tp); + restored += 1; } } - if (restored > 0) { - logger.info(`restoreTargetPathReservations: ${restored} Pfade aus Session wiederhergestellt`); - } - if (droppedUnsafe > 0) { - logger.warn(`restoreTargetPathReservations: ${droppedUnsafe} unsichere targetPath-Eintraege verworfen`); - } - this.reconcileDuplicateSuffixSessionItems(); + if (restored > 0) { + logger.info(`restoreTargetPathReservations: ${restored} Pfade aus Session wiederhergestellt`); + } + if (droppedUnsafe > 0) { + logger.warn(`restoreTargetPathReservations: ${droppedUnsafe} unsichere targetPath-Eintraege verworfen`); + } + this.reconcileDuplicateSuffixSessionItems(); // Fix legacy (N) suffix files: rename back to original if original path is free this.fixDuplicateSuffixFiles(); } @@ -5454,7 +5464,7 @@ export class DownloadManager extends EventEmitter { if (!targetPath || !item.totalBytes || item.totalBytes <= 0) continue; try { const stat = fs.statSync(targetPath); - const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || targetPath)); + const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || targetPath)); const persistedShortfall = item.downloadedBytes < expectedMinSize && stat.size >= expectedMinSize; if (stat.size < expectedMinSize) { logger.warn(`revalidateCompleted: ${item.fileName} ist nur ${humanSize(stat.size)} statt ${humanSize(item.totalBytes)}, setze auf queued`); @@ -5534,15 +5544,15 @@ export class DownloadManager extends EventEmitter { || normalizedError.includes("resume_download_underflow"); const archiveLikeTarget = String(item.fileName || diskState.diskPath || "").toLowerCase(); const archiveLike = /(?:\.part\d+\.rar|\.rar|\.r\d{2,3}|\.zip(?:\.\d+)?|\.7z(?:\.\d+)?|\.(?:tar(?:\.(?:gz|bz2|xz))?|tgz|tbz2|txz)|\.\d{3})$/i.test(archiveLikeTarget); - const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || diskState.diskPath || "")); - const looksComplete = diskState.exists - && diskState.fullOnDisk - && ( - diskState.reason === "ok" - || item.progressPercent >= 100 - || item.downloadedBytes >= diskState.minBytes - || (item.totalBytes != null && item.totalBytes > 0 && diskState.size >= expectedMinSize) - ); + const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || diskState.diskPath || "")); + const looksComplete = diskState.exists + && diskState.fullOnDisk + && ( + diskState.reason === "ok" + || item.progressPercent >= 100 + || item.downloadedBytes >= diskState.minBytes + || (item.totalBytes != null && item.totalBytes > 0 && diskState.size >= expectedMinSize) + ); if (!looksComplete || (knownShortfall > 0 && (underflowIndicated || archiveLike))) { return false; } @@ -8448,80 +8458,80 @@ export class DownloadManager extends EventEmitter { const retryDisplayLimit = retryLimitLabel(configuredRetryLimit); const maxAttemptsBySetting = configuredRetryLimit <= 0 ? Number.MAX_SAFE_INTEGER : configuredRetryLimit + 1; const maxAttempts = Math.max(1, Math.min(MAX_SAME_DIRECT_URL_ATTEMPTS, maxAttemptsBySetting)); - - let lastError = ""; - let effectiveTargetPath = targetPath; - let resumeRewindBytesNextAttempt = 0; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - let existingBytes = 0; - try { - const stat = await fs.promises.stat(effectiveTargetPath); - existingBytes = stat.size; - } catch { - // file does not exist - } - if (existingBytes > 0 && resumeRewindBytesNextAttempt > 0) { - const previousBytes = existingBytes; - const rewindBytes = Math.min(existingBytes, resumeRewindBytesNextAttempt); - const resumeStart = Math.max(0, existingBytes - rewindBytes); - try { - await fs.promises.truncate(effectiveTargetPath, resumeStart); - existingBytes = resumeStart; - item.downloadedBytes = Math.min(item.downloadedBytes, existingBytes); - logAttemptEvent("WARN", "Resume-Schutz aktiv: Teil-Datei vor Retry zurueckgespult", { - attempt, - previousBytes, - rewindBytes, - resumeStart - }); - } catch (rewindError) { - logAttemptEvent("WARN", "Resume-Schutz: Rueckspulen der Teil-Datei fehlgeschlagen", { - attempt, - previousBytes, - rewindBytes, - error: compactErrorText(rewindError) - }); - } finally { - resumeRewindBytesNextAttempt = 0; - } - } else if (resumeRewindBytesNextAttempt > 0) { - resumeRewindBytesNextAttempt = 0; - } - const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0)); - const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || effectiveTargetPath || ""); - // Guard against pre-allocated sparse files from a crashed session: - // if file size exceeds persisted downloadedBytes beyond the allowed - // mismatch threshold, the file was likely pre-allocated but only - // partially written before a hard crash. - // This must also run for persistedBytes=0, otherwise startup-resume can - // send Range=full-size and incorrectly accept HTTP 416 as "complete". - if (existingBytes > 0 && existingBytes > persistedBytes + preallocMismatchThreshold) { - try { - const previousBytes = existingBytes; - await fs.promises.truncate(effectiveTargetPath, persistedBytes); - existingBytes = persistedBytes; - logAttemptEvent("WARN", "Pre-alloc-Rest erkannt, Teil-Datei auf persistierte Bytes gekuerzt", { - attempt, - previousBytes, - persistedBytes - }); - } catch { - if (persistedBytes === 0) { - try { - await fs.promises.rm(effectiveTargetPath, { force: true }); - existingBytes = 0; - } catch { - // ignore - } - } - } - } - const suspiciousResumeFootprint = existingBytes > 0 - && existingBytes > persistedBytes + preallocMismatchThreshold; - const headers: Record = {}; - if (existingBytes > 0) { - headers.Range = `bytes=${existingBytes}-`; - } + + let lastError = ""; + let effectiveTargetPath = targetPath; + let resumeRewindBytesNextAttempt = 0; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + let existingBytes = 0; + try { + const stat = await fs.promises.stat(effectiveTargetPath); + existingBytes = stat.size; + } catch { + // file does not exist + } + if (existingBytes > 0 && resumeRewindBytesNextAttempt > 0) { + const previousBytes = existingBytes; + const rewindBytes = Math.min(existingBytes, resumeRewindBytesNextAttempt); + const resumeStart = Math.max(0, existingBytes - rewindBytes); + try { + await fs.promises.truncate(effectiveTargetPath, resumeStart); + existingBytes = resumeStart; + item.downloadedBytes = Math.min(item.downloadedBytes, existingBytes); + logAttemptEvent("WARN", "Resume-Schutz aktiv: Teil-Datei vor Retry zurueckgespult", { + attempt, + previousBytes, + rewindBytes, + resumeStart + }); + } catch (rewindError) { + logAttemptEvent("WARN", "Resume-Schutz: Rueckspulen der Teil-Datei fehlgeschlagen", { + attempt, + previousBytes, + rewindBytes, + error: compactErrorText(rewindError) + }); + } finally { + resumeRewindBytesNextAttempt = 0; + } + } else if (resumeRewindBytesNextAttempt > 0) { + resumeRewindBytesNextAttempt = 0; + } + const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0)); + const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || effectiveTargetPath || ""); + // Guard against pre-allocated sparse files from a crashed session: + // if file size exceeds persisted downloadedBytes beyond the allowed + // mismatch threshold, the file was likely pre-allocated but only + // partially written before a hard crash. + // This must also run for persistedBytes=0, otherwise startup-resume can + // send Range=full-size and incorrectly accept HTTP 416 as "complete". + if (existingBytes > 0 && existingBytes > persistedBytes + preallocMismatchThreshold) { + try { + const previousBytes = existingBytes; + await fs.promises.truncate(effectiveTargetPath, persistedBytes); + existingBytes = persistedBytes; + logAttemptEvent("WARN", "Pre-alloc-Rest erkannt, Teil-Datei auf persistierte Bytes gekuerzt", { + attempt, + previousBytes, + persistedBytes + }); + } catch { + if (persistedBytes === 0) { + try { + await fs.promises.rm(effectiveTargetPath, { force: true }); + existingBytes = 0; + } catch { + // ignore + } + } + } + } + const suspiciousResumeFootprint = existingBytes > 0 + && existingBytes > persistedBytes + preallocMismatchThreshold; + const headers: Record = {}; + if (existingBytes > 0) { + headers.Range = `bytes=${existingBytes}-`; + } logAttemptEvent("INFO", "HTTP-Download-Versuch vorbereitet", { attempt, maxAttempts: maxAttempts === Number.MAX_SAFE_INTEGER ? "infinite" : maxAttempts, @@ -8589,37 +8599,37 @@ export class DownloadManager extends EventEmitter { if (response.status === 416 && existingBytes > 0) { await response.arrayBuffer().catch(() => undefined); const rangeTotal = parseContentRangeTotal(response.headers.get("content-range")); - const expectedTotal = rangeTotal && rangeTotal > 0 - ? rangeTotal - : (knownTotal && knownTotal > 0 ? knownTotal : null); - const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE; - const closeEnoughToExpected = expectedTotal != null - && Math.abs(existingBytes - expectedTotal) <= sizeToleranceBytes; - if (expectedTotal != null && closeEnoughToExpected && !suspiciousResumeFootprint) { - const finalizedTotal = Math.max(existingBytes, expectedTotal); - item.totalBytes = finalizedTotal; - item.downloadedBytes = existingBytes; - item.progressPercent = 100; - item.speedBps = 0; + const expectedTotal = rangeTotal && rangeTotal > 0 + ? rangeTotal + : (knownTotal && knownTotal > 0 ? knownTotal : null); + const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE; + const closeEnoughToExpected = expectedTotal != null + && Math.abs(existingBytes - expectedTotal) <= sizeToleranceBytes; + if (expectedTotal != null && closeEnoughToExpected && !suspiciousResumeFootprint) { + const finalizedTotal = Math.max(existingBytes, expectedTotal); + item.totalBytes = finalizedTotal; + item.downloadedBytes = existingBytes; + item.progressPercent = 100; + item.speedBps = 0; item.updatedAt = nowMs(); logAttemptEvent("INFO", "HTTP 416 als vollständig behandelt", { existingBytes, - expectedTotal: finalizedTotal - }); - return { resumable: true }; - } - if (expectedTotal != null && closeEnoughToExpected && suspiciousResumeFootprint) { - logAttemptEvent("WARN", "HTTP 416 trotz Vollgroesse nicht als fertig gewertet (vermutlich pre-alloc)", { - attempt, - existingBytes, - persistedBytes, - expectedTotal - }); - } - - try { - await fs.promises.rm(effectiveTargetPath, { force: true }); - } catch { + expectedTotal: finalizedTotal + }); + return { resumable: true }; + } + if (expectedTotal != null && closeEnoughToExpected && suspiciousResumeFootprint) { + logAttemptEvent("WARN", "HTTP 416 trotz Vollgroesse nicht als fertig gewertet (vermutlich pre-alloc)", { + attempt, + existingBytes, + persistedBytes, + expectedTotal + }); + } + + try { + await fs.promises.rm(effectiveTargetPath, { force: true }); + } catch { // ignore } this.dropItemContribution(active.itemId); @@ -8698,8 +8708,8 @@ export class DownloadManager extends EventEmitter { const rawContentLength = Number(response.headers.get("content-length") || 0); const contentLength = Number.isFinite(rawContentLength) && rawContentLength > 0 ? rawContentLength : 0; - const parsedContentRange = parseContentRange(response.headers.get("content-range")); - const totalFromRange = parsedContentRange?.total ?? null; + const parsedContentRange = parseContentRange(response.headers.get("content-range")); + const totalFromRange = parsedContentRange?.total ?? null; const serverIgnoredRange = existingBytes > 0 && response.status === 200; const allowFreshOverwriteAfterResumeReset = serverIgnoredRange && active.resumeHardResetUsed @@ -8719,69 +8729,69 @@ export class DownloadManager extends EventEmitter { } throw new Error(`range_ignored_on_resume:${existingBytes}/${contentLength || 0}`); } - if (allowFreshOverwriteAfterResumeReset) { - logger.warn( - `Server ignorierte Range-Header nach Resume-Reset, ueberschreibe alte Teil-Datei frisch: ${item.fileName}` - ); + if (allowFreshOverwriteAfterResumeReset) { + logger.warn( + `Server ignorierte Range-Header nach Resume-Reset, ueberschreibe alte Teil-Datei frisch: ${item.fileName}` + ); logAttemptEvent("WARN", "Range ignoriert nach Resume-Reset, frischer Vollstart erlaubt", { attempt, existingBytes, contentLength, - directUrl - }); - } - if (existingBytes > 0 && response.status === 206) { - if (!parsedContentRange) { - logAttemptEvent("WARN", "Resume-Range-Header ungueltig oder fehlt", { - attempt, - existingBytes, - contentRange: response.headers.get("content-range") || "" - }); - try { - await response.body?.cancel(); - } catch { - // ignore - } - throw new Error(`range_mismatch_on_resume:${existingBytes}/invalid`); - } - if (parsedContentRange.start !== existingBytes) { - const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE; - const canTreatAsAlreadyComplete = contentLength === 0 - && parsedContentRange.start === 0 - && parsedContentRange.total != null - && Math.abs(existingBytes - parsedContentRange.total) <= sizeToleranceBytes; - if (canTreatAsAlreadyComplete) { - item.totalBytes = parsedContentRange.total; - item.downloadedBytes = existingBytes; - item.progressPercent = 100; - item.speedBps = 0; - item.updatedAt = nowMs(); - logAttemptEvent("WARN", "Resume-Range-Start abweichend, Datei aber bereits vollstaendig", { - attempt, - existingBytes, - totalFromRange: parsedContentRange.total, - contentLength - }); - return { resumable: true }; - } - logAttemptEvent("WARN", "Resume-Range-Start stimmt nicht mit lokaler Dateigroesse ueberein", { - attempt, - expectedStart: existingBytes, - actualStart: parsedContentRange.start, - actualEnd: parsedContentRange.end, - totalFromRange, - directUrl - }); - try { - await response.body?.cancel(); - } catch { - // ignore - } - throw new Error(`range_mismatch_on_resume:${existingBytes}/${parsedContentRange.start}`); - } - } - - const correctedRealDebridTotal = getAuthoritativeRealDebridTotal( + directUrl + }); + } + if (existingBytes > 0 && response.status === 206) { + if (!parsedContentRange) { + logAttemptEvent("WARN", "Resume-Range-Header ungueltig oder fehlt", { + attempt, + existingBytes, + contentRange: response.headers.get("content-range") || "" + }); + try { + await response.body?.cancel(); + } catch { + // ignore + } + throw new Error(`range_mismatch_on_resume:${existingBytes}/invalid`); + } + if (parsedContentRange.start !== existingBytes) { + const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE; + const canTreatAsAlreadyComplete = contentLength === 0 + && parsedContentRange.start === 0 + && parsedContentRange.total != null + && Math.abs(existingBytes - parsedContentRange.total) <= sizeToleranceBytes; + if (canTreatAsAlreadyComplete) { + item.totalBytes = parsedContentRange.total; + item.downloadedBytes = existingBytes; + item.progressPercent = 100; + item.speedBps = 0; + item.updatedAt = nowMs(); + logAttemptEvent("WARN", "Resume-Range-Start abweichend, Datei aber bereits vollstaendig", { + attempt, + existingBytes, + totalFromRange: parsedContentRange.total, + contentLength + }); + return { resumable: true }; + } + logAttemptEvent("WARN", "Resume-Range-Start stimmt nicht mit lokaler Dateigroesse ueberein", { + attempt, + expectedStart: existingBytes, + actualStart: parsedContentRange.start, + actualEnd: parsedContentRange.end, + totalFromRange, + directUrl + }); + try { + await response.body?.cancel(); + } catch { + // ignore + } + throw new Error(`range_mismatch_on_resume:${existingBytes}/${parsedContentRange.start}`); + } + } + + const correctedRealDebridTotal = getAuthoritativeRealDebridTotal( item.provider, knownTotal || 0, existingBytes, @@ -9340,40 +9350,46 @@ export class DownloadManager extends EventEmitter { } // Detect tiny error-response files (e.g. hoster returning "Forbidden" with HTTP 200). - // No legitimate file-hoster download is < 512 bytes. + // No legitimate file-hoster download is < 512 bytes, EXCEPT known small metadata + // files like .sfv (checksum verification), .nfo (release info), etc. if (written > 0 && written < 512) { - let snippet = ""; - try { - snippet = await fs.promises.readFile(effectiveTargetPath, "utf8"); - snippet = snippet.slice(0, 200).replace(/[\r\n]+/g, " ").trim(); - } catch { /* ignore */ } - const exactTinyBinary = Boolean( - item.totalBytes - && item.totalBytes > 0 - && written >= item.totalBytes - && isLargeBinaryLikePath(item.fileName || effectiveTargetPath) - ); - const snippetSuggestsError = /<(?:!doctype|html|body)\b|\b(?:forbidden|access denied|error|not found|expired|unavailable)\b/i.test(snippet); - if (exactTinyBinary && !snippetSuggestsError) { - logger.info(`Tiny Binary akzeptiert (${written} B): ${item.fileName || effectiveTargetPath}`); + const knownSmallFile = KNOWN_SMALL_FILE_RE.test(item.fileName || effectiveTargetPath); + if (knownSmallFile && ((!item.totalBytes || item.totalBytes <= 0) || written >= item.totalBytes)) { + logger.info(`Kleine Metadaten-Datei akzeptiert (${written} B): ${item.fileName || effectiveTargetPath}`); } else { - logger.warn(`Tiny download erkannt (${written} B): "${snippet}"`); - try { - await fs.promises.rm(effectiveTargetPath, { force: true }); - } catch { /* ignore */ } - this.releaseTargetPath(active.itemId); - this.dropItemContribution(active.itemId); - item.downloadedBytes = 0; - item.progressPercent = 0; - throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`); + let snippet = ""; + try { + snippet = await fs.promises.readFile(effectiveTargetPath, "utf8"); + snippet = snippet.slice(0, 200).replace(/[\r\n]+/g, " ").trim(); + } catch { /* ignore */ } + const exactTinyBinary = Boolean( + item.totalBytes + && item.totalBytes > 0 + && written >= item.totalBytes + && isLargeBinaryLikePath(item.fileName || effectiveTargetPath) + ); + const snippetSuggestsError = /<(?:!doctype|html|body)\b|\b(?:forbidden|access denied|error|not found|expired|unavailable)\b/i.test(snippet); + if (exactTinyBinary && !snippetSuggestsError) { + logger.info(`Tiny Binary akzeptiert (${written} B): ${item.fileName || effectiveTargetPath}`); + } else { + logger.warn(`Tiny download erkannt (${written} B): "${snippet}"`); + try { + await fs.promises.rm(effectiveTargetPath, { force: true }); + } catch { /* ignore */ } + this.releaseTargetPath(active.itemId); + this.dropItemContribution(active.itemId); + item.downloadedBytes = 0; + item.progressPercent = 0; + throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`); + } } } - const completionValidation = validateDownloadedFileCompletion({ - actualBytes: written, - plan: completionPlan, - toleranceBytes: isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE - }); + const completionValidation = validateDownloadedFileCompletion({ + actualBytes: written, + plan: completionPlan, + toleranceBytes: isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE + }); if (!completionValidation.ok) { const shortfall = Math.max(0, completionValidation.totalBytes - written); if (preAllocated) { @@ -9445,27 +9461,27 @@ export class DownloadManager extends EventEmitter { error: lastError, targetPath: effectiveTargetPath }); - if ( - normalizedLastError.startsWith("range_ignored_on_resume:") - || normalizedLastError.startsWith("range_mismatch_on_resume:") - ) { - throw new Error(`direct_link_retry_exhausted:${normalizedLastError}`); - } - if (attempt < maxAttempts && written > existingBytes && shouldRewindResumeTail(normalizedLastError)) { - resumeRewindBytesNextAttempt = Math.max(resumeRewindBytesNextAttempt, RESUME_REWIND_BYTES); - logAttemptEvent("WARN", "Resume-Schutz vorgemerkt: naechster Retry startet mit Rewind", { - attempt, - existingBytes, - written, - appendedBytes: Math.max(0, written - existingBytes), - rewindBytes: resumeRewindBytesNextAttempt, - error: normalizedLastError - }); - } - if (attempt < maxAttempts) { - item.retries += 1; - item.fullStatus = `Downloadfehler, retry ${attempt}/${maxAttempts} (Direktlink)`; - this.emitState(); + if ( + normalizedLastError.startsWith("range_ignored_on_resume:") + || normalizedLastError.startsWith("range_mismatch_on_resume:") + ) { + throw new Error(`direct_link_retry_exhausted:${normalizedLastError}`); + } + if (attempt < maxAttempts && written > existingBytes && shouldRewindResumeTail(normalizedLastError)) { + resumeRewindBytesNextAttempt = Math.max(resumeRewindBytesNextAttempt, RESUME_REWIND_BYTES); + logAttemptEvent("WARN", "Resume-Schutz vorgemerkt: naechster Retry startet mit Rewind", { + attempt, + existingBytes, + written, + appendedBytes: Math.max(0, written - existingBytes), + rewindBytes: resumeRewindBytesNextAttempt, + error: normalizedLastError + }); + } + if (attempt < maxAttempts) { + item.retries += 1; + item.fullStatus = `Downloadfehler, retry ${attempt}/${maxAttempts} (Direktlink)`; + this.emitState(); await sleep(retryDelayWithJitter(attempt, 250)); continue; } @@ -9943,8 +9959,8 @@ export class DownloadManager extends EventEmitter { try { const stat = await fs.promises.stat(part); // Find the item that owns this file to get its expected totalBytes - const ownerItem = this.findItemByDiskPath(pkg, part); - const minBytes = expectedMinBytes(ownerItem?.totalBytes ?? 0, isLargeBinaryLikePath(part)); + const ownerItem = this.findItemByDiskPath(pkg, part); + const minBytes = expectedMinBytes(ownerItem?.totalBytes ?? 0, isLargeBinaryLikePath(part)); if (stat.size < minBytes) { allMissingFullOnDisk = false; break; @@ -10478,72 +10494,72 @@ export class DownloadManager extends EventEmitter { if (item.status === "downloading" || item.status === "validating" || item.status === "integrity_check") { continue; } - if (!item.targetPath) { - continue; - } - if (!isPathInsideDir(item.targetPath, pkg.outputDir)) { - logger.warn(`Item-Recovery: Unsicherer targetPath verworfen (${item.fileName} -> ${item.targetPath})`); - this.releaseTargetPath(item.id); - this.dropItemContribution(item.id); - item.targetPath = ""; - item.status = "queued"; - item.attempts = 0; - item.downloadedBytes = 0; - item.progressPercent = 0; - item.speedBps = 0; - item.fullStatus = "Wartet (ungueltiger Zielpfad)"; - item.updatedAt = nowMs(); - continue; - } - try { - const stat = await fs.promises.stat(item.targetPath); - // Require file to be essentially complete — within one allocation unit of the - // expected size. The old 50% threshold incorrectly recovered partial downloads - // (e.g. 627 MB of 1001 MB) and triggered hybrid extraction on incomplete archives. - const minSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || item.targetPath)); - const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0)); - const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || item.targetPath || ""); - const suspiciousPreallocFootprint = item.totalBytes != null - && item.totalBytes > 0 - && stat.size >= minSize - && stat.size > persistedBytes + preallocMismatchThreshold; - if (stat.size >= minSize) { - // Re-check: another task may have started this item during the await - const latestItem = this.session.items[item.id]; - if (!latestItem || this.activeTasks.has(item.id) || latestItem.status === "downloading" - || latestItem.status === "validating" || latestItem.status === "integrity_check") { - continue; - } - if (suspiciousPreallocFootprint) { - logger.warn( - `Item-Recovery: ${item.fileName} uebersprungen – pre-alloc-Verdacht ` + - `(stat=${humanSize(stat.size)}, bytes=${humanSize(persistedBytes)}, total=${humanSize(item.totalBytes)})` - ); - try { - if (persistedBytes > 0) { - fs.truncateSync(item.targetPath, persistedBytes); - } else { - fs.rmSync(item.targetPath, { force: true }); - } - } catch { - // best-effort - } - item.status = "queued"; - item.attempts = 0; - item.downloadedBytes = persistedBytes; - item.progressPercent = item.totalBytes > 0 - ? Math.max(0, Math.min(99, Math.floor((persistedBytes / item.totalBytes) * 100))) - : 0; - item.speedBps = 0; - item.fullStatus = "Wartet (Auto-Recovery: pre-alloc)"; - item.updatedAt = nowMs(); - continue; - } - // Guard against pre-allocated sparse files from a hard crash: file has - // the full expected size but downloadedBytes is significantly behind. - if (item.downloadedBytes > 0 && item.totalBytes && item.totalBytes > 0 - && stat.size >= minSize - && item.downloadedBytes < item.totalBytes * 0.95) { + if (!item.targetPath) { + continue; + } + if (!isPathInsideDir(item.targetPath, pkg.outputDir)) { + logger.warn(`Item-Recovery: Unsicherer targetPath verworfen (${item.fileName} -> ${item.targetPath})`); + this.releaseTargetPath(item.id); + this.dropItemContribution(item.id); + item.targetPath = ""; + item.status = "queued"; + item.attempts = 0; + item.downloadedBytes = 0; + item.progressPercent = 0; + item.speedBps = 0; + item.fullStatus = "Wartet (ungueltiger Zielpfad)"; + item.updatedAt = nowMs(); + continue; + } + try { + const stat = await fs.promises.stat(item.targetPath); + // Require file to be essentially complete — within one allocation unit of the + // expected size. The old 50% threshold incorrectly recovered partial downloads + // (e.g. 627 MB of 1001 MB) and triggered hybrid extraction on incomplete archives. + const minSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || item.targetPath)); + const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0)); + const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || item.targetPath || ""); + const suspiciousPreallocFootprint = item.totalBytes != null + && item.totalBytes > 0 + && stat.size >= minSize + && stat.size > persistedBytes + preallocMismatchThreshold; + if (stat.size >= minSize) { + // Re-check: another task may have started this item during the await + const latestItem = this.session.items[item.id]; + if (!latestItem || this.activeTasks.has(item.id) || latestItem.status === "downloading" + || latestItem.status === "validating" || latestItem.status === "integrity_check") { + continue; + } + if (suspiciousPreallocFootprint) { + logger.warn( + `Item-Recovery: ${item.fileName} uebersprungen – pre-alloc-Verdacht ` + + `(stat=${humanSize(stat.size)}, bytes=${humanSize(persistedBytes)}, total=${humanSize(item.totalBytes)})` + ); + try { + if (persistedBytes > 0) { + fs.truncateSync(item.targetPath, persistedBytes); + } else { + fs.rmSync(item.targetPath, { force: true }); + } + } catch { + // best-effort + } + item.status = "queued"; + item.attempts = 0; + item.downloadedBytes = persistedBytes; + item.progressPercent = item.totalBytes > 0 + ? Math.max(0, Math.min(99, Math.floor((persistedBytes / item.totalBytes) * 100))) + : 0; + item.speedBps = 0; + item.fullStatus = "Wartet (Auto-Recovery: pre-alloc)"; + item.updatedAt = nowMs(); + continue; + } + // Guard against pre-allocated sparse files from a hard crash: file has + // the full expected size but downloadedBytes is significantly behind. + if (item.downloadedBytes > 0 && item.totalBytes && item.totalBytes > 0 + && stat.size >= minSize + && item.downloadedBytes < item.totalBytes * 0.95) { logger.warn(`Item-Recovery: ${item.fileName} uebersprungen – vermutlich pre-alloc (stat=${humanSize(stat.size)}, bytes=${humanSize(item.downloadedBytes)}, total=${humanSize(item.totalBytes)})`); continue; } diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index 7775417..19f5724 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -6140,6 +6140,65 @@ describe("download manager", () => { } }); + it("accepts small .sfv metadata files without rejecting them as suspicious", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + // SFV content is just CRC32 checksums — legitimately tiny + const sfvContent = Buffer.from("archive.part1.rar 1A2B3C4D\narchive.part2.rar 5E6F7A8B\n", "utf8"); + + const server = http.createServer((req, res) => { + res.statusCode = 200; + res.setHeader("Content-Length", String(sfvContent.length)); + res.end(sfvContent); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("server address unavailable"); + const directUrl = `http://127.0.0.1:${address.port}/checksum.sfv`; + + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes("/unrestrict/link")) { + return new Response( + JSON.stringify({ download: directUrl, filename: "archive.sfv", filesize: sfvContent.length }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + } + return originalFetch(input, init); + }; + + try { + const manager = new DownloadManager( + { + ...defaultSettings(), + token: "rd-token", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: false, + autoReconnect: false + }, + emptySession(), + createStoragePaths(path.join(root, "state")) + ); + + manager.addPackages([{ name: "sfv-test", links: ["https://dummy/sfv-file"] }]); + await manager.start(); + await waitFor(() => !manager.getSnapshot().session.running, 15000); + + const item = Object.values(manager.getSnapshot().session.items)[0]; + expect(item?.status).toBe("completed"); + expect(item?.retries).toBe(0); + expect(fs.existsSync(item.targetPath)).toBe(true); + const onDisk = fs.readFileSync(item.targetPath); + expect(onDisk.length).toBe(sfvContent.length); + } finally { + server.close(); + await once(server, "close"); + } + }); + it("limits AllDebrid rapidgator starts to one active task by default", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root);