Accept small metadata files (.sfv, .nfo, .nzb) without retry loops
SFV checksum verification files are legitimately tiny (~128 bytes) but were rejected by the "suspicious small download" detection, causing infinite "Direktlink erneuern" retry loops that blocked package extraction. - Add KNOWN_SMALL_FILE_RE for .sfv, .nfo, .nzb, .md5, .sha1, .sha256, .crc, .txt, .url, .lnk, .srr file extensions - Skip suspicious-small-download rejection for known small files when they match their expected size (or have no size expectation) - Skip tiny-download error detection for known small metadata files - Add test: verifies .sfv file downloads without retries and completes Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sucukdeluxe
parent
8ab01f3da4
commit
9d611bd749
@ -119,35 +119,38 @@ const ARCHIVE_SETTLE_MIN_DELAY_MS = 1500;
|
|||||||
|
|
||||||
const ARCHIVE_SETTLE_POLL_MS = 250;
|
const ARCHIVE_SETTLE_POLL_MS = 250;
|
||||||
|
|
||||||
const ARCHIVE_SETTLE_MAX_WAIT_MS = 5000;
|
const ARCHIVE_SETTLE_MAX_WAIT_MS = 5000;
|
||||||
|
|
||||||
const MAX_SAME_DIRECT_URL_ATTEMPTS = 3;
|
const MAX_SAME_DIRECT_URL_ATTEMPTS = 3;
|
||||||
|
|
||||||
const RESUME_REWIND_BYTES = 256 * 1024;
|
const RESUME_REWIND_BYTES = 256 * 1024;
|
||||||
|
|
||||||
const REALDEBRID_TOTAL_MISMATCH_TOLERANCE_BYTES = 64 * 1024;
|
const REALDEBRID_TOTAL_MISMATCH_TOLERANCE_BYTES = 64 * 1024;
|
||||||
|
|
||||||
const PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES = 1024 * 1024;
|
const PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES = 1024 * 1024;
|
||||||
|
|
||||||
const LARGE_BINARY_FILE_RE = /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|7z(?:\.\d+)?|tar|gz|bz2|xz|iso|mkv|mp4|avi|mov|wmv|m4v|ts|m2ts|webm|mp3|flac|aac|wav)$/i;
|
const LARGE_BINARY_FILE_RE = /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|7z(?:\.\d+)?|tar|gz|bz2|xz|iso|mkv|mp4|avi|mov|wmv|m4v|ts|m2ts|webm|mp3|flac|aac|wav)$/i;
|
||||||
|
|
||||||
function expectedMinBytes(totalBytes: number | null | undefined, strict: boolean): number {
|
/** Files that are legitimately tiny (< 5 KB) and should NOT be rejected as suspicious. */
|
||||||
if (!totalBytes || totalBytes <= 0) {
|
const KNOWN_SMALL_FILE_RE = /\.(?:sfv|nfo|nzb|md5|sha1|sha256|crc|txt|url|lnk|srr)$/i;
|
||||||
return 10240;
|
|
||||||
}
|
function expectedMinBytes(totalBytes: number | null | undefined, strict: boolean): number {
|
||||||
return strict ? totalBytes : Math.max(10240, totalBytes - ALLOCATION_UNIT_SIZE);
|
if (!totalBytes || totalBytes <= 0) {
|
||||||
}
|
return 10240;
|
||||||
|
}
|
||||||
function itemExpectedMinBytes(item: DownloadItem): number {
|
return strict ? totalBytes : Math.max(10240, totalBytes - ALLOCATION_UNIT_SIZE);
|
||||||
const strict = isLargeBinaryLikePath(item.targetPath || item.fileName || "");
|
}
|
||||||
return expectedMinBytes(item.totalBytes, strict);
|
|
||||||
}
|
function itemExpectedMinBytes(item: DownloadItem): number {
|
||||||
|
const strict = isLargeBinaryLikePath(item.targetPath || item.fileName || "");
|
||||||
function resolvePreallocResumeMismatchThreshold(pathHint: string): number {
|
return expectedMinBytes(item.totalBytes, strict);
|
||||||
return isLargeBinaryLikePath(pathHint)
|
}
|
||||||
? 0
|
|
||||||
: PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES;
|
function resolvePreallocResumeMismatchThreshold(pathHint: string): number {
|
||||||
}
|
return isLargeBinaryLikePath(pathHint)
|
||||||
|
? 0
|
||||||
|
: PREALLOC_RESUME_MISMATCH_THRESHOLD_BYTES;
|
||||||
|
}
|
||||||
|
|
||||||
function resolvePackageItemDiskPath(pkg: PackageEntry, item: DownloadItem): string | null {
|
function resolvePackageItemDiskPath(pkg: PackageEntry, item: DownloadItem): string | null {
|
||||||
if (item.targetPath) {
|
if (item.targetPath) {
|
||||||
@ -349,35 +352,35 @@ function cloneSettings(settings: AppSettings): AppSettings {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
type ParsedContentRange = {
|
type ParsedContentRange = {
|
||||||
start: number;
|
start: number;
|
||||||
end: number;
|
end: number;
|
||||||
total: number | null;
|
total: number | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
function parseContentRange(contentRange: string | null): ParsedContentRange | null {
|
function parseContentRange(contentRange: string | null): ParsedContentRange | null {
|
||||||
if (!contentRange) {
|
if (!contentRange) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i);
|
const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i);
|
||||||
if (!match) {
|
if (!match) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const start = Number(match[1]);
|
const start = Number(match[1]);
|
||||||
const end = Number(match[2]);
|
const end = Number(match[2]);
|
||||||
const total = match[3] === "*" ? null : Number(match[3]);
|
const total = match[3] === "*" ? null : Number(match[3]);
|
||||||
if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start) {
|
if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (total !== null && (!Number.isFinite(total) || total <= 0 || end >= total)) {
|
if (total !== null && (!Number.isFinite(total) || total <= 0 || end >= total)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return { start, end, total };
|
return { start, end, total };
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseContentRangeTotal(contentRange: string | null): number | null {
|
function parseContentRangeTotal(contentRange: string | null): number | null {
|
||||||
return parseContentRange(contentRange)?.total ?? null;
|
return parseContentRange(contentRange)?.total ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseContentDispositionFilename(contentDisposition: string | null): string {
|
function parseContentDispositionFilename(contentDisposition: string | null): string {
|
||||||
if (!contentDisposition) {
|
if (!contentDisposition) {
|
||||||
@ -437,6 +440,13 @@ function shouldRejectSuspiciousSmallDownload(
|
|||||||
const size = Math.max(0, Math.floor(Number(fileSizeOnDisk) || 0));
|
const size = Math.max(0, Math.floor(Number(fileSizeOnDisk) || 0));
|
||||||
const expected = Number.isFinite(expectedTotal || NaN) ? Math.max(0, Math.floor(expectedTotal || 0)) : 0;
|
const expected = Number.isFinite(expectedTotal || NaN) ? Math.max(0, Math.floor(expectedTotal || 0)) : 0;
|
||||||
const binaryLike = isLargeBinaryLikePath(filePath || fileName);
|
const binaryLike = isLargeBinaryLikePath(filePath || fileName);
|
||||||
|
const name = path.basename(String(filePath || fileName || ""));
|
||||||
|
|
||||||
|
// Known small files (e.g. .sfv, .nfo) are legitimately tiny — never reject them
|
||||||
|
// as long as they received the expected number of bytes (or we have no expectation).
|
||||||
|
if (KNOWN_SMALL_FILE_RE.test(name) && (expected <= 0 || size >= expected)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (size <= 0) {
|
if (size <= 0) {
|
||||||
return expected > 0 || binaryLike;
|
return expected > 0 || binaryLike;
|
||||||
@ -456,29 +466,29 @@ function shouldRejectSuspiciousSmallDownload(
|
|||||||
return binaryLike;
|
return binaryLike;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isFetchFailure(errorText: string): boolean {
|
function isFetchFailure(errorText: string): boolean {
|
||||||
const text = String(errorText || "").toLowerCase();
|
const text = String(errorText || "").toLowerCase();
|
||||||
return text.includes("fetch failed") || text.includes("socket hang up") || text.includes("econnreset") || text.includes("network error");
|
return text.includes("fetch failed") || text.includes("socket hang up") || text.includes("econnreset") || text.includes("network error");
|
||||||
}
|
}
|
||||||
|
|
||||||
function shouldRewindResumeTail(errorText: string): boolean {
|
function shouldRewindResumeTail(errorText: string): boolean {
|
||||||
const text = String(errorText || "").toLowerCase();
|
const text = String(errorText || "").toLowerCase();
|
||||||
if (!text) {
|
if (!text) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return text.includes("terminated")
|
return text.includes("terminated")
|
||||||
|| text.includes("stall_timeout")
|
|| text.includes("stall_timeout")
|
||||||
|| text.includes("slow_throughput")
|
|| text.includes("slow_throughput")
|
||||||
|| text.includes("write_drain_timeout")
|
|| text.includes("write_drain_timeout")
|
||||||
|| text.includes("premature close")
|
|| text.includes("premature close")
|
||||||
|| text.includes("unexpected eof")
|
|| text.includes("unexpected eof")
|
||||||
|| text.includes("download_underflow")
|
|| text.includes("download_underflow")
|
||||||
|| isFetchFailure(text);
|
|| isFetchFailure(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
function isHttp416Text(errorText: string): boolean {
|
function isHttp416Text(errorText: string): boolean {
|
||||||
return /(^|\D)416(\D|$)/.test(String(errorText || ""));
|
return /(^|\D)416(\D|$)/.test(String(errorText || ""));
|
||||||
}
|
}
|
||||||
|
|
||||||
function shouldPreflightFinalizeItemFromDisk(item: DownloadItem): boolean {
|
function shouldPreflightFinalizeItemFromDisk(item: DownloadItem): boolean {
|
||||||
const text = `${item.fullStatus || ""} ${item.lastError || ""}`.toLowerCase();
|
const text = `${item.fullStatus || ""} ${item.lastError || ""}`.toLowerCase();
|
||||||
@ -5258,35 +5268,35 @@ export class DownloadManager extends EventEmitter {
|
|||||||
* knows which files belong to which items. Without this, after restart all paths are
|
* knows which files belong to which items. Without this, after restart all paths are
|
||||||
* unclaimed and a new download with the same filename would create a "(1)" copy
|
* unclaimed and a new download with the same filename would create a "(1)" copy
|
||||||
* instead of reusing its own partial file — or worse, overwrite another item's file. */
|
* instead of reusing its own partial file — or worse, overwrite another item's file. */
|
||||||
private restoreTargetPathReservations(): void {
|
private restoreTargetPathReservations(): void {
|
||||||
let restored = 0;
|
let restored = 0;
|
||||||
let droppedUnsafe = 0;
|
let droppedUnsafe = 0;
|
||||||
for (const item of Object.values(this.session.items)) {
|
for (const item of Object.values(this.session.items)) {
|
||||||
const pkg = this.session.packages[item.packageId];
|
const pkg = this.session.packages[item.packageId];
|
||||||
if (!pkg) {
|
if (!pkg) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const tp = String(item.targetPath || "").trim();
|
const tp = String(item.targetPath || "").trim();
|
||||||
if (!tp) continue;
|
if (!tp) continue;
|
||||||
if (!isPathInsideDir(tp, pkg.outputDir)) {
|
if (!isPathInsideDir(tp, pkg.outputDir)) {
|
||||||
droppedUnsafe += 1;
|
droppedUnsafe += 1;
|
||||||
item.targetPath = "";
|
item.targetPath = "";
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const key = pathKey(tp);
|
const key = pathKey(tp);
|
||||||
if (!this.reservedTargetPaths.has(key)) {
|
if (!this.reservedTargetPaths.has(key)) {
|
||||||
this.reservedTargetPaths.set(key, item.id);
|
this.reservedTargetPaths.set(key, item.id);
|
||||||
this.claimedTargetPathByItem.set(item.id, tp);
|
this.claimedTargetPathByItem.set(item.id, tp);
|
||||||
restored += 1;
|
restored += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (restored > 0) {
|
if (restored > 0) {
|
||||||
logger.info(`restoreTargetPathReservations: ${restored} Pfade aus Session wiederhergestellt`);
|
logger.info(`restoreTargetPathReservations: ${restored} Pfade aus Session wiederhergestellt`);
|
||||||
}
|
}
|
||||||
if (droppedUnsafe > 0) {
|
if (droppedUnsafe > 0) {
|
||||||
logger.warn(`restoreTargetPathReservations: ${droppedUnsafe} unsichere targetPath-Eintraege verworfen`);
|
logger.warn(`restoreTargetPathReservations: ${droppedUnsafe} unsichere targetPath-Eintraege verworfen`);
|
||||||
}
|
}
|
||||||
this.reconcileDuplicateSuffixSessionItems();
|
this.reconcileDuplicateSuffixSessionItems();
|
||||||
// Fix legacy (N) suffix files: rename back to original if original path is free
|
// Fix legacy (N) suffix files: rename back to original if original path is free
|
||||||
this.fixDuplicateSuffixFiles();
|
this.fixDuplicateSuffixFiles();
|
||||||
}
|
}
|
||||||
@ -5454,7 +5464,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (!targetPath || !item.totalBytes || item.totalBytes <= 0) continue;
|
if (!targetPath || !item.totalBytes || item.totalBytes <= 0) continue;
|
||||||
try {
|
try {
|
||||||
const stat = fs.statSync(targetPath);
|
const stat = fs.statSync(targetPath);
|
||||||
const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || targetPath));
|
const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || targetPath));
|
||||||
const persistedShortfall = item.downloadedBytes < expectedMinSize && stat.size >= expectedMinSize;
|
const persistedShortfall = item.downloadedBytes < expectedMinSize && stat.size >= expectedMinSize;
|
||||||
if (stat.size < expectedMinSize) {
|
if (stat.size < expectedMinSize) {
|
||||||
logger.warn(`revalidateCompleted: ${item.fileName} ist nur ${humanSize(stat.size)} statt ${humanSize(item.totalBytes)}, setze auf queued`);
|
logger.warn(`revalidateCompleted: ${item.fileName} ist nur ${humanSize(stat.size)} statt ${humanSize(item.totalBytes)}, setze auf queued`);
|
||||||
@ -5534,15 +5544,15 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|| normalizedError.includes("resume_download_underflow");
|
|| normalizedError.includes("resume_download_underflow");
|
||||||
const archiveLikeTarget = String(item.fileName || diskState.diskPath || "").toLowerCase();
|
const archiveLikeTarget = String(item.fileName || diskState.diskPath || "").toLowerCase();
|
||||||
const archiveLike = /(?:\.part\d+\.rar|\.rar|\.r\d{2,3}|\.zip(?:\.\d+)?|\.7z(?:\.\d+)?|\.(?:tar(?:\.(?:gz|bz2|xz))?|tgz|tbz2|txz)|\.\d{3})$/i.test(archiveLikeTarget);
|
const archiveLike = /(?:\.part\d+\.rar|\.rar|\.r\d{2,3}|\.zip(?:\.\d+)?|\.7z(?:\.\d+)?|\.(?:tar(?:\.(?:gz|bz2|xz))?|tgz|tbz2|txz)|\.\d{3})$/i.test(archiveLikeTarget);
|
||||||
const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || diskState.diskPath || ""));
|
const expectedMinSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || diskState.diskPath || ""));
|
||||||
const looksComplete = diskState.exists
|
const looksComplete = diskState.exists
|
||||||
&& diskState.fullOnDisk
|
&& diskState.fullOnDisk
|
||||||
&& (
|
&& (
|
||||||
diskState.reason === "ok"
|
diskState.reason === "ok"
|
||||||
|| item.progressPercent >= 100
|
|| item.progressPercent >= 100
|
||||||
|| item.downloadedBytes >= diskState.minBytes
|
|| item.downloadedBytes >= diskState.minBytes
|
||||||
|| (item.totalBytes != null && item.totalBytes > 0 && diskState.size >= expectedMinSize)
|
|| (item.totalBytes != null && item.totalBytes > 0 && diskState.size >= expectedMinSize)
|
||||||
);
|
);
|
||||||
if (!looksComplete || (knownShortfall > 0 && (underflowIndicated || archiveLike))) {
|
if (!looksComplete || (knownShortfall > 0 && (underflowIndicated || archiveLike))) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -8448,80 +8458,80 @@ export class DownloadManager extends EventEmitter {
|
|||||||
const retryDisplayLimit = retryLimitLabel(configuredRetryLimit);
|
const retryDisplayLimit = retryLimitLabel(configuredRetryLimit);
|
||||||
const maxAttemptsBySetting = configuredRetryLimit <= 0 ? Number.MAX_SAFE_INTEGER : configuredRetryLimit + 1;
|
const maxAttemptsBySetting = configuredRetryLimit <= 0 ? Number.MAX_SAFE_INTEGER : configuredRetryLimit + 1;
|
||||||
const maxAttempts = Math.max(1, Math.min(MAX_SAME_DIRECT_URL_ATTEMPTS, maxAttemptsBySetting));
|
const maxAttempts = Math.max(1, Math.min(MAX_SAME_DIRECT_URL_ATTEMPTS, maxAttemptsBySetting));
|
||||||
|
|
||||||
let lastError = "";
|
let lastError = "";
|
||||||
let effectiveTargetPath = targetPath;
|
let effectiveTargetPath = targetPath;
|
||||||
let resumeRewindBytesNextAttempt = 0;
|
let resumeRewindBytesNextAttempt = 0;
|
||||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||||
let existingBytes = 0;
|
let existingBytes = 0;
|
||||||
try {
|
try {
|
||||||
const stat = await fs.promises.stat(effectiveTargetPath);
|
const stat = await fs.promises.stat(effectiveTargetPath);
|
||||||
existingBytes = stat.size;
|
existingBytes = stat.size;
|
||||||
} catch {
|
} catch {
|
||||||
// file does not exist
|
// file does not exist
|
||||||
}
|
}
|
||||||
if (existingBytes > 0 && resumeRewindBytesNextAttempt > 0) {
|
if (existingBytes > 0 && resumeRewindBytesNextAttempt > 0) {
|
||||||
const previousBytes = existingBytes;
|
const previousBytes = existingBytes;
|
||||||
const rewindBytes = Math.min(existingBytes, resumeRewindBytesNextAttempt);
|
const rewindBytes = Math.min(existingBytes, resumeRewindBytesNextAttempt);
|
||||||
const resumeStart = Math.max(0, existingBytes - rewindBytes);
|
const resumeStart = Math.max(0, existingBytes - rewindBytes);
|
||||||
try {
|
try {
|
||||||
await fs.promises.truncate(effectiveTargetPath, resumeStart);
|
await fs.promises.truncate(effectiveTargetPath, resumeStart);
|
||||||
existingBytes = resumeStart;
|
existingBytes = resumeStart;
|
||||||
item.downloadedBytes = Math.min(item.downloadedBytes, existingBytes);
|
item.downloadedBytes = Math.min(item.downloadedBytes, existingBytes);
|
||||||
logAttemptEvent("WARN", "Resume-Schutz aktiv: Teil-Datei vor Retry zurueckgespult", {
|
logAttemptEvent("WARN", "Resume-Schutz aktiv: Teil-Datei vor Retry zurueckgespult", {
|
||||||
attempt,
|
attempt,
|
||||||
previousBytes,
|
previousBytes,
|
||||||
rewindBytes,
|
rewindBytes,
|
||||||
resumeStart
|
resumeStart
|
||||||
});
|
});
|
||||||
} catch (rewindError) {
|
} catch (rewindError) {
|
||||||
logAttemptEvent("WARN", "Resume-Schutz: Rueckspulen der Teil-Datei fehlgeschlagen", {
|
logAttemptEvent("WARN", "Resume-Schutz: Rueckspulen der Teil-Datei fehlgeschlagen", {
|
||||||
attempt,
|
attempt,
|
||||||
previousBytes,
|
previousBytes,
|
||||||
rewindBytes,
|
rewindBytes,
|
||||||
error: compactErrorText(rewindError)
|
error: compactErrorText(rewindError)
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
resumeRewindBytesNextAttempt = 0;
|
resumeRewindBytesNextAttempt = 0;
|
||||||
}
|
}
|
||||||
} else if (resumeRewindBytesNextAttempt > 0) {
|
} else if (resumeRewindBytesNextAttempt > 0) {
|
||||||
resumeRewindBytesNextAttempt = 0;
|
resumeRewindBytesNextAttempt = 0;
|
||||||
}
|
}
|
||||||
const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0));
|
const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0));
|
||||||
const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || effectiveTargetPath || "");
|
const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || effectiveTargetPath || "");
|
||||||
// Guard against pre-allocated sparse files from a crashed session:
|
// Guard against pre-allocated sparse files from a crashed session:
|
||||||
// if file size exceeds persisted downloadedBytes beyond the allowed
|
// if file size exceeds persisted downloadedBytes beyond the allowed
|
||||||
// mismatch threshold, the file was likely pre-allocated but only
|
// mismatch threshold, the file was likely pre-allocated but only
|
||||||
// partially written before a hard crash.
|
// partially written before a hard crash.
|
||||||
// This must also run for persistedBytes=0, otherwise startup-resume can
|
// This must also run for persistedBytes=0, otherwise startup-resume can
|
||||||
// send Range=full-size and incorrectly accept HTTP 416 as "complete".
|
// send Range=full-size and incorrectly accept HTTP 416 as "complete".
|
||||||
if (existingBytes > 0 && existingBytes > persistedBytes + preallocMismatchThreshold) {
|
if (existingBytes > 0 && existingBytes > persistedBytes + preallocMismatchThreshold) {
|
||||||
try {
|
try {
|
||||||
const previousBytes = existingBytes;
|
const previousBytes = existingBytes;
|
||||||
await fs.promises.truncate(effectiveTargetPath, persistedBytes);
|
await fs.promises.truncate(effectiveTargetPath, persistedBytes);
|
||||||
existingBytes = persistedBytes;
|
existingBytes = persistedBytes;
|
||||||
logAttemptEvent("WARN", "Pre-alloc-Rest erkannt, Teil-Datei auf persistierte Bytes gekuerzt", {
|
logAttemptEvent("WARN", "Pre-alloc-Rest erkannt, Teil-Datei auf persistierte Bytes gekuerzt", {
|
||||||
attempt,
|
attempt,
|
||||||
previousBytes,
|
previousBytes,
|
||||||
persistedBytes
|
persistedBytes
|
||||||
});
|
});
|
||||||
} catch {
|
} catch {
|
||||||
if (persistedBytes === 0) {
|
if (persistedBytes === 0) {
|
||||||
try {
|
try {
|
||||||
await fs.promises.rm(effectiveTargetPath, { force: true });
|
await fs.promises.rm(effectiveTargetPath, { force: true });
|
||||||
existingBytes = 0;
|
existingBytes = 0;
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const suspiciousResumeFootprint = existingBytes > 0
|
const suspiciousResumeFootprint = existingBytes > 0
|
||||||
&& existingBytes > persistedBytes + preallocMismatchThreshold;
|
&& existingBytes > persistedBytes + preallocMismatchThreshold;
|
||||||
const headers: Record<string, string> = {};
|
const headers: Record<string, string> = {};
|
||||||
if (existingBytes > 0) {
|
if (existingBytes > 0) {
|
||||||
headers.Range = `bytes=${existingBytes}-`;
|
headers.Range = `bytes=${existingBytes}-`;
|
||||||
}
|
}
|
||||||
logAttemptEvent("INFO", "HTTP-Download-Versuch vorbereitet", {
|
logAttemptEvent("INFO", "HTTP-Download-Versuch vorbereitet", {
|
||||||
attempt,
|
attempt,
|
||||||
maxAttempts: maxAttempts === Number.MAX_SAFE_INTEGER ? "infinite" : maxAttempts,
|
maxAttempts: maxAttempts === Number.MAX_SAFE_INTEGER ? "infinite" : maxAttempts,
|
||||||
@ -8589,37 +8599,37 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (response.status === 416 && existingBytes > 0) {
|
if (response.status === 416 && existingBytes > 0) {
|
||||||
await response.arrayBuffer().catch(() => undefined);
|
await response.arrayBuffer().catch(() => undefined);
|
||||||
const rangeTotal = parseContentRangeTotal(response.headers.get("content-range"));
|
const rangeTotal = parseContentRangeTotal(response.headers.get("content-range"));
|
||||||
const expectedTotal = rangeTotal && rangeTotal > 0
|
const expectedTotal = rangeTotal && rangeTotal > 0
|
||||||
? rangeTotal
|
? rangeTotal
|
||||||
: (knownTotal && knownTotal > 0 ? knownTotal : null);
|
: (knownTotal && knownTotal > 0 ? knownTotal : null);
|
||||||
const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE;
|
const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE;
|
||||||
const closeEnoughToExpected = expectedTotal != null
|
const closeEnoughToExpected = expectedTotal != null
|
||||||
&& Math.abs(existingBytes - expectedTotal) <= sizeToleranceBytes;
|
&& Math.abs(existingBytes - expectedTotal) <= sizeToleranceBytes;
|
||||||
if (expectedTotal != null && closeEnoughToExpected && !suspiciousResumeFootprint) {
|
if (expectedTotal != null && closeEnoughToExpected && !suspiciousResumeFootprint) {
|
||||||
const finalizedTotal = Math.max(existingBytes, expectedTotal);
|
const finalizedTotal = Math.max(existingBytes, expectedTotal);
|
||||||
item.totalBytes = finalizedTotal;
|
item.totalBytes = finalizedTotal;
|
||||||
item.downloadedBytes = existingBytes;
|
item.downloadedBytes = existingBytes;
|
||||||
item.progressPercent = 100;
|
item.progressPercent = 100;
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
logAttemptEvent("INFO", "HTTP 416 als vollständig behandelt", {
|
logAttemptEvent("INFO", "HTTP 416 als vollständig behandelt", {
|
||||||
existingBytes,
|
existingBytes,
|
||||||
expectedTotal: finalizedTotal
|
expectedTotal: finalizedTotal
|
||||||
});
|
});
|
||||||
return { resumable: true };
|
return { resumable: true };
|
||||||
}
|
}
|
||||||
if (expectedTotal != null && closeEnoughToExpected && suspiciousResumeFootprint) {
|
if (expectedTotal != null && closeEnoughToExpected && suspiciousResumeFootprint) {
|
||||||
logAttemptEvent("WARN", "HTTP 416 trotz Vollgroesse nicht als fertig gewertet (vermutlich pre-alloc)", {
|
logAttemptEvent("WARN", "HTTP 416 trotz Vollgroesse nicht als fertig gewertet (vermutlich pre-alloc)", {
|
||||||
attempt,
|
attempt,
|
||||||
existingBytes,
|
existingBytes,
|
||||||
persistedBytes,
|
persistedBytes,
|
||||||
expectedTotal
|
expectedTotal
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fs.promises.rm(effectiveTargetPath, { force: true });
|
await fs.promises.rm(effectiveTargetPath, { force: true });
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
this.dropItemContribution(active.itemId);
|
this.dropItemContribution(active.itemId);
|
||||||
@ -8698,8 +8708,8 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|
|
||||||
const rawContentLength = Number(response.headers.get("content-length") || 0);
|
const rawContentLength = Number(response.headers.get("content-length") || 0);
|
||||||
const contentLength = Number.isFinite(rawContentLength) && rawContentLength > 0 ? rawContentLength : 0;
|
const contentLength = Number.isFinite(rawContentLength) && rawContentLength > 0 ? rawContentLength : 0;
|
||||||
const parsedContentRange = parseContentRange(response.headers.get("content-range"));
|
const parsedContentRange = parseContentRange(response.headers.get("content-range"));
|
||||||
const totalFromRange = parsedContentRange?.total ?? null;
|
const totalFromRange = parsedContentRange?.total ?? null;
|
||||||
const serverIgnoredRange = existingBytes > 0 && response.status === 200;
|
const serverIgnoredRange = existingBytes > 0 && response.status === 200;
|
||||||
const allowFreshOverwriteAfterResumeReset = serverIgnoredRange
|
const allowFreshOverwriteAfterResumeReset = serverIgnoredRange
|
||||||
&& active.resumeHardResetUsed
|
&& active.resumeHardResetUsed
|
||||||
@ -8719,69 +8729,69 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
throw new Error(`range_ignored_on_resume:${existingBytes}/${contentLength || 0}`);
|
throw new Error(`range_ignored_on_resume:${existingBytes}/${contentLength || 0}`);
|
||||||
}
|
}
|
||||||
if (allowFreshOverwriteAfterResumeReset) {
|
if (allowFreshOverwriteAfterResumeReset) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`Server ignorierte Range-Header nach Resume-Reset, ueberschreibe alte Teil-Datei frisch: ${item.fileName}`
|
`Server ignorierte Range-Header nach Resume-Reset, ueberschreibe alte Teil-Datei frisch: ${item.fileName}`
|
||||||
);
|
);
|
||||||
logAttemptEvent("WARN", "Range ignoriert nach Resume-Reset, frischer Vollstart erlaubt", {
|
logAttemptEvent("WARN", "Range ignoriert nach Resume-Reset, frischer Vollstart erlaubt", {
|
||||||
attempt,
|
attempt,
|
||||||
existingBytes,
|
existingBytes,
|
||||||
contentLength,
|
contentLength,
|
||||||
directUrl
|
directUrl
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (existingBytes > 0 && response.status === 206) {
|
if (existingBytes > 0 && response.status === 206) {
|
||||||
if (!parsedContentRange) {
|
if (!parsedContentRange) {
|
||||||
logAttemptEvent("WARN", "Resume-Range-Header ungueltig oder fehlt", {
|
logAttemptEvent("WARN", "Resume-Range-Header ungueltig oder fehlt", {
|
||||||
attempt,
|
attempt,
|
||||||
existingBytes,
|
existingBytes,
|
||||||
contentRange: response.headers.get("content-range") || ""
|
contentRange: response.headers.get("content-range") || ""
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
await response.body?.cancel();
|
await response.body?.cancel();
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
throw new Error(`range_mismatch_on_resume:${existingBytes}/invalid`);
|
throw new Error(`range_mismatch_on_resume:${existingBytes}/invalid`);
|
||||||
}
|
}
|
||||||
if (parsedContentRange.start !== existingBytes) {
|
if (parsedContentRange.start !== existingBytes) {
|
||||||
const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE;
|
const sizeToleranceBytes = isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE;
|
||||||
const canTreatAsAlreadyComplete = contentLength === 0
|
const canTreatAsAlreadyComplete = contentLength === 0
|
||||||
&& parsedContentRange.start === 0
|
&& parsedContentRange.start === 0
|
||||||
&& parsedContentRange.total != null
|
&& parsedContentRange.total != null
|
||||||
&& Math.abs(existingBytes - parsedContentRange.total) <= sizeToleranceBytes;
|
&& Math.abs(existingBytes - parsedContentRange.total) <= sizeToleranceBytes;
|
||||||
if (canTreatAsAlreadyComplete) {
|
if (canTreatAsAlreadyComplete) {
|
||||||
item.totalBytes = parsedContentRange.total;
|
item.totalBytes = parsedContentRange.total;
|
||||||
item.downloadedBytes = existingBytes;
|
item.downloadedBytes = existingBytes;
|
||||||
item.progressPercent = 100;
|
item.progressPercent = 100;
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
logAttemptEvent("WARN", "Resume-Range-Start abweichend, Datei aber bereits vollstaendig", {
|
logAttemptEvent("WARN", "Resume-Range-Start abweichend, Datei aber bereits vollstaendig", {
|
||||||
attempt,
|
attempt,
|
||||||
existingBytes,
|
existingBytes,
|
||||||
totalFromRange: parsedContentRange.total,
|
totalFromRange: parsedContentRange.total,
|
||||||
contentLength
|
contentLength
|
||||||
});
|
});
|
||||||
return { resumable: true };
|
return { resumable: true };
|
||||||
}
|
}
|
||||||
logAttemptEvent("WARN", "Resume-Range-Start stimmt nicht mit lokaler Dateigroesse ueberein", {
|
logAttemptEvent("WARN", "Resume-Range-Start stimmt nicht mit lokaler Dateigroesse ueberein", {
|
||||||
attempt,
|
attempt,
|
||||||
expectedStart: existingBytes,
|
expectedStart: existingBytes,
|
||||||
actualStart: parsedContentRange.start,
|
actualStart: parsedContentRange.start,
|
||||||
actualEnd: parsedContentRange.end,
|
actualEnd: parsedContentRange.end,
|
||||||
totalFromRange,
|
totalFromRange,
|
||||||
directUrl
|
directUrl
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
await response.body?.cancel();
|
await response.body?.cancel();
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
throw new Error(`range_mismatch_on_resume:${existingBytes}/${parsedContentRange.start}`);
|
throw new Error(`range_mismatch_on_resume:${existingBytes}/${parsedContentRange.start}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const correctedRealDebridTotal = getAuthoritativeRealDebridTotal(
|
const correctedRealDebridTotal = getAuthoritativeRealDebridTotal(
|
||||||
item.provider,
|
item.provider,
|
||||||
knownTotal || 0,
|
knownTotal || 0,
|
||||||
existingBytes,
|
existingBytes,
|
||||||
@ -9340,40 +9350,46 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Detect tiny error-response files (e.g. hoster returning "Forbidden" with HTTP 200).
|
// Detect tiny error-response files (e.g. hoster returning "Forbidden" with HTTP 200).
|
||||||
// No legitimate file-hoster download is < 512 bytes.
|
// No legitimate file-hoster download is < 512 bytes, EXCEPT known small metadata
|
||||||
|
// files like .sfv (checksum verification), .nfo (release info), etc.
|
||||||
if (written > 0 && written < 512) {
|
if (written > 0 && written < 512) {
|
||||||
let snippet = "";
|
const knownSmallFile = KNOWN_SMALL_FILE_RE.test(item.fileName || effectiveTargetPath);
|
||||||
try {
|
if (knownSmallFile && ((!item.totalBytes || item.totalBytes <= 0) || written >= item.totalBytes)) {
|
||||||
snippet = await fs.promises.readFile(effectiveTargetPath, "utf8");
|
logger.info(`Kleine Metadaten-Datei akzeptiert (${written} B): ${item.fileName || effectiveTargetPath}`);
|
||||||
snippet = snippet.slice(0, 200).replace(/[\r\n]+/g, " ").trim();
|
|
||||||
} catch { /* ignore */ }
|
|
||||||
const exactTinyBinary = Boolean(
|
|
||||||
item.totalBytes
|
|
||||||
&& item.totalBytes > 0
|
|
||||||
&& written >= item.totalBytes
|
|
||||||
&& isLargeBinaryLikePath(item.fileName || effectiveTargetPath)
|
|
||||||
);
|
|
||||||
const snippetSuggestsError = /<(?:!doctype|html|body)\b|\b(?:forbidden|access denied|error|not found|expired|unavailable)\b/i.test(snippet);
|
|
||||||
if (exactTinyBinary && !snippetSuggestsError) {
|
|
||||||
logger.info(`Tiny Binary akzeptiert (${written} B): ${item.fileName || effectiveTargetPath}`);
|
|
||||||
} else {
|
} else {
|
||||||
logger.warn(`Tiny download erkannt (${written} B): "${snippet}"`);
|
let snippet = "";
|
||||||
try {
|
try {
|
||||||
await fs.promises.rm(effectiveTargetPath, { force: true });
|
snippet = await fs.promises.readFile(effectiveTargetPath, "utf8");
|
||||||
} catch { /* ignore */ }
|
snippet = snippet.slice(0, 200).replace(/[\r\n]+/g, " ").trim();
|
||||||
this.releaseTargetPath(active.itemId);
|
} catch { /* ignore */ }
|
||||||
this.dropItemContribution(active.itemId);
|
const exactTinyBinary = Boolean(
|
||||||
item.downloadedBytes = 0;
|
item.totalBytes
|
||||||
item.progressPercent = 0;
|
&& item.totalBytes > 0
|
||||||
throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`);
|
&& written >= item.totalBytes
|
||||||
|
&& isLargeBinaryLikePath(item.fileName || effectiveTargetPath)
|
||||||
|
);
|
||||||
|
const snippetSuggestsError = /<(?:!doctype|html|body)\b|\b(?:forbidden|access denied|error|not found|expired|unavailable)\b/i.test(snippet);
|
||||||
|
if (exactTinyBinary && !snippetSuggestsError) {
|
||||||
|
logger.info(`Tiny Binary akzeptiert (${written} B): ${item.fileName || effectiveTargetPath}`);
|
||||||
|
} else {
|
||||||
|
logger.warn(`Tiny download erkannt (${written} B): "${snippet}"`);
|
||||||
|
try {
|
||||||
|
await fs.promises.rm(effectiveTargetPath, { force: true });
|
||||||
|
} catch { /* ignore */ }
|
||||||
|
this.releaseTargetPath(active.itemId);
|
||||||
|
this.dropItemContribution(active.itemId);
|
||||||
|
item.downloadedBytes = 0;
|
||||||
|
item.progressPercent = 0;
|
||||||
|
throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const completionValidation = validateDownloadedFileCompletion({
|
const completionValidation = validateDownloadedFileCompletion({
|
||||||
actualBytes: written,
|
actualBytes: written,
|
||||||
plan: completionPlan,
|
plan: completionPlan,
|
||||||
toleranceBytes: isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE
|
toleranceBytes: isLargeBinaryLikePath(item.fileName || effectiveTargetPath) ? 0 : ALLOCATION_UNIT_SIZE
|
||||||
});
|
});
|
||||||
if (!completionValidation.ok) {
|
if (!completionValidation.ok) {
|
||||||
const shortfall = Math.max(0, completionValidation.totalBytes - written);
|
const shortfall = Math.max(0, completionValidation.totalBytes - written);
|
||||||
if (preAllocated) {
|
if (preAllocated) {
|
||||||
@ -9445,27 +9461,27 @@ export class DownloadManager extends EventEmitter {
|
|||||||
error: lastError,
|
error: lastError,
|
||||||
targetPath: effectiveTargetPath
|
targetPath: effectiveTargetPath
|
||||||
});
|
});
|
||||||
if (
|
if (
|
||||||
normalizedLastError.startsWith("range_ignored_on_resume:")
|
normalizedLastError.startsWith("range_ignored_on_resume:")
|
||||||
|| normalizedLastError.startsWith("range_mismatch_on_resume:")
|
|| normalizedLastError.startsWith("range_mismatch_on_resume:")
|
||||||
) {
|
) {
|
||||||
throw new Error(`direct_link_retry_exhausted:${normalizedLastError}`);
|
throw new Error(`direct_link_retry_exhausted:${normalizedLastError}`);
|
||||||
}
|
}
|
||||||
if (attempt < maxAttempts && written > existingBytes && shouldRewindResumeTail(normalizedLastError)) {
|
if (attempt < maxAttempts && written > existingBytes && shouldRewindResumeTail(normalizedLastError)) {
|
||||||
resumeRewindBytesNextAttempt = Math.max(resumeRewindBytesNextAttempt, RESUME_REWIND_BYTES);
|
resumeRewindBytesNextAttempt = Math.max(resumeRewindBytesNextAttempt, RESUME_REWIND_BYTES);
|
||||||
logAttemptEvent("WARN", "Resume-Schutz vorgemerkt: naechster Retry startet mit Rewind", {
|
logAttemptEvent("WARN", "Resume-Schutz vorgemerkt: naechster Retry startet mit Rewind", {
|
||||||
attempt,
|
attempt,
|
||||||
existingBytes,
|
existingBytes,
|
||||||
written,
|
written,
|
||||||
appendedBytes: Math.max(0, written - existingBytes),
|
appendedBytes: Math.max(0, written - existingBytes),
|
||||||
rewindBytes: resumeRewindBytesNextAttempt,
|
rewindBytes: resumeRewindBytesNextAttempt,
|
||||||
error: normalizedLastError
|
error: normalizedLastError
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (attempt < maxAttempts) {
|
if (attempt < maxAttempts) {
|
||||||
item.retries += 1;
|
item.retries += 1;
|
||||||
item.fullStatus = `Downloadfehler, retry ${attempt}/${maxAttempts} (Direktlink)`;
|
item.fullStatus = `Downloadfehler, retry ${attempt}/${maxAttempts} (Direktlink)`;
|
||||||
this.emitState();
|
this.emitState();
|
||||||
await sleep(retryDelayWithJitter(attempt, 250));
|
await sleep(retryDelayWithJitter(attempt, 250));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -9943,8 +9959,8 @@ export class DownloadManager extends EventEmitter {
|
|||||||
try {
|
try {
|
||||||
const stat = await fs.promises.stat(part);
|
const stat = await fs.promises.stat(part);
|
||||||
// Find the item that owns this file to get its expected totalBytes
|
// Find the item that owns this file to get its expected totalBytes
|
||||||
const ownerItem = this.findItemByDiskPath(pkg, part);
|
const ownerItem = this.findItemByDiskPath(pkg, part);
|
||||||
const minBytes = expectedMinBytes(ownerItem?.totalBytes ?? 0, isLargeBinaryLikePath(part));
|
const minBytes = expectedMinBytes(ownerItem?.totalBytes ?? 0, isLargeBinaryLikePath(part));
|
||||||
if (stat.size < minBytes) {
|
if (stat.size < minBytes) {
|
||||||
allMissingFullOnDisk = false;
|
allMissingFullOnDisk = false;
|
||||||
break;
|
break;
|
||||||
@ -10478,72 +10494,72 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (item.status === "downloading" || item.status === "validating" || item.status === "integrity_check") {
|
if (item.status === "downloading" || item.status === "validating" || item.status === "integrity_check") {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!item.targetPath) {
|
if (!item.targetPath) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!isPathInsideDir(item.targetPath, pkg.outputDir)) {
|
if (!isPathInsideDir(item.targetPath, pkg.outputDir)) {
|
||||||
logger.warn(`Item-Recovery: Unsicherer targetPath verworfen (${item.fileName} -> ${item.targetPath})`);
|
logger.warn(`Item-Recovery: Unsicherer targetPath verworfen (${item.fileName} -> ${item.targetPath})`);
|
||||||
this.releaseTargetPath(item.id);
|
this.releaseTargetPath(item.id);
|
||||||
this.dropItemContribution(item.id);
|
this.dropItemContribution(item.id);
|
||||||
item.targetPath = "";
|
item.targetPath = "";
|
||||||
item.status = "queued";
|
item.status = "queued";
|
||||||
item.attempts = 0;
|
item.attempts = 0;
|
||||||
item.downloadedBytes = 0;
|
item.downloadedBytes = 0;
|
||||||
item.progressPercent = 0;
|
item.progressPercent = 0;
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
item.fullStatus = "Wartet (ungueltiger Zielpfad)";
|
item.fullStatus = "Wartet (ungueltiger Zielpfad)";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const stat = await fs.promises.stat(item.targetPath);
|
const stat = await fs.promises.stat(item.targetPath);
|
||||||
// Require file to be essentially complete — within one allocation unit of the
|
// Require file to be essentially complete — within one allocation unit of the
|
||||||
// expected size. The old 50% threshold incorrectly recovered partial downloads
|
// expected size. The old 50% threshold incorrectly recovered partial downloads
|
||||||
// (e.g. 627 MB of 1001 MB) and triggered hybrid extraction on incomplete archives.
|
// (e.g. 627 MB of 1001 MB) and triggered hybrid extraction on incomplete archives.
|
||||||
const minSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || item.targetPath));
|
const minSize = expectedMinBytes(item.totalBytes, isLargeBinaryLikePath(item.fileName || item.targetPath));
|
||||||
const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0));
|
const persistedBytes = Math.max(0, Math.floor(Number(item.downloadedBytes) || 0));
|
||||||
const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || item.targetPath || "");
|
const preallocMismatchThreshold = resolvePreallocResumeMismatchThreshold(item.fileName || item.targetPath || "");
|
||||||
const suspiciousPreallocFootprint = item.totalBytes != null
|
const suspiciousPreallocFootprint = item.totalBytes != null
|
||||||
&& item.totalBytes > 0
|
&& item.totalBytes > 0
|
||||||
&& stat.size >= minSize
|
&& stat.size >= minSize
|
||||||
&& stat.size > persistedBytes + preallocMismatchThreshold;
|
&& stat.size > persistedBytes + preallocMismatchThreshold;
|
||||||
if (stat.size >= minSize) {
|
if (stat.size >= minSize) {
|
||||||
// Re-check: another task may have started this item during the await
|
// Re-check: another task may have started this item during the await
|
||||||
const latestItem = this.session.items[item.id];
|
const latestItem = this.session.items[item.id];
|
||||||
if (!latestItem || this.activeTasks.has(item.id) || latestItem.status === "downloading"
|
if (!latestItem || this.activeTasks.has(item.id) || latestItem.status === "downloading"
|
||||||
|| latestItem.status === "validating" || latestItem.status === "integrity_check") {
|
|| latestItem.status === "validating" || latestItem.status === "integrity_check") {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (suspiciousPreallocFootprint) {
|
if (suspiciousPreallocFootprint) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`Item-Recovery: ${item.fileName} uebersprungen – pre-alloc-Verdacht ` +
|
`Item-Recovery: ${item.fileName} uebersprungen – pre-alloc-Verdacht ` +
|
||||||
`(stat=${humanSize(stat.size)}, bytes=${humanSize(persistedBytes)}, total=${humanSize(item.totalBytes)})`
|
`(stat=${humanSize(stat.size)}, bytes=${humanSize(persistedBytes)}, total=${humanSize(item.totalBytes)})`
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
if (persistedBytes > 0) {
|
if (persistedBytes > 0) {
|
||||||
fs.truncateSync(item.targetPath, persistedBytes);
|
fs.truncateSync(item.targetPath, persistedBytes);
|
||||||
} else {
|
} else {
|
||||||
fs.rmSync(item.targetPath, { force: true });
|
fs.rmSync(item.targetPath, { force: true });
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// best-effort
|
// best-effort
|
||||||
}
|
}
|
||||||
item.status = "queued";
|
item.status = "queued";
|
||||||
item.attempts = 0;
|
item.attempts = 0;
|
||||||
item.downloadedBytes = persistedBytes;
|
item.downloadedBytes = persistedBytes;
|
||||||
item.progressPercent = item.totalBytes > 0
|
item.progressPercent = item.totalBytes > 0
|
||||||
? Math.max(0, Math.min(99, Math.floor((persistedBytes / item.totalBytes) * 100)))
|
? Math.max(0, Math.min(99, Math.floor((persistedBytes / item.totalBytes) * 100)))
|
||||||
: 0;
|
: 0;
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
item.fullStatus = "Wartet (Auto-Recovery: pre-alloc)";
|
item.fullStatus = "Wartet (Auto-Recovery: pre-alloc)";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Guard against pre-allocated sparse files from a hard crash: file has
|
// Guard against pre-allocated sparse files from a hard crash: file has
|
||||||
// the full expected size but downloadedBytes is significantly behind.
|
// the full expected size but downloadedBytes is significantly behind.
|
||||||
if (item.downloadedBytes > 0 && item.totalBytes && item.totalBytes > 0
|
if (item.downloadedBytes > 0 && item.totalBytes && item.totalBytes > 0
|
||||||
&& stat.size >= minSize
|
&& stat.size >= minSize
|
||||||
&& item.downloadedBytes < item.totalBytes * 0.95) {
|
&& item.downloadedBytes < item.totalBytes * 0.95) {
|
||||||
logger.warn(`Item-Recovery: ${item.fileName} uebersprungen – vermutlich pre-alloc (stat=${humanSize(stat.size)}, bytes=${humanSize(item.downloadedBytes)}, total=${humanSize(item.totalBytes)})`);
|
logger.warn(`Item-Recovery: ${item.fileName} uebersprungen – vermutlich pre-alloc (stat=${humanSize(stat.size)}, bytes=${humanSize(item.downloadedBytes)}, total=${humanSize(item.totalBytes)})`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6140,6 +6140,65 @@ describe("download manager", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("accepts small .sfv metadata files without rejecting them as suspicious", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
// SFV content is just CRC32 checksums — legitimately tiny
|
||||||
|
const sfvContent = Buffer.from("archive.part1.rar 1A2B3C4D\narchive.part2.rar 5E6F7A8B\n", "utf8");
|
||||||
|
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
res.statusCode = 200;
|
||||||
|
res.setHeader("Content-Length", String(sfvContent.length));
|
||||||
|
res.end(sfvContent);
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(0, "127.0.0.1");
|
||||||
|
await once(server, "listening");
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") throw new Error("server address unavailable");
|
||||||
|
const directUrl = `http://127.0.0.1:${address.port}/checksum.sfv`;
|
||||||
|
|
||||||
|
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||||
|
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||||
|
if (url.includes("/unrestrict/link")) {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({ download: directUrl, filename: "archive.sfv", filesize: sfvContent.length }),
|
||||||
|
{ status: 200, headers: { "Content-Type": "application/json" } }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return originalFetch(input, init);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const manager = new DownloadManager(
|
||||||
|
{
|
||||||
|
...defaultSettings(),
|
||||||
|
token: "rd-token",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: path.join(root, "extract"),
|
||||||
|
autoExtract: false,
|
||||||
|
autoReconnect: false
|
||||||
|
},
|
||||||
|
emptySession(),
|
||||||
|
createStoragePaths(path.join(root, "state"))
|
||||||
|
);
|
||||||
|
|
||||||
|
manager.addPackages([{ name: "sfv-test", links: ["https://dummy/sfv-file"] }]);
|
||||||
|
await manager.start();
|
||||||
|
await waitFor(() => !manager.getSnapshot().session.running, 15000);
|
||||||
|
|
||||||
|
const item = Object.values(manager.getSnapshot().session.items)[0];
|
||||||
|
expect(item?.status).toBe("completed");
|
||||||
|
expect(item?.retries).toBe(0);
|
||||||
|
expect(fs.existsSync(item.targetPath)).toBe(true);
|
||||||
|
const onDisk = fs.readFileSync(item.targetPath);
|
||||||
|
expect(onDisk.length).toBe(sfvContent.length);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
await once(server, "close");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("limits AllDebrid rapidgator starts to one active task by default", async () => {
|
it("limits AllDebrid rapidgator starts to one active task by default", async () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user