From 272b43d59e5660f0dcb745452869bdd33af0bb18 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Fri, 6 Mar 2026 20:05:53 +0100 Subject: [PATCH] Release v1.6.88 --- package-lock.json | 4 +- package.json | 2 +- src/main/download-manager.ts | 394 ++++++++++++++++++++++++++++++--- tests/download-manager.test.ts | 358 +++++++++++++++++++++++++++++- 4 files changed, 724 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6495aad..a33046f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "real-debrid-downloader", - "version": "1.6.83", + "version": "1.6.88", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "real-debrid-downloader", - "version": "1.6.83", + "version": "1.6.88", "license": "MIT", "dependencies": { "adm-zip": "^0.5.16", diff --git a/package.json b/package.json index 6620b10..e88cd20 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "real-debrid-downloader", - "version": "1.6.87", + "version": "1.6.88", "description": "Desktop downloader", "main": "build/main/main/main.js", "author": "Sucukdeluxe", diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 4ee5de8..b15348e 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -4,6 +4,7 @@ import os from "node:os"; import { EventEmitter } from "node:events"; import { v4 as uuidv4 } from "uuid"; import { + AllDebridHostInfo, AppSettings, DownloadItem, DownloadStats, @@ -38,7 +39,7 @@ function releaseTlsSkip(): void { } } import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup"; -import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkRapidgatorOnline } from "./debrid"; +import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkRapidgatorOnline, fetchAllDebridHostInfo } from "./debrid"; import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree } from "./extractor"; import { validateFileAgainstManifest } from "./integrity"; import { logger } from "./logger"; @@ -76,6 +77,14 @@ const DEFAULT_LOW_THROUGHPUT_TIMEOUT_MS = 120000; const DEFAULT_LOW_THROUGHPUT_MIN_BYTES = 64 * 1024; +const MINI_DOWNLOAD_RETRY_THRESHOLD_BYTES = 1024 * 1024; + +const ALLDEBRID_HOST_INFO_TTL_MS = 60000; + +const ALLDEBRID_START_STAGGER_MS = 2500; + +const LARGE_BINARY_FILE_RE = /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|7z(?:\.\d+)?|tar|gz|bz2|xz|iso|mkv|mp4|avi|mov|wmv|m4v|ts|m2ts|webm|mp3|flac|aac|wav)$/i; + function getDownloadStallTimeoutMs(): number { const fromEnv = Number(process.env.RD_STALL_TIMEOUT_MS ?? NaN); if (Number.isFinite(fromEnv) && fromEnv >= 2000 && fromEnv <= 600000) { @@ -239,6 +248,46 @@ function isArchiveLikePath(filePath: string): boolean { return /\.(?:part\d+\.rar|rar|r\d{2,3}|zip(?:\.\d+)?|z\d{1,3}|7z(?:\.\d+)?)$/i.test(lower); } +function extractHosterKey(link: string): string { + try { + const host = new URL(link).hostname.replace(/^www\./, "").toLowerCase(); + const parts = host.split("."); + return parts.length >= 2 ? parts[parts.length - 2] : host; + } catch { + return ""; + } +} + +function isLargeBinaryLikePath(filePath: string): boolean { + const lower = path.basename(String(filePath || "")).toLowerCase(); + return isArchiveLikePath(lower) || LARGE_BINARY_FILE_RE.test(lower); +} + +function shouldRejectSuspiciousSmallDownload( + filePath: string, + fileName: string, + fileSizeOnDisk: number, + expectedTotal: number | null +): boolean { + const size = Math.max(0, Math.floor(Number(fileSizeOnDisk) || 0)); + const expected = Number.isFinite(expectedTotal || NaN) ? Math.max(0, Math.floor(expectedTotal || 0)) : 0; + const binaryLike = isLargeBinaryLikePath(filePath || fileName); + + if (size <= 0) { + return expected > 0 || binaryLike; + } + if (size < 512) { + return true; + } + if (size >= MINI_DOWNLOAD_RETRY_THRESHOLD_BYTES) { + return false; + } + if (expected >= MINI_DOWNLOAD_RETRY_THRESHOLD_BYTES) { + return true; + } + return binaryLike; +} + function isFetchFailure(errorText: string): boolean { const text = String(errorText || "").toLowerCase(); return text.includes("fetch failed") || text.includes("socket hang up") || text.includes("econnreset") || text.includes("network error"); @@ -282,6 +331,23 @@ function isProviderBusyUnrestrictError(errorText: string): boolean { || text.includes("zu viele downloads"); } +function isTemporaryUnrestrictError(errorText: string): boolean { + const text = String(errorText || "").toLowerCase(); + return text.includes("server error") + || text.includes("internal server error") + || text.includes("temporarily unavailable") + || text.includes("temporary unavailable") + || text.includes("temporarily disabled") + || text.includes("try again later") + || text.includes("service unavailable") + || text.includes("host is down") + || text.includes("maintenance") + || text.includes("bad gateway") + || text.includes("gateway timeout") + || text.includes("cloudflare") + || text.includes("worker error"); +} + function isFinishedStatus(status: DownloadStatus): boolean { return status === "completed" || status === "failed" || status === "cancelled"; } @@ -941,6 +1007,10 @@ export class DownloadManager extends EventEmitter { private providerFailures = new Map(); + private allDebridHostInfoCache = new Map(); + + private providerStartReservations = new Map(); + private lastStaleResetAt = 0; private onHistoryEntryCallback?: HistoryEntryCallback; @@ -972,6 +1042,7 @@ export class DownloadManager extends EventEmitter { next.totalDownloadedAllTime = Math.max(next.totalDownloadedAllTime || 0, this.settings.totalDownloadedAllTime || 0); this.settings = next; this.debridService.setSettings(next); + this.allDebridHostInfoCache.clear(); this.resolveExistingQueuedOpaqueFilenames(); void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (setSettings): ${compactErrorText(err)}`)); if (next.completedCleanupPolicy !== "never") { @@ -1322,6 +1393,7 @@ export class DownloadManager extends EventEmitter { this.runCompletedPackages.clear(); this.historyRecordedPackages.clear(); this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.retryStateByItem.clear(); this.reservedTargetPaths.clear(); this.claimedTargetPathByItem.clear(); @@ -2922,6 +2994,7 @@ export class DownloadManager extends EventEmitter { this.runOutcomes.clear(); this.runCompletedPackages.clear(); this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.retryStateByItem.clear(); this.itemContributedBytes.clear(); this.reservedTargetPaths.clear(); @@ -3029,6 +3102,7 @@ export class DownloadManager extends EventEmitter { this.runOutcomes.clear(); this.runCompletedPackages.clear(); this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.retryStateByItem.clear(); this.itemContributedBytes.clear(); this.reservedTargetPaths.clear(); @@ -3132,6 +3206,7 @@ export class DownloadManager extends EventEmitter { this.runOutcomes.clear(); this.runCompletedPackages.clear(); this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.retryStateByItem.clear(); this.reservedTargetPaths.clear(); this.claimedTargetPathByItem.clear(); @@ -3204,6 +3279,7 @@ export class DownloadManager extends EventEmitter { this.session.reconnectUntil = 0; this.session.reconnectReason = ""; this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.retryStateByItem.clear(); this.lastGlobalProgressBytes = this.session.totalDownloadedBytes; this.lastGlobalProgressAt = nowMs(); @@ -3315,6 +3391,7 @@ export class DownloadManager extends EventEmitter { this.runOutcomes.clear(); this.runCompletedPackages.clear(); this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.nonResumableActive = 0; this.session.summaryText = ""; // Persist synchronously on shutdown to guarantee data is written before process exits @@ -3347,6 +3424,7 @@ export class DownloadManager extends EventEmitter { // and abort long-stuck validating/downloading tasks so they get retried fresh. if (wasPaused && !this.session.paused) { this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); // Reset provider circuit breaker so items don't sit in cooldown after unpause this.providerFailures.clear(); @@ -4201,8 +4279,9 @@ export class DownloadManager extends EventEmitter { // ── Provider Circuit Breaker ────────────────────────────────────────── private recordProviderFailure(provider: string): void { + const key = String(provider || "").trim() || "unknown"; const now = nowMs(); - const entry = this.providerFailures.get(provider) || { count: 0, lastFailAt: 0, cooldownUntil: 0 }; + const entry = this.providerFailures.get(key) || { count: 0, lastFailAt: 0, cooldownUntil: 0 }; // Decay: if last failure was >120s ago, reset count (transient burst is over) if (entry.lastFailAt > 0 && now - entry.lastFailAt > 120000) { entry.count = 0; @@ -4211,7 +4290,7 @@ export class DownloadManager extends EventEmitter { // This prevents 8 parallel downloads failing at once from immediately hitting the threshold if (entry.lastFailAt > 0 && now - entry.lastFailAt < 2000) { entry.lastFailAt = now; - this.providerFailures.set(provider, entry); + this.providerFailures.set(key, entry); return; } entry.count += 1; @@ -4221,20 +4300,21 @@ export class DownloadManager extends EventEmitter { const tier = entry.count >= 80 ? 3 : entry.count >= 50 ? 2 : entry.count >= 35 ? 1 : 0; const cooldownMs = [30000, 60000, 120000, 300000][tier]; entry.cooldownUntil = now + cooldownMs; - logger.warn(`Provider Circuit-Breaker: ${provider} ${entry.count} konsekutive Fehler, Cooldown ${cooldownMs / 1000}s`); + logger.warn(`Provider Circuit-Breaker: ${key} ${entry.count} konsekutive Fehler, Cooldown ${cooldownMs / 1000}s`); // Invalidate mega-debrid session on cooldown to force fresh login - if (provider === "megadebrid" && this.invalidateMegaSessionFn) { + if (key === "megadebrid" && this.invalidateMegaSessionFn) { try { this.invalidateMegaSessionFn(); } catch { /* ignore */ } } } - this.providerFailures.set(provider, entry); + this.providerFailures.set(key, entry); } private recordProviderSuccess(provider: string): void { - if (this.providerFailures.has(provider)) { - this.providerFailures.delete(provider); + const key = String(provider || "").trim() || "unknown"; + if (this.providerFailures.has(key)) { + this.providerFailures.delete(key); } } @@ -4248,7 +4328,8 @@ export class DownloadManager extends EventEmitter { } private getProviderCooldownRemaining(provider: string): number { - const entry = this.providerFailures.get(provider); + const key = String(provider || "").trim() || "unknown"; + const entry = this.providerFailures.get(key); if (!entry || entry.cooldownUntil <= 0) { return 0; } @@ -4262,6 +4343,248 @@ export class DownloadManager extends EventEmitter { return remaining; } + private isProviderConfigured(provider: DebridProvider): boolean { + if ((this.settings.disabledProviders || []).includes(provider)) { + return false; + } + if (provider === "realdebrid") { + return Boolean(this.settings.realDebridUseWebLogin || this.settings.token.trim()); + } + if (provider === "megadebrid") { + return Boolean(this.settings.megaLogin.trim() && this.settings.megaPassword.trim()); + } + if (provider === "bestdebrid") { + return Boolean(this.settings.bestDebridUseWebLogin || this.settings.bestToken.trim()); + } + if (provider === "alldebrid") { + return Boolean(this.settings.allDebridUseWebLogin || this.settings.allDebridToken.trim()); + } + if (provider === "ddownload") { + return Boolean(this.settings.ddownloadLogin.trim() && this.settings.ddownloadPassword.trim()); + } + if (provider === "onefichier") { + return Boolean(this.settings.oneFichierApiKey.trim()); + } + if (provider === "debridlink") { + return Boolean(this.settings.debridLinkApiKeys.trim()); + } + if (provider === "linksnappy") { + return Boolean(this.settings.linkSnappyLogin.trim() && this.settings.linkSnappyPassword.trim()); + } + return false; + } + + private getExpectedProviderForItem(item: DownloadItem): DebridProvider | null { + if (item.provider) { + return item.provider; + } + + const hosterKey = extractHosterKey(item.url); + const routing = this.settings.hosterRouting || {}; + const routedProvider = hosterKey ? routing[hosterKey] : undefined; + if (routedProvider && this.isProviderConfigured(routedProvider)) { + return routedProvider; + } + + const order = [ + this.settings.providerPrimary, + this.settings.providerSecondary !== "none" ? this.settings.providerSecondary : null, + this.settings.providerTertiary !== "none" ? this.settings.providerTertiary : null + ].filter(Boolean) as DebridProvider[]; + + const seen = new Set(); + for (const provider of order) { + if (seen.has(provider)) { + continue; + } + seen.add(provider); + if (this.isProviderConfigured(provider)) { + return provider; + } + } + + return null; + } + + private getProviderFailureKeyForItem(item: DownloadItem, providerOverride?: DebridProvider | string | null): string { + const provider = String(providerOverride || item.provider || this.getExpectedProviderForItem(item) || "unknown").trim() || "unknown"; + const hosterKey = extractHosterKey(item.url); + if (provider === "alldebrid" && hosterKey) { + return `${provider}:${hosterKey}`; + } + return provider; + } + + private getActiveTaskCountForFailureKey(failureKey: string, excludeItemId?: string): number { + let count = 0; + for (const active of this.activeTasks.values()) { + if (excludeItemId && active.itemId === excludeItemId) { + continue; + } + const activeItem = this.session.items[active.itemId]; + if (!activeItem) { + continue; + } + if (this.getProviderFailureKeyForItem(activeItem) === failureKey) { + count += 1; + } + } + return count; + } + + private getProviderActiveTaskCount(provider: DebridProvider): number { + let count = 0; + for (const active of this.activeTasks.values()) { + const activeItem = this.session.items[active.itemId]; + if (!activeItem) { + continue; + } + if (this.getExpectedProviderForItem(activeItem) === provider) { + count += 1; + } + } + return count; + } + + private getPacedStartKeyForItem(item: DownloadItem): string | null { + const provider = this.getExpectedProviderForItem(item); + if (provider !== "alldebrid") { + return null; + } + return provider; + } + + private reservePacedStartForItem(item: DownloadItem, now: number): boolean { + const paceKey = this.getPacedStartKeyForItem(item); + if (!paceKey) { + return false; + } + + const activeCount = this.getProviderActiveTaskCount("alldebrid"); + if (activeCount <= 0 && !this.providerStartReservations.has(paceKey)) { + return false; + } + + const baseDelayMs = activeCount * ALLDEBRID_START_STAGGER_MS; + const reservedAt = this.providerStartReservations.get(paceKey) || 0; + const earliestAt = Math.max(now + baseDelayMs, reservedAt); + if (earliestAt <= now) { + return false; + } + + const existingReadyAt = this.retryAfterByItem.get(item.id) || 0; + const scheduledAt = Math.max(existingReadyAt, earliestAt); + this.retryAfterByItem.set(item.id, scheduledAt); + this.providerStartReservations.set(paceKey, scheduledAt + ALLDEBRID_START_STAGGER_MS); + item.status = "queued"; + item.speedBps = 0; + item.fullStatus = `AllDebrid Start in ${Math.max(1, Math.ceil((scheduledAt - now) / 1000))}s`; + item.updatedAt = now; + return true; + } + + private getAllDebridStartLimit(hosterKey: string): number { + if (hosterKey !== "rapidgator") { + return Number.MAX_SAFE_INTEGER; + } + const cached = this.allDebridHostInfoCache.get(hosterKey); + const apiLimit = cached?.info.limitSimuDl; + if (Number.isFinite(apiLimit) && (apiLimit as number) > 0) { + return Math.max(1, Math.min(2, Math.floor(apiLimit as number))); + } + return 1; + } + + private shouldDelayStartForItem(item: DownloadItem): boolean { + const provider = this.getExpectedProviderForItem(item); + if (provider !== "alldebrid") { + return false; + } + const hosterKey = extractHosterKey(item.url); + if (hosterKey !== "rapidgator") { + return false; + } + const failureKey = this.getProviderFailureKeyForItem(item, provider); + const startLimit = this.getAllDebridStartLimit(hosterKey); + return this.getActiveTaskCountForFailureKey(failureKey) >= startLimit; + } + + private async getAllDebridHostInfoCached(hosterKey: string, signal?: AbortSignal, forceRefresh = false): Promise { + const normalizedHost = String(hosterKey || "").trim().toLowerCase(); + if (!normalizedHost || this.settings.allDebridUseWebLogin) { + return null; + } + const token = this.settings.allDebridToken.trim(); + if (!token) { + return null; + } + + const cached = this.allDebridHostInfoCache.get(normalizedHost); + if (!forceRefresh && cached && nowMs() - cached.cachedAt <= ALLDEBRID_HOST_INFO_TTL_MS) { + return cached.info; + } + + try { + const info = await fetchAllDebridHostInfo(token, normalizedHost, signal); + this.allDebridHostInfoCache.set(normalizedHost, { info, cachedAt: nowMs() }); + return info; + } catch (error) { + const errorText = compactErrorText(error); + logger.warn(`AllDebrid Host-Info Fehler für ${normalizedHost}: ${errorText}`); + return cached?.info || null; + } + } + + private async maybeApplyAllDebridRapidgatorBackoff(item: DownloadItem, active: ActiveTask): Promise { + const provider = this.getExpectedProviderForItem(item); + if (provider !== "alldebrid") { + return false; + } + + const hosterKey = extractHosterKey(item.url); + if (hosterKey !== "rapidgator") { + return false; + } + + const failureKey = this.getProviderFailureKeyForItem(item, provider); + const activePeers = this.getActiveTaskCountForFailureKey(failureKey, item.id); + const info = await this.getAllDebridHostInfoCached(hosterKey, active.abortController.signal, activePeers <= 0); + const startLimit = this.getAllDebridStartLimit(hosterKey); + + if (activePeers >= startLimit) { + const delayMs = Math.min(45000, 5000 + activePeers * 3000); + this.queueRetry(item, active, delayMs, `AllDebrid ${hosterKey}: Slot belegt (${activePeers}/${startLimit})`); + return true; + } + + if (!info) { + return false; + } + + if (info.state === "down") { + const delayMs = 60000; + this.applyProviderBusyBackoff(failureKey, delayMs); + this.queueRetry(item, active, delayMs + 1000, `AllDebrid ${info.host}: ${info.statusLabel}`); + return true; + } + + if (info.limitSimuDl !== null && info.limitSimuDl <= 0) { + const delayMs = 45000; + this.applyProviderBusyBackoff(failureKey, delayMs); + this.queueRetry(item, active, delayMs + 1000, `AllDebrid ${info.host}: keine freien Slots`); + return true; + } + + if (info.quota !== null && info.quota <= 0) { + const delayMs = 120000; + this.applyProviderBusyBackoff(failureKey, delayMs); + this.queueRetry(item, active, delayMs + 1000, `AllDebrid ${info.host}: Quota aufgebraucht`); + return true; + } + + return false; + } + private resetStaleRetryState(): void { const now = nowMs(); // Reset retry counters for items queued >10 min without progress @@ -4540,6 +4863,12 @@ export class DownloadManager extends EventEmitter { this.retryAfterByItem.delete(itemId); } if (item.status === "queued" || item.status === "reconnect_wait") { + if (this.reservePacedStartForItem(item, now)) { + continue; + } + if (this.shouldDelayStartForItem(item)) { + continue; + } return { packageId, itemId }; } } @@ -4741,7 +5070,7 @@ export class DownloadManager extends EventEmitter { throw new Error(`aborted:${active.abortReason}`); } // Check provider cooldown before attempting unrestrict - const cooldownProvider = item.provider || this.settings.providerPrimary || "unknown"; + const cooldownProvider = this.getProviderFailureKeyForItem(item); const cooldownMs = this.getProviderCooldownRemaining(cooldownProvider); if (cooldownMs > 0) { const delayMs = Math.min(cooldownMs + 1000, 310000); @@ -4750,6 +5079,11 @@ export class DownloadManager extends EventEmitter { this.emitState(); return; } + if (await this.maybeApplyAllDebridRapidgatorBackoff(item, active)) { + this.persistSoon(); + this.emitState(); + return; + } const unrestrictTimeoutSignal = AbortSignal.timeout(getUnrestrictTimeoutMs()); const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]); let unrestricted; @@ -4765,8 +5099,10 @@ export class DownloadManager extends EventEmitter { const errText = compactErrorText(unrestrictError); if (isUnrestrictFailure(errText)) { this.recordProviderFailure(cooldownProvider); - if (isProviderBusyUnrestrictError(errText)) { - const busyCooldownMs = Math.min(60000, 12000 + Number(active.unrestrictRetries || 0) * 3000); + if (isProviderBusyUnrestrictError(errText) || isTemporaryUnrestrictError(errText)) { + const busyCooldownMs = isTemporaryUnrestrictError(errText) + ? Math.min(180000, 20000 + Number(active.unrestrictRetries || 0) * 10000) + : Math.min(60000, 12000 + Number(active.unrestrictRetries || 0) * 3000); this.applyProviderBusyBackoff(cooldownProvider, busyCooldownMs); } } @@ -4776,7 +5112,7 @@ export class DownloadManager extends EventEmitter { throw new Error(`aborted:${active.abortReason}`); } // Unrestrict succeeded - reset provider failure counter - this.recordProviderSuccess(unrestricted.provider); + this.recordProviderSuccess(this.getProviderFailureKeyForItem(item, unrestricted.provider)); item.provider = unrestricted.provider; item.retries += unrestricted.retriesUsed; item.fileName = sanitizeFilename(unrestricted.fileName || filenameFromUrl(item.url)); @@ -4868,13 +5204,11 @@ export class DownloadManager extends EventEmitter { // file does not exist } } - const expectsNonEmptyFile = (item.totalBytes || 0) > 0 || isArchiveLikePath(finalTargetPath || item.fileName); - // Catch both empty files (0 B) and suspiciously small error-response files. - // A real archive part or video file should be at least 1 KB. - const tooSmall = expectsNonEmptyFile && ( - fileSizeOnDisk <= 0 - || fileSizeOnDisk < 512 - || (item.totalBytes && item.totalBytes > 10240 && fileSizeOnDisk < 1024) + const tooSmall = shouldRejectSuspiciousSmallDownload( + finalTargetPath, + item.fileName, + fileSizeOnDisk, + item.totalBytes ); if (tooSmall) { try { @@ -4889,7 +5223,7 @@ export class DownloadManager extends EventEmitter { item.totalBytes = (item.totalBytes || 0) > 0 ? item.totalBytes : null; item.speedBps = 0; item.updatedAt = nowMs(); - throw new Error(`Datei zu klein (${humanSize(fileSizeOnDisk)}, erwartet ${item.totalBytes ? humanSize(item.totalBytes) : ">1 KB"})`); + throw new Error(`Datei zu klein (${humanSize(fileSizeOnDisk)}, erwartet ${item.totalBytes ? humanSize(item.totalBytes) : ">= 1 MB"})`); } done = true; @@ -5022,11 +5356,10 @@ export class DownloadManager extends EventEmitter { } let stallDelayMs = retryDelayWithJitter(active.stallRetries, 200); // Respect provider cooldown - if (item.provider) { - const providerCooldown = this.getProviderCooldownRemaining(item.provider); - if (providerCooldown > stallDelayMs) { - stallDelayMs = providerCooldown + 1000; - } + const providerCooldownKey = this.getProviderFailureKeyForItem(item); + const providerCooldown = this.getProviderCooldownRemaining(providerCooldownKey); + if (providerCooldown > stallDelayMs) { + stallDelayMs = providerCooldown + 1000; } const retryText = wasValidating ? `Link-Umwandlung hing, Retry ${active.stallRetries}/${retryDisplayLimit}` @@ -5129,10 +5462,12 @@ export class DownloadManager extends EventEmitter { if (isUnrestrictFailure(errorText) && active.unrestrictRetries < maxUnrestrictRetries) { active.unrestrictRetries += 1; item.retries += 1; - const failureProvider = item.provider || this.settings.providerPrimary || "unknown"; + const failureProvider = this.getProviderFailureKeyForItem(item); this.recordProviderFailure(failureProvider); - if (isProviderBusyUnrestrictError(errorText)) { - const busyCooldownMs = Math.min(60000, 12000 + Number(active.unrestrictRetries || 0) * 3000); + if (isProviderBusyUnrestrictError(errorText) || isTemporaryUnrestrictError(errorText)) { + const busyCooldownMs = isTemporaryUnrestrictError(errorText) + ? Math.min(180000, 20000 + Number(active.unrestrictRetries || 0) * 10000) + : Math.min(60000, 12000 + Number(active.unrestrictRetries || 0) * 3000); this.applyProviderBusyBackoff(failureProvider, busyCooldownMs); } // Escalating backoff: 5s, 7.5s, 11s, 17s, 25s, 38s, ... up to 120s @@ -7320,6 +7655,7 @@ export class DownloadManager extends EventEmitter { this.runCompletedPackages.clear(); } this.retryAfterByItem.clear(); + this.providerStartReservations.clear(); this.retryStateByItem.clear(); this.reservedTargetPaths.clear(); this.claimedTargetPathByItem.clear(); diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index cb847c9..3957ba4 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -22,10 +22,26 @@ async function waitFor(predicate: () => boolean, timeoutMs = 15000): Promise { +async function removeDirWithRetries(dir: string): Promise { + let lastError: unknown = null; + for (let attempt = 1; attempt <= 5; attempt += 1) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + return; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, attempt * 80)); + } + } + if (lastError) { + throw lastError; + } +} + +afterEach(async () => { globalThis.fetch = originalFetch; for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true }); + await removeDirWithRetries(dir); } }); @@ -2819,6 +2835,344 @@ describe("download manager", () => { } }); + it("retries suspicious mini files under 1 MB until the full file arrives", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + const binary = Buffer.alloc(2 * 1024 * 1024, 21); + let directCalls = 0; + + const server = http.createServer((req, res) => { + if ((req.url || "") !== "/mini-retry") { + res.statusCode = 404; + res.end("not-found"); + return; + } + + directCalls += 1; + if (directCalls === 1) { + const tiny = Buffer.from("temporary error", "utf8"); + res.statusCode = 200; + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Length", String(tiny.length)); + res.end(tiny); + return; + } + + res.statusCode = 200; + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Length", String(binary.length)); + res.end(binary); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("server address unavailable"); + } + const directUrl = `http://127.0.0.1:${address.port}/mini-retry`; + + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes("/unrestrict/link")) { + return new Response( + JSON.stringify({ + download: directUrl, + filename: "mini-retry.part01.rar", + filesize: binary.length + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + } + return originalFetch(input, init); + }; + + try { + const manager = new DownloadManager( + { + ...defaultSettings(), + token: "rd-token", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: false, + autoReconnect: false + }, + emptySession(), + createStoragePaths(path.join(root, "state")) + ); + + manager.addPackages([{ name: "mini-retry", links: ["https://dummy/mini-retry"] }]); + await manager.start(); + await waitFor(() => !manager.getSnapshot().session.running, 30000); + + const item = Object.values(manager.getSnapshot().session.items)[0]; + expect(item?.status).toBe("completed"); + expect(directCalls).toBeGreaterThan(1); + expect(fs.existsSync(item.targetPath)).toBe(true); + expect(fs.statSync(item.targetPath).size).toBe(binary.length); + } finally { + server.close(); + await once(server, "close"); + } + }); + + it("limits AllDebrid rapidgator starts to one active task by default", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + const binary = Buffer.alloc(2 * 1024 * 1024, 6); + let unlockInFlight = 0; + let maxUnlockInFlight = 0; + + const server = http.createServer((req, res) => { + const route = req.url || ""; + if (route !== "/rg-1" && route !== "/rg-2") { + res.statusCode = 404; + res.end("not-found"); + return; + } + setTimeout(() => { + res.statusCode = 200; + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Length", String(binary.length)); + res.end(binary); + }, 1500); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("server address unavailable"); + } + + const link1 = "https://rapidgator.net/file/12345678901234567890123456789012/file1.mkv.html"; + const link2 = "https://rapidgator.net/file/abcdefabcdefabcdefabcdefabcdef12/file2.mkv.html"; + const directUrl1 = `http://127.0.0.1:${address.port}/rg-1`; + const directUrl2 = `http://127.0.0.1:${address.port}/rg-2`; + + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const method = String( + init?.method + || (typeof input === "string" || input instanceof URL ? "" : input.method || "") + ).toUpperCase(); + + if (url.includes("/user/hosts")) { + return new Response( + JSON.stringify({ + status: "success", + data: { + hosts: { + rapidgator: { + name: "Rapidgator", + status: true, + quota: 50, + quotaMax: 100, + quotaType: "traffic", + limitSimuDl: 1 + } + } + } + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + } + + if (url.includes("/link/unlock")) { + unlockInFlight += 1; + maxUnlockInFlight = Math.max(maxUnlockInFlight, unlockInFlight); + try { + await new Promise((resolve) => setTimeout(resolve, 120)); + const body = init?.body; + const bodyText = body instanceof URLSearchParams ? body.toString() : String(body || ""); + const originalLink = new URLSearchParams(bodyText).get("link") || ""; + const directUrl = originalLink === link2 ? directUrl2 : directUrl1; + const fileName = originalLink === link2 ? "rg-2.mkv" : "rg-1.mkv"; + return new Response( + JSON.stringify({ + status: "success", + data: { + link: directUrl, + filename: fileName, + filesize: binary.length + } + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + } finally { + unlockInFlight = Math.max(0, unlockInFlight - 1); + } + } + + if (url.startsWith("https://rapidgator.net/")) { + if (method === "HEAD") { + return new Response(null, { status: 200 }); + } + return new Response("Rapidgator", { + status: 200, + headers: { "Content-Type": "text/html" } + }); + } + + return originalFetch(input, init); + }; + + try { + const manager = new DownloadManager( + { + ...defaultSettings(), + allDebridToken: "ad-token", + providerPrimary: "alldebrid", + providerSecondary: "none", + providerTertiary: "none", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: false, + autoReconnect: false, + enableIntegrityCheck: false, + maxParallel: 2 + }, + emptySession(), + createStoragePaths(path.join(root, "state")) + ); + + manager.addPackages([{ name: "rg-all-debrid", links: [link1, link2] }]); + await manager.start(); + await waitFor(() => { + const items = Object.values(manager.getSnapshot().session.items); + return items.some((item) => item.status === "downloading") && maxUnlockInFlight >= 1; + }, 15000); + await new Promise((resolve) => setTimeout(resolve, 400)); + + const items = Object.values(manager.getSnapshot().session.items); + expect(items).toHaveLength(2); + expect(items.filter((item) => item.status === "downloading" || item.status === "completed")).toHaveLength(1); + expect(items.filter((item) => item.status === "queued" || item.status === "validating" || item.status === "reconnect_wait")).toHaveLength(1); + expect(maxUnlockInFlight).toBe(1); + manager.stop(); + await waitFor(() => !manager.getSnapshot().session.running, 15000); + } finally { + server.close(); + await once(server, "close"); + } + }, 35000); + + it("staggeres AllDebrid starts by 2.5 seconds per active download", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + const binary = Buffer.alloc(512 * 1024, 5); + + const server = http.createServer((req, res) => { + const route = req.url || ""; + if (route !== "/ad-1" && route !== "/ad-2" && route !== "/ad-3") { + res.statusCode = 404; + res.end("not-found"); + return; + } + setTimeout(() => { + res.statusCode = 200; + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Length", String(binary.length)); + res.end(binary); + }, 1800); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("server address unavailable"); + } + + const link1 = "https://host-a.example/file1.bin"; + const link2 = "https://host-b.example/file2.bin"; + const link3 = "https://host-c.example/file3.bin"; + const directUrl1 = `http://127.0.0.1:${address.port}/ad-1`; + const directUrl2 = `http://127.0.0.1:${address.port}/ad-2`; + const directUrl3 = `http://127.0.0.1:${address.port}/ad-3`; + + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes("/link/unlock")) { + const body = init?.body; + const bodyText = body instanceof URLSearchParams ? body.toString() : String(body || ""); + const originalLink = new URLSearchParams(bodyText).get("link") || ""; + const directUrl = originalLink === link2 ? directUrl2 : originalLink === link3 ? directUrl3 : directUrl1; + const fileName = originalLink === link2 ? "ad-2.bin" : originalLink === link3 ? "ad-3.bin" : "ad-1.bin"; + return new Response( + JSON.stringify({ + status: "success", + data: { + link: directUrl, + filename: fileName, + filesize: binary.length + } + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + } + return originalFetch(input, init); + }; + + try { + const manager = new DownloadManager( + { + ...defaultSettings(), + allDebridToken: "ad-token", + providerPrimary: "alldebrid", + providerSecondary: "none", + providerTertiary: "none", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: false, + autoReconnect: false, + enableIntegrityCheck: false, + maxParallel: 3 + }, + emptySession(), + createStoragePaths(path.join(root, "state")) + ); + + manager.addPackages([{ name: "ad-paced", links: [link1, link2, link3] }]); + await manager.start(); + + const managerInternals = manager as unknown as { retryAfterByItem: Map }; + await waitFor(() => managerInternals.retryAfterByItem.size >= 2, 5000); + + const now = Date.now(); + const readyTimes = [...managerInternals.retryAfterByItem.values()].sort((a, b) => a - b); + expect(readyTimes).toHaveLength(2); + const firstDelay = readyTimes[0] - now; + const secondDelay = readyTimes[1] - now; + expect(firstDelay).toBeGreaterThan(1500); + expect(firstDelay).toBeLessThan(4500); + expect(secondDelay).toBeGreaterThan(3500); + expect(secondDelay).toBeLessThan(7000); + expect(secondDelay - firstDelay).toBeGreaterThan(1500); + + manager.stop(); + await waitFor(() => !manager.getSnapshot().session.running, 15000); + } finally { + server.close(); + await once(server, "close"); + } + }, 20000); + it("creates extract directory only at extraction and marks items as Entpackt", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root);