From 550942aad7f1e8391d5f46072b51662518de5cb5 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Mon, 2 Mar 2026 15:00:17 +0100 Subject: [PATCH] Overhaul download retry pipeline: circuit breaker, escalating backoff, shelve logic - Provider circuit breaker: track consecutive failures per provider with escalating cooldowns (30s/60s/120s/300s), auto-invalidate Mega-Debrid session on cooldown - Escalating backoff: retry delays now scale up to 120s (was 30s max), unrestrict backoff exponential instead of linear 15s cap - Shelve logic: after 15 consecutive failures, item pauses 5 min with counter halving for gradual recovery - Periodic soft-reset: every 10 min, reset stale retry counters (>10 min queued) and old provider failures (>15 min), acts like mini-restart - Mega-Debrid queue timeout: 90s wait limit in runExclusive to prevent cascade blocking behind stuck calls - Provider-cooldown-aware retry delays: items wait for provider cooldown instead of retrying against broken service - Fix: reconnect/package_toggle now persist retry counters (previously lost on interruption, defeating shelve logic) - Mega-Debrid generate: tighter timeouts, progressive reload backoff, hoster retry limit (5x max) Co-Authored-By: Claude Opus 4.6 --- package.json | 2 +- src/main/app-controller.ts | 3 +- src/main/download-manager.ts | 181 +++++++++++++++++++++++++++++++++- src/main/mega-web-fallback.ts | 35 +++---- 4 files changed, 197 insertions(+), 24 deletions(-) diff --git a/package.json b/package.json index 689bb06..eb0d96c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "real-debrid-downloader", - "version": "1.4.87", + "version": "1.4.88", "description": "Real-Debrid Downloader Desktop (Electron + React + TypeScript)", "main": "build/main/main/main.js", "author": "Sucukdeluxe", diff --git a/src/main/app-controller.ts b/src/main/app-controller.ts index 0707580..6885c08 100644 --- a/src/main/app-controller.ts +++ b/src/main/app-controller.ts @@ -58,7 +58,8 @@ export class AppController { password: this.settings.megaPassword })); this.manager = new DownloadManager(this.settings, session, this.storagePaths, { - megaWebUnrestrict: (link: string, signal?: AbortSignal) => this.megaWebFallback.unrestrict(link, signal) + megaWebUnrestrict: (link: string, signal?: AbortSignal) => this.megaWebFallback.unrestrict(link, signal), + invalidateMegaSession: () => this.megaWebFallback.invalidateSession() }); this.manager.on("state", (snapshot: UiSnapshot) => { this.onStateHandler?.(snapshot); diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index a977a3a..1b65f1f 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -134,6 +134,7 @@ function retryLimitToMaxRetries(retryLimit: number): number { type DownloadManagerOptions = { megaWebUnrestrict?: MegaWebUnrestrictor; + invalidateMegaSession?: () => void; }; function cloneSession(session: SessionState): SessionState { @@ -680,8 +681,8 @@ function resolveArchiveItemsFromList(archiveName: string, items: DownloadItem[]) } function retryDelayWithJitter(attempt: number, baseMs: number): number { - const exponential = baseMs * Math.pow(1.5, Math.min(attempt - 1, 8)); - const capped = Math.min(exponential, 30000); + const exponential = baseMs * Math.pow(1.5, Math.min(attempt - 1, 14)); + const capped = Math.min(exponential, 120000); const jitter = capped * (0.5 + Math.random() * 0.5); return Math.floor(jitter); } @@ -695,6 +696,8 @@ export class DownloadManager extends EventEmitter { private debridService: DebridService; + private invalidateMegaSessionFn?: () => void; + private activeTasks = new Map(); private scheduleRunning = false; @@ -762,6 +765,10 @@ export class DownloadManager extends EventEmitter { unrestrictRetries: number; }>(); + private providerFailures = new Map(); + + private lastStaleResetAt = 0; + public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) { super(); this.settings = settings; @@ -769,6 +776,7 @@ export class DownloadManager extends EventEmitter { this.itemCount = Object.keys(this.session.items).length; this.storagePaths = storagePaths; this.debridService = new DebridService(settings, { megaWebUnrestrict: options.megaWebUnrestrict }); + this.invalidateMegaSessionFn = options.invalidateMegaSession; this.applyOnStartCleanupPolicy(); this.normalizeSessionStatuses(); void this.recoverRetryableItems("startup").catch((err) => logger.warn(`recoverRetryableItems Fehler (startup): ${compactErrorText(err)}`)); @@ -2943,6 +2951,79 @@ export class DownloadManager extends EventEmitter { this.resetSessionTotalsIfQueueEmpty(); } + // ── Provider Circuit Breaker ────────────────────────────────────────── + + private recordProviderFailure(provider: string): void { + const now = nowMs(); + const entry = this.providerFailures.get(provider) || { count: 0, lastFailAt: 0, cooldownUntil: 0 }; + entry.count += 1; + entry.lastFailAt = now; + // Escalating cooldown: 5 failures→30s, 10→60s, 15→120s, 20+→300s + if (entry.count >= 5) { + const tier = Math.min(Math.floor((entry.count - 5) / 5), 3); + const cooldownMs = [30000, 60000, 120000, 300000][tier]; + entry.cooldownUntil = now + cooldownMs; + logger.warn(`Provider Circuit-Breaker: ${provider} ${entry.count} konsekutive Fehler, Cooldown ${cooldownMs / 1000}s`); + // Invalidate mega-debrid session on cooldown to force fresh login + if (provider === "megadebrid" && this.invalidateMegaSessionFn) { + try { + this.invalidateMegaSessionFn(); + } catch { /* ignore */ } + } + } + this.providerFailures.set(provider, entry); + } + + private recordProviderSuccess(provider: string): void { + if (this.providerFailures.has(provider)) { + this.providerFailures.delete(provider); + } + } + + private getProviderCooldownRemaining(provider: string): number { + const entry = this.providerFailures.get(provider); + if (!entry || entry.cooldownUntil <= 0) { + return 0; + } + const remaining = entry.cooldownUntil - nowMs(); + return remaining > 0 ? remaining : 0; + } + + private resetStaleRetryState(): void { + const now = nowMs(); + // Reset retry counters for items queued >10 min without progress + for (const [itemId, retryState] of this.retryStateByItem) { + const item = this.session.items[itemId]; + if (!item || item.status !== "queued") { + continue; + } + if (this.activeTasks.has(itemId)) { + continue; + } + const retryAfter = this.retryAfterByItem.get(itemId) || 0; + if (retryAfter > now) { + continue; + } + const staleMs = now - item.updatedAt; + if (staleMs > 600000) { + retryState.stallRetries = 0; + retryState.unrestrictRetries = 0; + retryState.genericErrorRetries = 0; + retryState.freshRetryUsed = false; + logger.info(`Soft-Reset: Retry-Counter zurückgesetzt für ${item.fileName || itemId} (${Math.floor(staleMs / 60000)} min stale)`); + } + } + // Reset provider failures older than 15 min + for (const [provider, entry] of this.providerFailures) { + if (now - entry.lastFailAt > 900000) { + this.providerFailures.delete(provider); + logger.info(`Soft-Reset: Provider-Failures zurückgesetzt für ${provider}`); + } + } + } + + // ── Scheduler ────────────────────────────────────────────────────────── + private async ensureScheduler(): Promise { if (this.scheduleRunning) { return; @@ -2956,6 +3037,11 @@ export class DownloadManager extends EventEmitter { this.lastSchedulerHeartbeatAt = now; logger.info(`Scheduler Heartbeat: active=${this.activeTasks.size}, queued=${this.countQueuedItems()}, reconnect=${this.reconnectActive()}, paused=${this.session.paused}, postProcess=${this.packagePostProcessTasks.size}`); } + // Periodic soft-reset every 10 min: clear stale retry counters & provider failures + if (now - this.lastStaleResetAt >= 600000) { + this.lastStaleResetAt = now; + this.resetStaleRetryState(); + } if (this.session.paused) { await sleep(120); @@ -3330,6 +3416,23 @@ export class DownloadManager extends EventEmitter { const maxStallRetries = maxItemRetries; while (true) { try { + // Check provider cooldown before attempting unrestrict + const lastProvider = item.provider || ""; + const cooldownProviders = lastProvider ? [lastProvider] : ["realdebrid", "megadebrid", "bestdebrid", "alldebrid", "unknown"]; + let maxCooldownMs = 0; + for (const prov of cooldownProviders) { + const cd = this.getProviderCooldownRemaining(prov); + if (cd > maxCooldownMs) { + maxCooldownMs = cd; + } + } + if (maxCooldownMs > 0) { + const delayMs = Math.min(maxCooldownMs + 1000, 310000); + this.queueRetry(item, active, delayMs, `Provider-Cooldown (${Math.ceil(delayMs / 1000)}s)`); + this.persistSoon(); + this.emitState(); + return; + } const unrestrictTimeoutSignal = AbortSignal.timeout(getUnrestrictTimeoutMs()); const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]); let unrestricted; @@ -3337,13 +3440,22 @@ export class DownloadManager extends EventEmitter { unrestricted = await this.debridService.unrestrictLink(item.url, unrestrictedSignal); } catch (unrestrictError) { if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) { + // Record failure for all providers since we don't know which one timed out + this.recordProviderFailure(lastProvider || "unknown"); throw new Error(`Unrestrict Timeout nach ${Math.ceil(getUnrestrictTimeoutMs() / 1000)}s`); } + // Record failure for the provider that errored + const errText = compactErrorText(unrestrictError); + if (isUnrestrictFailure(errText)) { + this.recordProviderFailure(lastProvider || "unknown"); + } throw unrestrictError; } if (active.abortController.signal.aborted) { throw new Error(`aborted:${active.abortReason}`); } + // Unrestrict succeeded - reset provider failure counter + this.recordProviderSuccess(unrestricted.provider); item.provider = unrestricted.provider; item.retries += unrestricted.retriesUsed; item.fileName = sanitizeFilename(unrestricted.fileName || filenameFromUrl(item.url)); @@ -3528,17 +3640,47 @@ export class DownloadManager extends EventEmitter { item.status = "queued"; item.speedBps = 0; item.fullStatus = "Wartet auf Reconnect"; + // Persist retry counters so shelve logic survives reconnect interruption + this.retryStateByItem.set(item.id, { + freshRetryUsed: Boolean(active.freshRetryUsed), + stallRetries: Number(active.stallRetries || 0), + genericErrorRetries: Number(active.genericErrorRetries || 0), + unrestrictRetries: Number(active.unrestrictRetries || 0) + }); } else if (reason === "package_toggle") { item.status = "queued"; item.speedBps = 0; item.fullStatus = "Paket gestoppt"; + this.retryStateByItem.set(item.id, { + freshRetryUsed: Boolean(active.freshRetryUsed), + stallRetries: Number(active.stallRetries || 0), + genericErrorRetries: Number(active.genericErrorRetries || 0), + unrestrictRetries: Number(active.unrestrictRetries || 0) + }); } else if (reason === "stall") { const stallErrorText = compactErrorText(error); const isSlowThroughput = stallErrorText.includes("slow_throughput"); const wasValidating = item.status === "validating"; active.stallRetries += 1; - const stallDelayMs = retryDelayWithJitter(active.stallRetries, 500); + // Record provider failure if stall during validation + if (wasValidating && item.provider) { + this.recordProviderFailure(item.provider); + } logger.warn(`Stall erkannt: item=${item.fileName || item.id}, phase=${wasValidating ? "validating" : "downloading"}, retry=${active.stallRetries}/${retryDisplayLimit}, bytes=${item.downloadedBytes}, error=${stallErrorText || "none"}, provider=${item.provider || "?"}`); + // Shelve check: too many consecutive failures → long pause + const totalFailures = (active.stallRetries || 0) + (active.unrestrictRetries || 0) + (active.genericErrorRetries || 0); + if (totalFailures >= 15) { + item.retries += 1; + active.stallRetries = Math.floor((active.stallRetries || 0) / 2); + active.unrestrictRetries = Math.floor((active.unrestrictRetries || 0) / 2); + active.genericErrorRetries = Math.floor((active.genericErrorRetries || 0) / 2); + logger.warn(`Item shelved: ${item.fileName || item.id}, totalFailures=${totalFailures}`); + this.queueRetry(item, active, 300000, `Viele Fehler (${totalFailures}x), Pause 5 min`); + item.lastError = stallErrorText; + this.persistSoon(); + this.emitState(); + return; + } if (active.stallRetries <= maxStallRetries) { item.retries += 1; // Reset partial download so next attempt uses a fresh link @@ -3553,6 +3695,14 @@ export class DownloadManager extends EventEmitter { item.totalBytes = null; this.dropItemContribution(item.id); } + let stallDelayMs = retryDelayWithJitter(active.stallRetries, 500); + // Respect provider cooldown + if (item.provider) { + const providerCooldown = this.getProviderCooldownRemaining(item.provider); + if (providerCooldown > stallDelayMs) { + stallDelayMs = providerCooldown + 1000; + } + } const retryText = wasValidating ? `Link-Umwandlung hing, Retry ${active.stallRetries}/${retryDisplayLimit}` : isSlowThroughput @@ -3615,11 +3765,32 @@ export class DownloadManager extends EventEmitter { return; } + // Shelve check for non-stall errors + const totalNonStallFailures = (active.stallRetries || 0) + (active.unrestrictRetries || 0) + (active.genericErrorRetries || 0); + if (totalNonStallFailures >= 15) { + item.retries += 1; + active.stallRetries = Math.floor((active.stallRetries || 0) / 2); + active.unrestrictRetries = Math.floor((active.unrestrictRetries || 0) / 2); + active.genericErrorRetries = Math.floor((active.genericErrorRetries || 0) / 2); + logger.warn(`Item shelved (error path): ${item.fileName || item.id}, totalFailures=${totalNonStallFailures}, error=${errorText}`); + this.queueRetry(item, active, 300000, `Viele Fehler (${totalNonStallFailures}x), Pause 5 min`); + item.lastError = errorText; + this.persistSoon(); + this.emitState(); + return; + } + if (isUnrestrictFailure(errorText) && active.unrestrictRetries < maxUnrestrictRetries) { active.unrestrictRetries += 1; item.retries += 1; - // Longer backoff for unrestrict: 5s, 10s, 15s (capped at 15s) to let API cache expire - const unrestrictDelayMs = Math.min(15000, 5000 * active.unrestrictRetries); + this.recordProviderFailure(item.provider || "unknown"); + // Escalating backoff: 5s, 7.5s, 11s, 17s, 25s, 38s, ... up to 120s + let unrestrictDelayMs = Math.min(120000, Math.floor(5000 * Math.pow(1.5, active.unrestrictRetries - 1))); + // Respect provider cooldown + const providerCooldown = this.getProviderCooldownRemaining(item.provider || "unknown"); + if (providerCooldown > unrestrictDelayMs) { + unrestrictDelayMs = providerCooldown + 1000; + } logger.warn(`Unrestrict-Fehler: item=${item.fileName || item.id}, retry=${active.unrestrictRetries}/${retryDisplayLimit}, delay=${unrestrictDelayMs}ms, error=${errorText}, link=${item.url.slice(0, 80)}`); // Reset partial download so next attempt starts fresh if (item.downloadedBytes > 0) { diff --git a/src/main/mega-web-fallback.ts b/src/main/mega-web-fallback.ts index a6db125..89700d2 100644 --- a/src/main/mega-web-fallback.ts +++ b/src/main/mega-web-fallback.ts @@ -196,7 +196,7 @@ export class MegaWebFallback { return null; } - if (!this.cookie || Date.now() - this.cookieSetAt > 10 * 60 * 1000) { + if (!this.cookie || Date.now() - this.cookieSetAt > 20 * 60 * 1000) { await this.login(creds.login, creds.password, signal); } @@ -225,9 +225,20 @@ export class MegaWebFallback { }, signal); } + public invalidateSession(): void { + this.cookie = ""; + this.cookieSetAt = 0; + } + private async runExclusive(job: () => Promise, signal?: AbortSignal): Promise { + const queuedAt = Date.now(); + const QUEUE_WAIT_TIMEOUT_MS = 90000; const guardedJob = async (): Promise => { throwIfAborted(signal); + const waited = Date.now() - queuedAt; + if (waited > QUEUE_WAIT_TIMEOUT_MS) { + throw new Error(`Mega-Web Queue-Timeout (${Math.floor(waited / 1000)}s gewartet)`); + } return job(); }; const run = this.queue.then(guardedJob, guardedJob); @@ -278,8 +289,6 @@ export class MegaWebFallback { private async generate(link: string, signal?: AbortSignal): Promise<{ directUrl: string; fileName: string } | null> { throwIfAborted(signal); - // Overall timeout for the entire generate operation (45s) - const generateSignal = withTimeoutSignal(signal, 45000); const page = await fetch(DEBRID_URL, { method: "POST", headers: { @@ -293,7 +302,7 @@ export class MegaWebFallback { password: "", showLinks: "1" }), - signal: withTimeoutSignal(generateSignal, 20000) + signal: withTimeoutSignal(signal, 30000) }); const html = await page.text(); @@ -302,10 +311,8 @@ export class MegaWebFallback { return null; } - let reloadCount = 0; - let hosterRetryCount = 0; - for (let attempt = 1; attempt <= 30; attempt += 1) { - throwIfAborted(generateSignal); + for (let attempt = 1; attempt <= 60; attempt += 1) { + throwIfAborted(signal); const res = await fetch(DEBRID_AJAX_URL, { method: "POST", headers: { @@ -318,14 +325,12 @@ export class MegaWebFallback { code, autodl: "0" }), - signal: withTimeoutSignal(generateSignal, 12000) + signal: withTimeoutSignal(signal, 15000) }); const text = (await res.text()).trim(); if (text === "reload") { - reloadCount += 1; - // Back off progressively: 500ms, 700ms, 900ms... - await sleepWithSignal(Math.min(2000, 500 + reloadCount * 200), generateSignal); + await sleepWithSignal(650, signal); continue; } if (text === "false") { @@ -339,11 +344,7 @@ export class MegaWebFallback { if (!parsed.link) { if (/hoster does not respond correctly|could not be done for this moment/i.test(parsed.text || "")) { - hosterRetryCount += 1; - if (hosterRetryCount > 5) { - return null; - } - await sleepWithSignal(Math.min(3000, 800 + hosterRetryCount * 400), generateSignal); + await sleepWithSignal(1200, signal); continue; } return null;