Overhaul download retry pipeline: circuit breaker, escalating backoff, shelve logic

- Provider circuit breaker: track consecutive failures per provider with
  escalating cooldowns (30s/60s/120s/300s), auto-invalidate Mega-Debrid
  session on cooldown
- Escalating backoff: retry delays now scale up to 120s (was 30s max),
  unrestrict backoff exponential instead of linear 15s cap
- Shelve logic: after 15 consecutive failures, item pauses 5 min with
  counter halving for gradual recovery
- Periodic soft-reset: every 10 min, reset stale retry counters (>10 min
  queued) and old provider failures (>15 min), acts like mini-restart
- Mega-Debrid queue timeout: 90s wait limit in runExclusive to prevent
  cascade blocking behind stuck calls
- Provider-cooldown-aware retry delays: items wait for provider cooldown
  instead of retrying against broken service
- Fix: reconnect/package_toggle now persist retry counters (previously
  lost on interruption, defeating shelve logic)
- Mega-Debrid generate: tighter timeouts, progressive reload backoff,
  hoster retry limit (5x max)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "real-debrid-downloader",
-  "version": "1.4.87",
+  "version": "1.4.88",
   "description": "Real-Debrid Downloader Desktop (Electron + React + TypeScript)",
   "main": "build/main/main/main.js",
   "author": "Sucukdeluxe",

src/main/app-controller.ts

@@ -58,7 +58,8 @@ export class AppController {
       password: this.settings.megaPassword
     }));
     this.manager = new DownloadManager(this.settings, session, this.storagePaths, {
-      megaWebUnrestrict: (link: string, signal?: AbortSignal) => this.megaWebFallback.unrestrict(link, signal)
+      megaWebUnrestrict: (link: string, signal?: AbortSignal) => this.megaWebFallback.unrestrict(link, signal),
+      invalidateMegaSession: () => this.megaWebFallback.invalidateSession()
     });
     this.manager.on("state", (snapshot: UiSnapshot) => {
       this.onStateHandler?.(snapshot);

src/main/download-manager.ts

@@ -134,6 +134,7 @@ function retryLimitToMaxRetries(retryLimit: number): number {
 
 type DownloadManagerOptions = {
   megaWebUnrestrict?: MegaWebUnrestrictor;
+  invalidateMegaSession?: () => void;
 };
 
 function cloneSession(session: SessionState): SessionState {
@@ -680,8 +681,8 @@ function resolveArchiveItemsFromList(archiveName: string, items: DownloadItem[])
 }
 
 function retryDelayWithJitter(attempt: number, baseMs: number): number {
-  const exponential = baseMs * Math.pow(1.5, Math.min(attempt - 1, 8));
+  const exponential = baseMs * Math.pow(1.5, Math.min(attempt - 1, 14));
-  const capped = Math.min(exponential, 30000);
+  const capped = Math.min(exponential, 120000);
   const jitter = capped * (0.5 + Math.random() * 0.5);
   return Math.floor(jitter);
 }
@@ -695,6 +696,8 @@ export class DownloadManager extends EventEmitter {
 
   private debridService: DebridService;
 
+  private invalidateMegaSessionFn?: () => void;
+
   private activeTasks = new Map<string, ActiveTask>();
 
   private scheduleRunning = false;
@@ -762,6 +765,10 @@ export class DownloadManager extends EventEmitter {
     unrestrictRetries: number;
   }>();
 
+  private providerFailures = new Map<string, { count: number; lastFailAt: number; cooldownUntil: number }>();
+
+  private lastStaleResetAt = 0;
+
   public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) {
     super();
     this.settings = settings;
@@ -769,6 +776,7 @@ export class DownloadManager extends EventEmitter {
     this.itemCount = Object.keys(this.session.items).length;
     this.storagePaths = storagePaths;
     this.debridService = new DebridService(settings, { megaWebUnrestrict: options.megaWebUnrestrict });
+    this.invalidateMegaSessionFn = options.invalidateMegaSession;
     this.applyOnStartCleanupPolicy();
     this.normalizeSessionStatuses();
     void this.recoverRetryableItems("startup").catch((err) => logger.warn(`recoverRetryableItems Fehler (startup): ${compactErrorText(err)}`));
@@ -2943,6 +2951,79 @@ export class DownloadManager extends EventEmitter {
     this.resetSessionTotalsIfQueueEmpty();
   }
 
+  // ── Provider Circuit Breaker ──────────────────────────────────────────
+
+  private recordProviderFailure(provider: string): void {
+    const now = nowMs();
+    const entry = this.providerFailures.get(provider) || { count: 0, lastFailAt: 0, cooldownUntil: 0 };
+    entry.count += 1;
+    entry.lastFailAt = now;
+    // Escalating cooldown: 5 failures→30s, 10→60s, 15→120s, 20+→300s
+    if (entry.count >= 5) {
+      const tier = Math.min(Math.floor((entry.count - 5) / 5), 3);
+      const cooldownMs = [30000, 60000, 120000, 300000][tier];
+      entry.cooldownUntil = now + cooldownMs;
+      logger.warn(`Provider Circuit-Breaker: ${provider} ${entry.count} konsekutive Fehler, Cooldown ${cooldownMs / 1000}s`);
+      // Invalidate mega-debrid session on cooldown to force fresh login
+      if (provider === "megadebrid" && this.invalidateMegaSessionFn) {
+        try {
+          this.invalidateMegaSessionFn();
+        } catch { /* ignore */ }
+      }
+    }
+    this.providerFailures.set(provider, entry);
+  }
+
+  private recordProviderSuccess(provider: string): void {
+    if (this.providerFailures.has(provider)) {
+      this.providerFailures.delete(provider);
+    }
+  }
+
+  private getProviderCooldownRemaining(provider: string): number {
+    const entry = this.providerFailures.get(provider);
+    if (!entry || entry.cooldownUntil <= 0) {
+      return 0;
+    }
+    const remaining = entry.cooldownUntil - nowMs();
+    return remaining > 0 ? remaining : 0;
+  }
+
+  private resetStaleRetryState(): void {
+    const now = nowMs();
+    // Reset retry counters for items queued >10 min without progress
+    for (const [itemId, retryState] of this.retryStateByItem) {
+      const item = this.session.items[itemId];
+      if (!item || item.status !== "queued") {
+        continue;
+      }
+      if (this.activeTasks.has(itemId)) {
+        continue;
+      }
+      const retryAfter = this.retryAfterByItem.get(itemId) || 0;
+      if (retryAfter > now) {
+        continue;
+      }
+      const staleMs = now - item.updatedAt;
+      if (staleMs > 600000) {
+        retryState.stallRetries = 0;
+        retryState.unrestrictRetries = 0;
+        retryState.genericErrorRetries = 0;
+        retryState.freshRetryUsed = false;
+        logger.info(`Soft-Reset: Retry-Counter zurückgesetzt für ${item.fileName || itemId} (${Math.floor(staleMs / 60000)} min stale)`);
+      }
+    }
+    // Reset provider failures older than 15 min
+    for (const [provider, entry] of this.providerFailures) {
+      if (now - entry.lastFailAt > 900000) {
+        this.providerFailures.delete(provider);
+        logger.info(`Soft-Reset: Provider-Failures zurückgesetzt für ${provider}`);
+      }
+    }
+  }
+
+  // ── Scheduler ──────────────────────────────────────────────────────────
+
   private async ensureScheduler(): Promise<void> {
     if (this.scheduleRunning) {
       return;
@@ -2956,6 +3037,11 @@ export class DownloadManager extends EventEmitter {
           this.lastSchedulerHeartbeatAt = now;
           logger.info(`Scheduler Heartbeat: active=${this.activeTasks.size}, queued=${this.countQueuedItems()}, reconnect=${this.reconnectActive()}, paused=${this.session.paused}, postProcess=${this.packagePostProcessTasks.size}`);
         }
+        // Periodic soft-reset every 10 min: clear stale retry counters & provider failures
+        if (now - this.lastStaleResetAt >= 600000) {
+          this.lastStaleResetAt = now;
+          this.resetStaleRetryState();
+        }
 
         if (this.session.paused) {
           await sleep(120);
@@ -3330,6 +3416,23 @@ export class DownloadManager extends EventEmitter {
     const maxStallRetries = maxItemRetries;
     while (true) {
       try {
+        // Check provider cooldown before attempting unrestrict
+        const lastProvider = item.provider || "";
+        const cooldownProviders = lastProvider ? [lastProvider] : ["realdebrid", "megadebrid", "bestdebrid", "alldebrid", "unknown"];
+        let maxCooldownMs = 0;
+        for (const prov of cooldownProviders) {
+          const cd = this.getProviderCooldownRemaining(prov);
+          if (cd > maxCooldownMs) {
+            maxCooldownMs = cd;
+          }
+        }
+        if (maxCooldownMs > 0) {
+          const delayMs = Math.min(maxCooldownMs + 1000, 310000);
+          this.queueRetry(item, active, delayMs, `Provider-Cooldown (${Math.ceil(delayMs / 1000)}s)`);
+          this.persistSoon();
+          this.emitState();
+          return;
+        }
         const unrestrictTimeoutSignal = AbortSignal.timeout(getUnrestrictTimeoutMs());
         const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]);
         let unrestricted;
@@ -3337,13 +3440,22 @@ export class DownloadManager extends EventEmitter {
           unrestricted = await this.debridService.unrestrictLink(item.url, unrestrictedSignal);
         } catch (unrestrictError) {
           if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) {
+            // Record failure for all providers since we don't know which one timed out
+            this.recordProviderFailure(lastProvider || "unknown");
             throw new Error(`Unrestrict Timeout nach ${Math.ceil(getUnrestrictTimeoutMs() / 1000)}s`);
           }
+          // Record failure for the provider that errored
+          const errText = compactErrorText(unrestrictError);
+          if (isUnrestrictFailure(errText)) {
+            this.recordProviderFailure(lastProvider || "unknown");
+          }
           throw unrestrictError;
         }
         if (active.abortController.signal.aborted) {
           throw new Error(`aborted:${active.abortReason}`);
         }
+        // Unrestrict succeeded - reset provider failure counter
+        this.recordProviderSuccess(unrestricted.provider);
         item.provider = unrestricted.provider;
         item.retries += unrestricted.retriesUsed;
         item.fileName = sanitizeFilename(unrestricted.fileName || filenameFromUrl(item.url));
@@ -3528,17 +3640,47 @@ export class DownloadManager extends EventEmitter {
           item.status = "queued";
           item.speedBps = 0;
           item.fullStatus = "Wartet auf Reconnect";
+          // Persist retry counters so shelve logic survives reconnect interruption
+          this.retryStateByItem.set(item.id, {
+            freshRetryUsed: Boolean(active.freshRetryUsed),
+            stallRetries: Number(active.stallRetries || 0),
+            genericErrorRetries: Number(active.genericErrorRetries || 0),
+            unrestrictRetries: Number(active.unrestrictRetries || 0)
+          });
         } else if (reason === "package_toggle") {
           item.status = "queued";
           item.speedBps = 0;
           item.fullStatus = "Paket gestoppt";
+          this.retryStateByItem.set(item.id, {
+            freshRetryUsed: Boolean(active.freshRetryUsed),
+            stallRetries: Number(active.stallRetries || 0),
+            genericErrorRetries: Number(active.genericErrorRetries || 0),
+            unrestrictRetries: Number(active.unrestrictRetries || 0)
+          });
         } else if (reason === "stall") {
           const stallErrorText = compactErrorText(error);
           const isSlowThroughput = stallErrorText.includes("slow_throughput");
           const wasValidating = item.status === "validating";
           active.stallRetries += 1;
-          const stallDelayMs = retryDelayWithJitter(active.stallRetries, 500);
+          // Record provider failure if stall during validation
+          if (wasValidating && item.provider) {
+            this.recordProviderFailure(item.provider);
+          }
           logger.warn(`Stall erkannt: item=${item.fileName || item.id}, phase=${wasValidating ? "validating" : "downloading"}, retry=${active.stallRetries}/${retryDisplayLimit}, bytes=${item.downloadedBytes}, error=${stallErrorText || "none"}, provider=${item.provider || "?"}`);
+          // Shelve check: too many consecutive failures → long pause
+          const totalFailures = (active.stallRetries || 0) + (active.unrestrictRetries || 0) + (active.genericErrorRetries || 0);
+          if (totalFailures >= 15) {
+            item.retries += 1;
+            active.stallRetries = Math.floor((active.stallRetries || 0) / 2);
+            active.unrestrictRetries = Math.floor((active.unrestrictRetries || 0) / 2);
+            active.genericErrorRetries = Math.floor((active.genericErrorRetries || 0) / 2);
+            logger.warn(`Item shelved: ${item.fileName || item.id}, totalFailures=${totalFailures}`);
+            this.queueRetry(item, active, 300000, `Viele Fehler (${totalFailures}x), Pause 5 min`);
+            item.lastError = stallErrorText;
+            this.persistSoon();
+            this.emitState();
+            return;
+          }
           if (active.stallRetries <= maxStallRetries) {
             item.retries += 1;
             // Reset partial download so next attempt uses a fresh link
@@ -3553,6 +3695,14 @@ export class DownloadManager extends EventEmitter {
               item.totalBytes = null;
               this.dropItemContribution(item.id);
             }
+            let stallDelayMs = retryDelayWithJitter(active.stallRetries, 500);
+            // Respect provider cooldown
+            if (item.provider) {
+              const providerCooldown = this.getProviderCooldownRemaining(item.provider);
+              if (providerCooldown > stallDelayMs) {
+                stallDelayMs = providerCooldown + 1000;
+              }
+            }
             const retryText = wasValidating
               ? `Link-Umwandlung hing, Retry ${active.stallRetries}/${retryDisplayLimit}`
               : isSlowThroughput
@@ -3615,11 +3765,32 @@ export class DownloadManager extends EventEmitter {
             return;
           }
 
+          // Shelve check for non-stall errors
+          const totalNonStallFailures = (active.stallRetries || 0) + (active.unrestrictRetries || 0) + (active.genericErrorRetries || 0);
+          if (totalNonStallFailures >= 15) {
+            item.retries += 1;
+            active.stallRetries = Math.floor((active.stallRetries || 0) / 2);
+            active.unrestrictRetries = Math.floor((active.unrestrictRetries || 0) / 2);
+            active.genericErrorRetries = Math.floor((active.genericErrorRetries || 0) / 2);
+            logger.warn(`Item shelved (error path): ${item.fileName || item.id}, totalFailures=${totalNonStallFailures}, error=${errorText}`);
+            this.queueRetry(item, active, 300000, `Viele Fehler (${totalNonStallFailures}x), Pause 5 min`);
+            item.lastError = errorText;
+            this.persistSoon();
+            this.emitState();
+            return;
+          }
+
           if (isUnrestrictFailure(errorText) && active.unrestrictRetries < maxUnrestrictRetries) {
             active.unrestrictRetries += 1;
             item.retries += 1;
-            // Longer backoff for unrestrict: 5s, 10s, 15s (capped at 15s) to let API cache expire
+            this.recordProviderFailure(item.provider || "unknown");
-            const unrestrictDelayMs = Math.min(15000, 5000 * active.unrestrictRetries);
+            // Escalating backoff: 5s, 7.5s, 11s, 17s, 25s, 38s, ... up to 120s
+            let unrestrictDelayMs = Math.min(120000, Math.floor(5000 * Math.pow(1.5, active.unrestrictRetries - 1)));
+            // Respect provider cooldown
+            const providerCooldown = this.getProviderCooldownRemaining(item.provider || "unknown");
+            if (providerCooldown > unrestrictDelayMs) {
+              unrestrictDelayMs = providerCooldown + 1000;
+            }
             logger.warn(`Unrestrict-Fehler: item=${item.fileName || item.id}, retry=${active.unrestrictRetries}/${retryDisplayLimit}, delay=${unrestrictDelayMs}ms, error=${errorText}, link=${item.url.slice(0, 80)}`);
             // Reset partial download so next attempt starts fresh
             if (item.downloadedBytes > 0) {

src/main/mega-web-fallback.ts

@@ -196,7 +196,7 @@ export class MegaWebFallback {
         return null;
       }
 
-      if (!this.cookie || Date.now() - this.cookieSetAt > 10 * 60 * 1000) {
+      if (!this.cookie || Date.now() - this.cookieSetAt > 20 * 60 * 1000) {
         await this.login(creds.login, creds.password, signal);
       }
 
@@ -225,9 +225,20 @@ export class MegaWebFallback {
     }, signal);
   }
 
+  public invalidateSession(): void {
+    this.cookie = "";
+    this.cookieSetAt = 0;
+  }
+
   private async runExclusive<T>(job: () => Promise<T>, signal?: AbortSignal): Promise<T> {
+    const queuedAt = Date.now();
+    const QUEUE_WAIT_TIMEOUT_MS = 90000;
     const guardedJob = async (): Promise<T> => {
       throwIfAborted(signal);
+      const waited = Date.now() - queuedAt;
+      if (waited > QUEUE_WAIT_TIMEOUT_MS) {
+        throw new Error(`Mega-Web Queue-Timeout (${Math.floor(waited / 1000)}s gewartet)`);
+      }
       return job();
     };
     const run = this.queue.then(guardedJob, guardedJob);
@@ -278,8 +289,6 @@ export class MegaWebFallback {
 
   private async generate(link: string, signal?: AbortSignal): Promise<{ directUrl: string; fileName: string } | null> {
     throwIfAborted(signal);
-    // Overall timeout for the entire generate operation (45s)
-    const generateSignal = withTimeoutSignal(signal, 45000);
     const page = await fetch(DEBRID_URL, {
       method: "POST",
       headers: {
@@ -293,7 +302,7 @@ export class MegaWebFallback {
         password: "",
         showLinks: "1"
       }),
-      signal: withTimeoutSignal(generateSignal, 20000)
+      signal: withTimeoutSignal(signal, 30000)
     });
 
     const html = await page.text();
@@ -302,10 +311,8 @@ export class MegaWebFallback {
       return null;
     }
 
-    let reloadCount = 0;
+    for (let attempt = 1; attempt <= 60; attempt += 1) {
-    let hosterRetryCount = 0;
+      throwIfAborted(signal);
-    for (let attempt = 1; attempt <= 30; attempt += 1) {
-      throwIfAborted(generateSignal);
       const res = await fetch(DEBRID_AJAX_URL, {
         method: "POST",
         headers: {
@@ -318,14 +325,12 @@ export class MegaWebFallback {
           code,
           autodl: "0"
         }),
-        signal: withTimeoutSignal(generateSignal, 12000)
+        signal: withTimeoutSignal(signal, 15000)
       });
 
       const text = (await res.text()).trim();
       if (text === "reload") {
-        reloadCount += 1;
+        await sleepWithSignal(650, signal);
-        // Back off progressively: 500ms, 700ms, 900ms...
-        await sleepWithSignal(Math.min(2000, 500 + reloadCount * 200), generateSignal);
         continue;
       }
       if (text === "false") {
@@ -339,11 +344,7 @@ export class MegaWebFallback {
 
       if (!parsed.link) {
         if (/hoster does not respond correctly|could not be done for this moment/i.test(parsed.text || "")) {
-          hosterRetryCount += 1;
+          await sleepWithSignal(1200, signal);
-          if (hosterRetryCount > 5) {
-            return null;
-          }
-          await sleepWithSignal(Math.min(3000, 800 + hosterRetryCount * 400), generateSignal);
           continue;
         }
         return null;