From 3bd6e3b23e15f1300e3cfeecac3809cbe77f3777 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Wed, 17 Jun 2026 04:22:20 +0200 Subject: [PATCH] =?UTF-8?q?H=C3=A4rtung=20Download/Rotation=20(Audit-Batch?= =?UTF-8?q?=201):=20Scheduler-Freeze,=20Cooldown-Respekt,=20Daily-Reset,?= =?UTF-8?q?=20Kategorisierung?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aus einem adversarisch verifizierten Multi-Agent-Audit (14 confirmed/11 refuted): #1 HIGH Scheduler-Freeze: findNextQueuedItem hatte keinen activeTasks-Guard. Wird ein Item zurueckgesetzt/ueberschrieben, waehrend sein alter Task noch in einem nicht-abbrechbaren await parkt (z.B. Integritaets-Check), liefert findNextQueuedItem dasselbe Item, startItem lehnt es ab ohne activeTasks zu verkleinern → der SYNCHRONE Admission-Loop dreht endlos → Event-Loop friert permanent ein. Fix: `if (this.activeTasks.has(itemId)) continue;`. Repro-Test (ohne Fix haengt sogar der vitest-Timeout — Freeze bewiesen). #2/#3 MED mega_debrid_cooldown: wurde verworfen: kein Parser (nur das debrid_link-Analogon) → Item lief in den generischen 5s-Exponential-Backoff und fragte cooled Accounts im Sekundentakt erneut an. Neuer parseMegaDebridCooldownRetry (nimmt das frueheste Cooldown-Ende ueber alle Accounts) + Handler VOR transient/generic → Item wartet die echte Cooldown-Zeit. #7/#8 MED Mega per-Account Tages-Usage wurde am Tageswechsel nie zurueckgesetzt (ensureProviderDailyUsageFresh ruecksetzte nur provider/debrid-link), und weil es den Tagesschluessel zuerst weiterstellte, lief auch der Reset in addMegaDebridAccountDailyUsageBytes ins Leere → Accounts blieben den ganzen Tag faelschlich "am Limit" und schrumpften das (neue) Pro-Account-Umwandlungslimit. Fix: megaDebridAccountDailyUsageBytes im Tageswechsel mit zuruecksetzen. #14 LOW classifyAccountFailure: rate_limit-Branch vor quota (quota matchte "limit" in "rate limit" → Fehl-Kategorisierung). 852/852 gruen, tsc unveraendert (6). #4 (empty→until-restart) bewusst deferred. --- src/main/debrid.ts | 18 ++++---- src/main/download-manager.ts | 33 ++++++++++++++ tasks/audit-loop.md | 72 +++++++++++++++++++++++++++++ tests/debrid.test.ts | 36 +++++++++++++++ tests/download-manager.test.ts | 82 ++++++++++++++++++++++++++++++++++ tests/unrestrict-retry.test.ts | 31 ++++++++++++- 6 files changed, 262 insertions(+), 10 deletions(-) create mode 100644 tasks/audit-loop.md diff --git a/src/main/debrid.ts b/src/main/debrid.ts index 00c5d8e..5fb5402 100644 --- a/src/main/debrid.ts +++ b/src/main/debrid.ts @@ -2183,6 +2183,15 @@ class MegaDebridClient { return { fatal: true, cooldownMs: 0, message: errorText, category: "skip" }; } + if (/rate.?limit|too.?many|429/i.test(errorText)) { + return { + fatal: false, + cooldownMs: MEGA_DEBRID_ACCOUNT_COOLDOWN_MS, + message: `Rate-Limit (${errorText})`, + category: "rate_limit" + }; + } + if (/quota|limit|exceeded|bandwidth/i.test(errorText)) { return { fatal: false, @@ -2202,15 +2211,6 @@ class MegaDebridClient { }; } - if (/rate.?limit|too.?many|429/i.test(errorText)) { - return { - fatal: false, - cooldownMs: MEGA_DEBRID_ACCOUNT_COOLDOWN_MS, - message: `Rate-Limit (${errorText})`, - category: "rate_limit" - }; - } - if (/antwort\s+leer|empty\s+response|leere\s+antwort/i.test(errorText)) { return { fatal: false, diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index a432331..01dcee7 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -645,6 +645,20 @@ function parseDebridLinkCooldownRetry(errorText: string): { delayMs: number; det return { delayMs, detail }; } +export function parseMegaDebridCooldownRetry(errorText: string): { delayMs: number; detail: string } | null { + const text = String(errorText || ""); + const matches = [...text.matchAll(/mega_debrid_cooldown:(\d+)/gi)]; + if (matches.length === 0) { + return null; + } + const delays = matches.map((m) => Number(m[1])).filter((n) => Number.isFinite(n) && n > 0); + if (delays.length === 0) { + return null; + } + const delayMs = Math.max(1000, Math.min(15 * 60 * 1000, Math.min(...delays))); + return { delayMs, detail: text.replace(/mega_debrid_cooldown:\d+:/i, "").trim() }; +} + function parseDebridLinkTerminalFailure(errorText: string): { kind: "invalid_all" | "no_active_key"; detail: string } | null { const raw = String(errorText || ""); const match = raw.match(/debrid_link_(invalid_all|no_active_key):(.*)$/i); @@ -7764,6 +7778,7 @@ export class DownloadManager extends EventEmitter { this.settings.providerDailyUsageDay = currentDay; this.settings.providerDailyUsageBytes = {}; this.settings.debridLinkApiKeyDailyUsageBytes = {}; + this.settings.megaDebridAccountDailyUsageBytes = {}; this.statsCache = null; this.statsCacheAt = 0; if (persist) { @@ -8474,6 +8489,7 @@ export class DownloadManager extends EventEmitter { const retryAfter = this.retryAfterByItem.get(itemId) || 0; if (retryAfter > now) continue; if (item.status !== "queued" && item.status !== "reconnect_wait") continue; + if (this.activeTasks.has(itemId)) continue; if (this.delayPacedStartForItem(item, now)) continue; if (this.shouldDelayStartForItem(item)) continue; @@ -9349,6 +9365,23 @@ export class DownloadManager extends EventEmitter { return; } + const megaCooldownRetry = parseMegaDebridCooldownRetry(errorText); + if (megaCooldownRetry && active.unrestrictRetries < maxUnrestrictRetries) { + active.unrestrictRetries += 1; + item.retries += 1; + logger.warn(`Mega-Debrid Account-Cooldown: item=${item.fileName || item.id}, retry=${active.unrestrictRetries}/${retryDisplayLimit}, delay=${megaCooldownRetry.delayMs}ms, link=${item.url.slice(0, 80)}`); + this.queueRetry( + item, + active, + megaCooldownRetry.delayMs, + `Mega-Debrid Cooldown, neuer Versuch in ${Math.ceil(megaCooldownRetry.delayMs / 1000)}s` + ); + item.lastError = megaCooldownRetry.detail || errorText; + this.persistSoon(); + this.emitState(); + return; + } + if (isMegaDebridTransientResolveFailure(errorText) && active.unrestrictRetries < maxUnrestrictRetries) { active.unrestrictRetries += 1; item.retries += 1; diff --git a/tasks/audit-loop.md b/tasks/audit-loop.md new file mode 100644 index 0000000..4be9259 --- /dev/null +++ b/tasks/audit-loop.md @@ -0,0 +1,72 @@ +# Autonomer Audit-Loop — Download/Fehler/Rotation (Goal 2026-06-17, 8h) + +Disziplin: erst BELEGEN (Code-Zitat + konkretes Szenario), dann adversarisch verifizieren, +dann TDD-Fix. Kein Blind-Fix. Tests gruen + tsc=6 nach jeder Runde. Periodisch releasen. + +## Runde 1 (laeuft) +- Discover+Verify-Workflow ueber 7 Subsysteme (scheduler-slots, unrestrict-retry, mega-rotation, + classify-cooldown, mega-web-token, provider-chain-timeout, account-availability). + +### Meine unabhaengigen Verdachtsfaelle (Cross-Check gegen Workflow) +1. **Web-Selbst-Cooldown (Analog zum API-214-Bug, HOCH):** Caller-Timeout = 60s + (`DEFAULT_UNRESTRICT_TIMEOUT_MS`, download-manager 114) umschliesst die GANZE Kette. + Mega-Web braucht legitim laenger (per-Account-Queue bis 90s + Login + Generate). + Feuert die 60s nach >=8s (`MEGA_DEBRID_ABORT_MIN_RUN_MS_DEFAULT`=8000), setzt die + Rotation `aborted:debrid` → 120s Account-Cooldown (debrid.ts ~2037-2048), obwohl der + Account GESUND ist — die App hat aufgegeben. → Kaskade ueber Accounts. Live im jf.zip + belegt: `Mega-Debrid Web | TIMEOUT_COOLDOWN | reason=aborted:debrid | cooldownSec=120`. + Fix-Kandidat: (a) per-Provider-Timeout statt globaler 60s; und/oder (b) Caller-Timeout- + Abort NICHT als Account-Cooldown werten (EMA-Demotion regelt langsame Accounts bereits), + oder nur sehr kurz. +2. **Globaler 60s-Timeout kappt Failover (Advisor-bewiesen, HOCH):** download-manager 8759 + `AbortSignal.any([cancel, timeout])` → bei Provider1-Verbrauch des Budgets abortet das + Signal → debrid.ts 3805 `signal.aborted` → throw, kein nextProvider. Fix: per-Provider- + AbortSignal.timeout, Stop nur bei USER-Cancel. +3. **Exponential-Backoff bis 120s** (generic unrestrict retry) — Item sitzt bis 2 min. + Pruefen ob fuer haeufige transiente Faelle zu lang. + +## Bestaetigte Bugs (Workflow R1: 14 confirmed / 11 refuted) — priorisiert +- [IN ARBEIT] #1 HIGH Scheduler-Freeze: findNextQueuedItem ohne activeTasks-Guard → synchroner + Admission-Loop dreht endlos wenn ein reset/overwrite-Item noch im activeTasks parkt (non-abort- + observing await, z.B. Integrity-Check). Fix: `if (this.activeTasks.has(itemId)) continue;`. TDD-Test + (Freeze-Repro mit non-abort Mock + resetItems) geschrieben. +- #2/#3 MED mega_debrid_cooldown: Delay verworfen — kein Parser (nur debrid_link_cooldown). Fix: + Parser fuer beide Praefixe, queueRetry mit echtem delayMs. (Erklaert Rapid-Retry-trotz-Cooldown im jf.zip.) +- #7/#8 MED Mega per-Account Daily-Usage wird am Tagesgrenze NIE resettet → Accounts faelschlich "am + Limit" → schrumpft MEIN neues serialized-limit. Fix: megaDebridAccountDailyUsageBytes in + ensureProviderDailyUsageFresh resetten. +- #4 MED transiente leere Web-Antwort → permanenter until-restart-Park (limitSignal vom generischen + "antwort leer"). Fix: limitSignal nur vom echten Daily-Limit (NO_SERVER_RE). +- #5 MED Web echte Bad-Credentials erreichen invalid-Branch nicht (werden ewig retried). Fix: echte + Web-Login-Fehlerphrasen in invalid-Branch. +- #6 MED onefichier/ddownload-Routing ignoriert autoProviderFallback=off. Fix: Guard in catch. +- #14 LOW Regex-Ordering classify: quota-Branch shadowt rate_limit. Fix: rate_limit vor quota. +- #9 LOW overwrite wipet frisch geclaimten targetPath via altem .finally. +- #10 LOW HTTP416 shared counter mit genericErrorRetries. +- #11 LOW fresh-retry preempt typed transient handlers. +- #12 LOW 15-failure-shelve + shared counters → mehr Retries als retryLimit. +- #13 LOW self-poison: queue-wait zaehlt zu elapsedMs → abort-cooldown (Analog zu meinem Web-Verdacht #1). + +## Refutiert / Nicht-Bug (11) — nicht anfassen +providerStartReservations dead-state; debrid_link_cooldown cleanup; supprimé-fallthrough; mega-web 180s +aborts whole rotation; EMA-removed-premise; quota-no-park asymmetry; connectApi single-flight cancel-couple; +per-account queue chain-break (NON-BUG); mega-web slot-hold (NON-BUG); provider abort-vs-timeout heuristic; +daily-limit aggregate early-exit. + +## Fixes (TDD, mit Test + Release) +### Batch 1 → v1.7.215 (Suite laeuft) +- [x] #1 HIGH Scheduler-Freeze: `findNextQueuedItem` activeTasks-Guard. Repro-Test (ohne Fix haengt der + Event-Loop so hart, dass nicht mal vitest-Timeout feuert = Freeze empirisch bewiesen). Mit Fix 288ms. +- [x] #2/#3 MED parseMegaDebridCooldownRetry (export) + Handler VOR transient/generic branch → Item wartet + den ECHTEN Cooldown (min ueber alle Accounts) statt 5s-Busy-Loop. 5 Parser-Tests. +- [x] #7/#8 MED megaDebridAccountDailyUsageBytes Reset in ensureProviderDailyUsageFresh (laeuft via + getSnapshot, also auch im Stall). Test: Tagesgrenze → leer. +- [x] #14 LOW rate_limit-Branch VOR quota (quota matchte "limit" in "rate limit"). Test: rate_limit-Kategorie. +- [deferred] #4 empty-response→until-restart-park: 3-consecutive-streak ist reale Mitigation gegen transiente + Blips; Mega-empty-Semantik nicht sicher verifizierbar → kein Blind-Change. + +### Batch 2 (geplant, naechste Runden) +- Web-Timeout-Selbstcooldown-Familie: #13 elapsedMs inkl. queue-wait → ranLongEnough-Gate auf WORK-Zeit + (mega-web traced workMs schon); + 60s-Caller-Timeout vs Web-Queue(90s) Mismatch. Braucht workMs-Threading. +- #5 Web echte Bad-Creds erreichen invalid nie. #6 onefichier/ddownload routing ignoriert fallback=off. +- #9 overwrite targetPath-wipe. #10 HTTP416 shared counter. #11 fresh-retry preempt. #12 shelve+shared counter. diff --git a/tests/debrid.test.ts b/tests/debrid.test.ts index 844b2ce..4e05558 100644 --- a/tests/debrid.test.ts +++ b/tests/debrid.test.ts @@ -1417,6 +1417,42 @@ describe("debrid service", () => { } }); + it("categorizes a Mega-Debrid 'rate limit' error as rate_limit, not quota (regex ordering)", async () => { + const settings = { + ...defaultSettings(), + token: "", + bestToken: "", + allDebridToken: "", + megaLogin: "user", + megaPassword: "pass", + megaCredentials: "user:pass", + megaDebridApiEnabled: true, + megaDebridWebEnabled: false, + providerPrimary: "megadebrid-api" as const, + providerSecondary: "none" as const, + providerTertiary: "none" as const, + autoProviderFallback: true + }; + + globalThis.fetch = (async (input: RequestInfo | URL): Promise => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes("action=connectUser")) { + return new Response(JSON.stringify({ response_code: "ok", token: "tok", vip_end: Math.floor(Date.now() / 1000) + 999999 }), { status: 200, headers: { "Content-Type": "application/json" } }); + } + if (url.includes("action=getLink")) { + return new Response(JSON.stringify({ response_code: "error", response_text: "Rate limit exceeded, too many requests" }), { status: 200, headers: { "Content-Type": "application/json" } }); + } + return new Response("not-found", { status: 404 }); + }) as typeof fetch; + + const service = new DebridService(settings); + await service.unrestrictLink("https://rapidgator.net/file/rl.rar.html").then(() => null, (e: unknown) => e); + + const cooldown = getMegaDebridAccountCooldownState(`${getMegaDebridAccountId("user")}:api`); + expect(cooldown).not.toBeNull(); + expect(cooldown!.category).toBe("rate_limit"); + }); + it("uses Mega Web only when it is configured as a separate fallback provider", async () => { const settings = { ...defaultSettings(), diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index d59c1f0..17f60f5 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -6721,6 +6721,88 @@ describe("download manager", () => { await new Promise((resolve) => setTimeout(resolve, 150)); }); + it("resets Mega-Debrid per-account daily usage at the day boundary (not just provider/debrid-link)", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + const acctId = getMegaDebridAccountId("mega-user"); + + const manager = new DownloadManager( + { + ...defaultSettings(), + megaCredentials: "mega-user:mega-pass", + providerDailyUsageDay: "2000-01-01", + providerDailyUsageBytes: { megadebrid: 9_000_000_000 } as Record, + megaDebridAccountDailyUsageBytes: { [acctId]: 9_000_000_000 } as Record, + megaDebridAccountDailyLimitBytes: { [acctId]: 1_000_000_000 } as Record + }, + emptySession(), + createStoragePaths(path.join(root, "state")), + {} + ); + + const snap = manager.getSnapshot(); + expect(snap.settings.providerDailyUsageDay).not.toBe("2000-01-01"); + expect(snap.settings.megaDebridAccountDailyUsageBytes || {}).toEqual({}); + expect(snap.settings.providerDailyUsageBytes || {}).toEqual({}); + }); + + it("does not freeze the scheduler when a reset item's old task is parked in a non-abort-observing await", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); + tempDirs.push(root); + + let convCalls = 0; + const manager = new DownloadManager( + { + ...defaultSettings(), + megaCredentials: "mega-user:mega-pass", + megaDebridWebEnabled: true, + megaDebridApiEnabled: false, + megaDebridPreferApi: false, + providerOrder: [], + providerPrimary: "megadebrid", + providerSecondary: "none", + providerTertiary: "none", + outputDir: path.join(root, "downloads"), + extractDir: path.join(root, "extract"), + autoExtract: false, + autoReconnect: false, + enableIntegrityCheck: false, + maxParallel: 4 + }, + emptySession(), + createStoragePaths(path.join(root, "state")), + { + megaWebUnrestrict: vi.fn(async (): Promise => { + convCalls += 1; + return await new Promise(() => { /* never settles, ignores abort */ }); + }) + } + ); + + manager.addPackages([{ + name: "freeze-repro", + links: [ + "https://rapidgator.net/file/freeze-1.part1.rar.html", + "https://rapidgator.net/file/freeze-2.part2.rar.html", + "https://rapidgator.net/file/freeze-3.part3.rar.html" + ] + }]); + + await manager.start(); + await waitFor(() => convCalls === 1, 10000); + + const validatingItem = Object.values(manager.getSnapshot().session.items).find((i) => i.status === "validating"); + expect(validatingItem).toBeTruthy(); + + manager.resetItems([validatingItem!.id]); + + await waitFor(() => convCalls === 2, 5000); + expect(convCalls).toBe(2); + + manager.stop(); + await new Promise((resolve) => setTimeout(resolve, 150)); + }, 20000); + it("serializes Mega-Debrid API conversions to one per account (no single-token hammering)", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-")); tempDirs.push(root); diff --git a/tests/unrestrict-retry.test.ts b/tests/unrestrict-retry.test.ts index 36dc973..dac77f0 100644 --- a/tests/unrestrict-retry.test.ts +++ b/tests/unrestrict-retry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { transientResolveRetryDelayMs } from "../src/main/download-manager"; +import { transientResolveRetryDelayMs, parseMegaDebridCooldownRetry } from "../src/main/download-manager"; describe("transientResolveRetryDelayMs (fast, bounded retry for transient resolve failures)", () => { it("starts fast (<= 3s) instead of the 5s..120s exponential", () => { @@ -29,3 +29,32 @@ describe("transientResolveRetryDelayMs (fast, bounded retry for transient resolv } }); }); + +describe("parseMegaDebridCooldownRetry (honor the encoded account-cooldown delay)", () => { + it("parses the encoded delay from a bare mega_debrid_cooldown error", () => { + const r = parseMegaDebridCooldownRetry("mega_debrid_cooldown:20330:Mega-Debrid (Account 2/2, Da******el): Token error"); + expect(r).not.toBeNull(); + expect(r!.delayMs).toBe(20330); + expect(r!.detail).toContain("Mega-Debrid"); + }); + + it("parses it when embedded in the aggregated provider-chain error", () => { + const aggregated = "Unrestrict fehlgeschlagen: Mega-Debrid API: mega_debrid_cooldown:20330:Mega-Debrid (Account 2/2): Token error"; + expect(parseMegaDebridCooldownRetry(aggregated)!.delayMs).toBe(20330); + }); + + it("takes the SOONEST (min) cooldown when several accounts are cooled", () => { + const both = "Mega-Debrid API: mega_debrid_cooldown:116285:web | Mega-Debrid API: mega_debrid_cooldown:20330:api"; + expect(parseMegaDebridCooldownRetry(both)!.delayMs).toBe(20330); + }); + + it("clamps to [1s, 15min]", () => { + expect(parseMegaDebridCooldownRetry("mega_debrid_cooldown:1:x")!.delayMs).toBe(1000); + expect(parseMegaDebridCooldownRetry("mega_debrid_cooldown:99999999:x")!.delayMs).toBe(15 * 60 * 1000); + }); + + it("returns null when there is no mega cooldown marker", () => { + expect(parseMegaDebridCooldownRetry("Datei beim Hoster gerade nicht abrufbar")).toBeNull(); + expect(parseMegaDebridCooldownRetry("debrid_link_cooldown:5000:x")).toBeNull(); + }); +});