fix: harden rotation recovery and support diagnostics

Persist resume recovery across restarts, reconcile lifecycle controls with authoritative state, and wait for locked partial files before clean restarts. Capture real account-attempt trails, redact historical account masks from support bundles, and keep all active telemetry on a stable 500 ms cadence.
2026-08-13 06:26:24 +02:00
parent e1b4708952
commit 88399c5dd0
15 changed files with 626 additions and 161 deletions
@@ -1735,6 +1735,33 @@ function retryDelayWithJitter(attempt: number, baseMs: number): number {
  return Math.floor(jitter);
 }

+function clearResumeRecoveryState(item: DownloadItem): void {
+  delete item.resumeLinkRenewalFailures;
+  delete item.resumeHardResetUsed;
+  delete item.resumeResetPending;
+}
+
+async function removeResumePartialForReset(targetPath: string): Promise<boolean> {
+  const retryableCodes = new Set(["EBUSY", "EPERM", "EACCES"]);
+  for (let attempt = 0; attempt < 4; attempt += 1) {
+    try {
+      fs.rmSync(targetPath, { force: true });
+    } catch (error) {
+      const code = String((error as NodeJS.ErrnoException)?.code || "");
+      if (!retryableCodes.has(code)) {
+        return false;
+      }
+    }
+    if (!fs.existsSync(targetPath)) {
+      return true;
+    }
+    if (attempt < 3) {
+      await sleep((attempt + 1) * 100);
+    }
+  }
+  return !fs.existsSync(targetPath);
+}
+
 function isMegaDebridProviderKey(value: string): boolean {
  return value === "megadebrid"
    || value === "megadebrid-api"
@@ -2083,9 +2110,9 @@ export class DownloadManager extends EventEmitter {
    });
  }

-  private logItemOnly(
-    item: DownloadItem,
-    level: "INFO" | "WARN" | "ERROR",
+  private logItemOnly(
+    item: DownloadItem,
+    level: "INFO" | "WARN" | "ERROR",
    message: string,
    fields?: Record<string, unknown>
  ): void {
@@ -2098,9 +2125,21 @@ export class DownloadManager extends EventEmitter {
      fileName: item.fileName,
      status: item.status,
      targetPath: item.targetPath,
-      ...fields
-    });
-  }
+      ...fields
+    });
+  }
+
+  private logRotationEventForItem(item: DownloadItem, event: RotationEvent): void {
+    this.logItemOnly(item, event.level, "Account-Rotation", {
+      provider: event.provider,
+      account: event.accountLabel,
+      event: event.event,
+      reason: event.reason,
+      category: event.category,
+      cooldownSec: event.cooldownSec,
+      next: event.next
+    });
+  }

  private collectRenameMatchTokensForItem(pkg: PackageEntry, item: DownloadItem): string[] {
    const tokens = new Set<string>();
@@ -5434,9 +5473,10 @@ export class DownloadManager extends EventEmitter {
      this.runOutcomes.delete(itemId);
      this.runItemIds.delete(itemId);
      this.retryAfterByItem.delete(itemId);
-      this.retryStateByItem.delete(itemId);
-
-      item.status = "queued";
+      this.retryStateByItem.delete(itemId);
+      clearResumeRecoveryState(item);
+
+      item.status = "queued";
      item.downloadedBytes = 0;
      item.totalBytes = null;
      item.progressPercent = 0;
@@ -5517,9 +5557,10 @@ export class DownloadManager extends EventEmitter {
      this.dropItemContribution(itemId);
      this.runOutcomes.delete(itemId);
      this.retryAfterByItem.delete(itemId);
-      this.retryStateByItem.delete(itemId);
-
-      item.status = "queued";
+      this.retryStateByItem.delete(itemId);
+      clearResumeRecoveryState(item);
+
+      item.status = "queued";
      item.downloadedBytes = 0;
      item.totalBytes = null;
      item.progressPercent = 0;
@@ -6238,9 +6279,10 @@ export class DownloadManager extends EventEmitter {
        item.updatedAt = nowMs();
        continue;
      }
-      if (item.status === "extracting" || item.status === "integrity_check") {
-        item.status = "completed";
-        item.fullStatus = `Fertig (${humanSize(item.downloadedBytes)})`;
+      if (item.status === "extracting" || item.status === "integrity_check") {
+        item.status = "completed";
+        clearResumeRecoveryState(item);
+        item.fullStatus = `Fertig (${humanSize(item.downloadedBytes)})`;
        item.speedBps = 0;
        item.updatedAt = nowMs();
      } else if (item.status === "downloading"
@@ -6529,11 +6571,11 @@ export class DownloadManager extends EventEmitter {
      return;
    }
    const itemCount = this.itemCount;
-    const emitDelay = this.session.running
-      ? itemCount >= 1500
-        ? 700
-        : itemCount >= 700
-          ? 500
+    const emitDelay = this.session.running
+      ? itemCount >= 1500
+        ? 500
+        : itemCount >= 700
+          ? 500
          : itemCount >= 250
            ? 300
            : 150
@@ -6986,8 +7028,9 @@ export class DownloadManager extends EventEmitter {
      error: errorText || undefined
    });

-    item.status = "completed";
-    item.fullStatus = this.settings.autoExtract
+    item.status = "completed";
+    clearResumeRecoveryState(item);
+    item.fullStatus = this.settings.autoExtract
      ? "Entpacken - Ausstehend"
      : `Fertig (${humanSize(diskState.size)})`;
    item.downloadedBytes = diskState.size;
@@ -9010,7 +9053,7 @@ export class DownloadManager extends EventEmitter {
    return count;
  }

-  private queueRetry(item: DownloadItem, active: ActiveTask, delayMs: number, statusText: string): void {
+  private queueRetry(item: DownloadItem, active: ActiveTask, delayMs: number, statusText: string): void {
    const waitMs = Math.max(0, Math.floor(delayMs));
    item.status = "queued";
    item.speedBps = 0;
@@ -9019,13 +9062,14 @@ export class DownloadManager extends EventEmitter {
    item.attempts = 0;
    active.abortController = new AbortController();
    active.abortReason = "none";
-    this.retryStateByItem.set(item.id, {
+    this.retryStateByItem.set(item.id, {
      freshRetryUsed: Boolean(active.freshRetryUsed),
      resumeHardResetUsed: Boolean(active.resumeHardResetUsed),
      stallRetries: Number(active.stallRetries || 0),
      genericErrorRetries: Number(active.genericErrorRetries || 0),
-      unrestrictRetries: Number(active.unrestrictRetries || 0)
-    });
+      unrestrictRetries: Number(active.unrestrictRetries || 0)
+    });
+    item.resumeHardResetUsed = Boolean(active.resumeHardResetUsed) || undefined;
    this.logPackageForItem(item, "WARN", "Retry eingeplant", {
      delayMs: waitMs,
      statusText,
@@ -9039,6 +9083,25 @@ export class DownloadManager extends EventEmitter {
    const pkg = this.session.packages[item.packageId];
    if (pkg) this.refreshPackageStatus(pkg);
  }
+
+  private async applyPendingResumeReset(item: DownloadItem, active: ActiveTask, targetPath: string): Promise<boolean> {
+    const removed = !targetPath || await removeResumePartialForReset(targetPath);
+    if (!removed) {
+      active.resumeHardResetUsed = false;
+      delete item.resumeHardResetUsed;
+      return false;
+    }
+    active.resumeHardResetUsed = true;
+    item.resumeHardResetUsed = true;
+    delete item.resumeResetPending;
+    this.releaseTargetPath(item.id);
+    this.dropItemContribution(item.id);
+    item.downloadedBytes = 0;
+    item.totalBytes = null;
+    item.progressPercent = 0;
+    item.speedBps = 0;
+    return true;
+  }

  private scheduleHttp416Retry(
    item: DownloadItem,
@@ -9199,19 +9262,32 @@ export class DownloadManager extends EventEmitter {
      return;
    }

-    const retryState = this.retryStateByItem.get(item.id) || {
-      freshRetryUsed: false,
-      resumeHardResetUsed: false,
-      stallRetries: 0,
-      genericErrorRetries: 0,
-      unrestrictRetries: 0
+    const retryState = this.retryStateByItem.get(item.id) || {
+      freshRetryUsed: false,
+      resumeHardResetUsed: Boolean(item.resumeHardResetUsed),
+      stallRetries: 0,
+      genericErrorRetries: Math.max(0, Number(item.resumeLinkRenewalFailures || 0)),
+      unrestrictRetries: 0
    };
    this.retryStateByItem.set(item.id, retryState);
    active.freshRetryUsed = retryState.freshRetryUsed;
    active.resumeHardResetUsed = retryState.resumeHardResetUsed;
    active.stallRetries = retryState.stallRetries;
-    active.genericErrorRetries = retryState.genericErrorRetries;
-    active.unrestrictRetries = retryState.unrestrictRetries;
+    active.genericErrorRetries = retryState.genericErrorRetries;
+    active.unrestrictRetries = retryState.unrestrictRetries;
+    if (item.resumeResetPending) {
+      const resetTargetPath = String(item.targetPath || "").trim();
+      const resetApplied = await this.applyPendingResumeReset(item, active, resetTargetPath);
+      this.queueRetry(
+        item,
+        active,
+        resetApplied ? 300 : 1000,
+        resetApplied ? "Resume-Fehler erkannt, kompletter Neuversuch" : "Warte auf Teildatei-Freigabe"
+      );
+      this.persistSoon();
+      this.emitState();
+      return;
+    }
    const configuredRetryLimit = normalizeRetryLimit(this.settings.retryLimit);
    const retryDisplayLimit = retryLimitLabel(configuredRetryLimit);
    const maxItemRetries = retryLimitToMaxRetries(configuredRetryLimit);
@@ -9291,30 +9367,33 @@ export class DownloadManager extends EventEmitter {
        const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]);
        let unrestricted;
        try {
-          unrestricted = await runWithConversionTrace(
-            {
-              itemId: item.id,
-              itemName: item.fileName || item.id,
-              link: item.url,
-              providerOrder: (this.settings.providerOrder || []).join(",") || String(this.getExpectedProviderForItem(item) || "?")
-            },
-            async () => {
-              traceConversionNote("slots", this.describeSlotOccupancy());
-              traceConversionNote("retry", Number(active.unrestrictRetries || 0));
-              try {
-                return await this.debridService.unrestrictLink(item.url, unrestrictedSignal, undefined, preferredLeadProvider);
-              } catch (innerError) {
-                if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) {
-                  traceConversionPhase({
-                    phase: "caller-timeout",
-                    outcome: "timeout",
-                    detail: `Caller-Budget ${Math.ceil(unrestrictTimeoutMs / 1000)}s erschoepft (siehe letzte Phase fuer in-flight Provider/Account)`
-                  });
-                }
-                throw innerError;
-              }
-            }
-          );
+          unrestricted = await runWithRotationItemSink(
+            (event) => this.logRotationEventForItem(item, event),
+            () => runWithConversionTrace(
+              {
+                itemId: item.id,
+                itemName: item.fileName || item.id,
+                link: item.url,
+                providerOrder: (this.settings.providerOrder || []).join(",") || String(this.getExpectedProviderForItem(item) || "?")
+              },
+              async () => {
+                traceConversionNote("slots", this.describeSlotOccupancy());
+                traceConversionNote("retry", Number(active.unrestrictRetries || 0));
+                try {
+                  return await this.debridService.unrestrictLink(item.url, unrestrictedSignal, undefined, preferredLeadProvider);
+                } catch (innerError) {
+                  if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) {
+                    traceConversionPhase({
+                      phase: "caller-timeout",
+                      outcome: "timeout",
+                      detail: `Caller-Budget ${Math.ceil(unrestrictTimeoutMs / 1000)}s erschoepft (siehe letzte Phase fuer in-flight Provider/Account)`
+                    });
+                  }
+                  throw innerError;
+                }
+              }
+            )
+          );
        } catch (unrestrictError) {
          if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) {
            this.recordProviderFailure(cooldownProvider);
@@ -9518,8 +9597,9 @@ export class DownloadManager extends EventEmitter {
        }

        const completedAt = nowMs();
-        item.status = "completed";
-        item.fullStatus = this.settings.autoExtract
+        item.status = "completed";
+        clearResumeRecoveryState(item);
+        item.fullStatus = this.settings.autoExtract
          ? "Entpacken - Ausstehend"
          : `Fertig (${humanSize(item.downloadedBytes)})`;
        item.progressPercent = 100;
@@ -9779,32 +9859,30 @@ export class DownloadManager extends EventEmitter {
              return;
            }
            if (isResumeHardResetReason(exhaustedReason, active.genericErrorRetries) && !active.resumeHardResetUsed) {
-              active.resumeHardResetUsed = true;
-              item.retries += 1;
-              logger.warn(`Resume-Neustart: item=${item.fileName || item.id}, error=${exhaustedReason}, provider=${item.provider || "?"}`);
-              const resetTargetPath = claimedTargetPath || String(item.targetPath || "").trim();
-              if (resetTargetPath) {
-                try {
-                  fs.rmSync(resetTargetPath, { force: true });
-                } catch {
-                }
-              }
-              this.releaseTargetPath(item.id);
-              this.dropItemContribution(item.id);
-              item.lastError = exhaustedReason;
-              item.downloadedBytes = 0;
-              item.totalBytes = null;
-              item.progressPercent = 0;
-              this.queueRetry(item, active, 300, "Resume-Fehler erkannt, kompletter Neuversuch");
+              item.retries += 1;
+              logger.warn(`Resume-Neustart: item=${item.fileName || item.id}, error=${exhaustedReason}, provider=${item.provider || "?"}`);
+              const resetTargetPath = claimedTargetPath || String(item.targetPath || "").trim();
+              item.resumeResetPending = true;
+              item.lastError = exhaustedReason;
+              const resetApplied = await this.applyPendingResumeReset(item, active, resetTargetPath);
+              this.queueRetry(
+                item,
+                active,
+                resetApplied ? 300 : 1000,
+                resetApplied ? "Resume-Fehler erkannt, kompletter Neuversuch" : "Warte auf Teildatei-Freigabe"
+              );
              this.persistSoon();
              this.emitState();
              return;
            }
          }
-          if (directLinkRetryMatch && active.genericErrorRetries < maxGenericErrorRetries) {
-            active.genericErrorRetries += 1;
+          if (directLinkRetryMatch && active.genericErrorRetries < maxGenericErrorRetries) {
+            active.genericErrorRetries += 1;
            item.retries += 1;
-            const exhaustedReason = compactErrorText(directLinkRetryMatch[1] || errorText).replace(/^Error:\s*/i, "");
+            const exhaustedReason = compactErrorText(directLinkRetryMatch[1] || errorText).replace(/^Error:\s*/i, "");
+            if (isResumeHardResetReason(exhaustedReason, 1)) {
+              item.resumeLinkRenewalFailures = active.genericErrorRetries;
+            }
            const refreshDelayMs = retryDelayWithJitter(active.genericErrorRetries, 200);
            logger.warn(
              `Direktlink erschöpft: item=${item.fileName || item.id}, ` +
@@ -11265,9 +11343,10 @@ export class DownloadManager extends EventEmitter {
    return recovered + finalized;
  }

-  private queueItemForRetry(item: DownloadItem, options: { hardReset: boolean; reason: string }): void {
-    this.retryStateByItem.delete(item.id);
-    const targetPath = String(item.targetPath || "").trim();
+  private queueItemForRetry(item: DownloadItem, options: { hardReset: boolean; reason: string }): void {
+    this.retryStateByItem.delete(item.id);
+    clearResumeRecoveryState(item);
+    const targetPath = String(item.targetPath || "").trim();
    if (options.hardReset && targetPath) {
      try {
        fs.rmSync(targetPath, { force: true });
@@ -12213,8 +12292,9 @@ export class DownloadManager extends EventEmitter {
            continue;
          }
          logger.info(`Item-Recovery: ${item.fileName} war "${item.status}" aber Datei existiert (${humanSize(stat.size)}), setze auf completed`);
-          item.status = "completed";
-          item.fullStatus = this.settings.autoExtract ? "Entpacken - Ausstehend" : `Fertig (${humanSize(stat.size)})`;
+          item.status = "completed";
+          clearResumeRecoveryState(item);
+          item.fullStatus = this.settings.autoExtract ? "Entpacken - Ausstehend" : `Fertig (${humanSize(stat.size)})`;
          item.downloadedBytes = stat.size;
          item.progressPercent = 100;
          item.speedBps = 0;