fix(notifications): harden stall monitor lifecycle

2026-08-22 07:37:40 +02:00
parent 311475c93b
commit b3f60eada8
4 changed files with 309 additions and 71 deletions
@@ -22,6 +22,7 @@ import { serializeRealDebridApiAccounts } from "../src/shared/real-debrid-accoun
 import { getRenameLogPath, initRenameLog, shutdownRenameLog } from "../src/main/rename-log";
 import { UnrestrictedLink } from "../src/main/realdebrid";
 import { resetVideoToolingCache } from "../src/main/video-processor";
+import { createDownloadHealthState, evaluateDownloadHealth } from "../src/main/download-health-monitor";
 import type { AppSettings, DownloadItem, HistoryEntry, PackageEntry } from "../src/shared/types";

 const tempDirs: string[] = [];
@@ -14698,7 +14699,7 @@ describe("package lifecycle telemetry boundaries", () => {
 });

 describe("download health snapshot", () => {
-  function createHealthManager(root: string) {
+  function createHealthManager(root: string, settings = defaultSettings()) {
    const session = emptySession();
    const packageId = "private-package-id";
    const itemId = "private-item-id";
@@ -14741,7 +14742,7 @@ describe("download health snapshot", () => {
      createdAt: 2_000,
      updatedAt: now
    };
-    const manager = new DownloadManager(defaultSettings(), session, createStoragePaths(path.join(root, "state")));
+    const manager = new DownloadManager(settings, session, createStoragePaths(path.join(root, "state")));
    const state = manager as any;
    session.running = true;
    session.items[itemId].status = "downloading";
@@ -14909,6 +14910,91 @@ describe("download health snapshot", () => {
    expect(health.nextRetryAt).toBe(0);
  });

+  it("freezes health while two active downloads wait in the real global speed-limit queue", async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(10_000);
+    try {
+      const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-global-throttle-"));
+      tempDirs.push(root);
+      const settings = {
+        ...defaultSettings(),
+        speedLimitEnabled: true,
+        speedLimitMode: "global" as const,
+        speedLimitKbps: 1
+      };
+      const { manager, session, state, packageId, itemId } = createHealthManager(root, settings);
+      const secondItemId = "private-throttled-item-2";
+      session.items[secondItemId] = {
+        ...session.items[itemId],
+        id: secondItemId
+      };
+      session.packages[packageId].itemIds.push(secondItemId);
+      state.runItemIds.add(secondItemId);
+      const firstActive = {
+        itemId,
+        packageId,
+        abortController: new AbortController(),
+        abortReason: "none",
+        resumable: true,
+        nonResumableCounted: false,
+        phase: "downloading",
+        phaseStartedAt: 10_000,
+        phaseDeadlineAt: 0,
+        blockedOnDiskWrite: false,
+        blockedOnDiskSince: 0,
+        blockedOnThrottleUntil: 0
+      };
+      const secondActive = {
+        ...firstActive,
+        itemId: secondItemId,
+        abortController: new AbortController()
+      };
+      state.activeTasks.set(itemId, firstActive);
+      state.activeTasks.set(secondItemId, secondActive);
+
+      await state.applySpeedLimit(100 * 1024, 0, 10_000, firstActive);
+      const secondWait = state.applySpeedLimit(100 * 1024, 0, 10_000, secondActive);
+      const firstWait = state.applySpeedLimit(100 * 1024, 0, 10_000, firstActive);
+      const waitsSettled = Promise.allSettled([secondWait, firstWait]);
+      await Promise.resolve();
+
+      let healthState = createDownloadHealthState();
+      const events = [];
+      for (const now of [10_000, 55_000, 105_000]) {
+        vi.setSystemTime(now);
+        const result = evaluateDownloadHealth(
+          healthState,
+          manager.getDownloadHealthSnapshot(now),
+          now,
+          {
+            stallAfterMs: 90_000,
+            cooldownMs: 600_000,
+            notifyOnStall: true,
+            notifyOnRecovery: true
+          }
+        );
+        healthState = result.state;
+        events.push(...result.events);
+      }
+
+      expect(firstActive.blockedOnThrottleUntil).toBeGreaterThan(105_000);
+      expect(secondActive.blockedOnThrottleUntil).toBeGreaterThan(105_000);
+      expect(manager.getDownloadHealthSnapshot(105_000).blockedOnThrottleUntil).toBeGreaterThan(105_000);
+      expect(healthState.status).toBe("expected_wait");
+      expect(healthState.suspiciousDurationMs).toBe(0);
+      expect(events).toEqual([]);
+
+      firstActive.abortController.abort("test-finished");
+      secondActive.abortController.abort("test-finished");
+      await vi.runAllTimersAsync();
+      await waitsSettled;
+      expect(firstActive.blockedOnThrottleUntil).toBe(0);
+      expect(secondActive.blockedOnThrottleUntil).toBe(0);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
  it("counts one technical recovery when the existing global watchdog restarts stalled tasks", () => {
    const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-watchdog-"));
    tempDirs.push(root);
@@ -1,3 +1,6 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
 import { describe, expect, it, vi } from "vitest";

 const electron = vi.hoisted(() => {
@@ -33,6 +36,13 @@ vi.mock("electron", () => ({
 }));

 import { AppController } from "../src/main/app-controller";
+import {
+  DownloadHealthMonitor,
+  createDownloadHealthState,
+  loadDownloadHealthState,
+  saveDownloadHealthState,
+  type DownloadHealthSnapshot
+} from "../src/main/download-health-monitor";

 function deferred(): { promise: Promise<void>; resolve: () => void } {
  let resolve = () => {};
@@ -97,4 +107,116 @@ describe("main shutdown lifecycle", () => {
    expect(cleanup).toHaveBeenCalledTimes(1);
    expect(onError).not.toHaveBeenCalled();
  });
+
+  it("waits for a running health sample and persistently closes an alerted incident before shutdown returns", async () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-shutdown-"));
+    try {
+      const filePath = path.join(root, "health.json");
+      const runFingerprint = "a".repeat(64);
+      const queueFingerprint = "b".repeat(64);
+      saveDownloadHealthState(filePath, createDownloadHealthState({
+        status: "alerted",
+        runFingerprint,
+        queueFingerprint,
+        suspiciousDurationMs: 90_000,
+        suspiciousSamples: 3,
+        incidentStartedAt: 10_000,
+        alertedAt: 90_000,
+        lastAlertAt: 90_000,
+        cooldownUntil: 690_000
+      }));
+      let shuttingDown = false;
+      const healthSnapshot = (completionSequence: number): DownloadHealthSnapshot => ({
+        runActive: true,
+        runFingerprint,
+        queueFingerprint,
+        openItems: 1,
+        openPackages: 1,
+        knownDownloadedBytes: 4096,
+        activeTasks: 1,
+        startableItems: 0,
+        lastSchedulerTickAt: 100_000,
+        downloadProgressSequence: 0,
+        itemCompletionSequence: completionSequence,
+        lastPositiveByteAt: 0,
+        technicalRecoveryCount: 0,
+        paused: false,
+        reconnectUntil: 0,
+        nextRetryAt: 0,
+        providerCooldownUntil: 0,
+        blockedOnDisk: false,
+        blockedOnThrottleUntil: 0,
+        activePhaseDeadlineAt: 0,
+        terminalFailure: false,
+        manualStop: false,
+        shuttingDown,
+        currentSpeedBps: 0
+      });
+      const runningEvaluation = deferred();
+      const controller = Object.create(AppController.prototype) as any;
+      controller.downloadHealthTimer = setInterval(() => {}, 60_000);
+      controller.downloadHealthTimer.unref?.();
+      controller.downloadHealthMonitor = new DownloadHealthMonitor(filePath);
+      controller.downloadHealthEvaluation = runningEvaluation.promise.finally(() => {
+        controller.downloadHealthEvaluation = null;
+      });
+      controller.runtimeStatsTimer = null;
+      controller.notificationOutbox = {
+        enqueue: vi.fn(async () => undefined),
+        drainForShutdown: vi.fn(async () => undefined)
+      };
+      controller.manager = {
+        suspendDownloadHealthMonitoring: vi.fn(() => { shuttingDown = true; }),
+        getDownloadHealthSnapshot: vi.fn(() => healthSnapshot(1)),
+        flushNotificationsForShutdown: vi.fn(async () => undefined),
+        prepareForShutdown: vi.fn()
+      };
+      controller.megaWebFallback = { dispose: vi.fn() };
+      controller.realDebridWebFallbacks = new Map();
+      controller.pendingRealDebridWebAccountIds = new Map();
+      controller.allDebridWebFallback = { dispose: vi.fn() };
+      controller.bestDebridWebFallback = { dispose: vi.fn() };
+      controller.shutdownLogStorage = vi.fn();
+      controller.audit = vi.fn();
+      controller.settings = {
+        historyRetentionMode: "never",
+        notifyUrl: "https://discord.example.test/webhook",
+        notifyOnDownloadStall: true,
+        notifyOnDownloadRecovery: true,
+        notifyStallAfterSeconds: 90,
+        notifyStallCooldownMinutes: 10
+      };
+
+      const shutdown = controller.shutdown();
+
+      expect(controller.downloadHealthTimer).toBeNull();
+      expect(controller.notificationOutbox.drainForShutdown).not.toHaveBeenCalled();
+      runningEvaluation.resolve();
+      await shutdown;
+
+      const closed = loadDownloadHealthState(filePath);
+      expect(closed.status).toBe("idle");
+      expect(closed.alertedAt).toBe(0);
+      expect(closed.restartPending).toBe(false);
+
+      const recoveredEvents: unknown[] = [];
+      const restarted = new DownloadHealthMonitor(filePath);
+      await restarted.sample(healthSnapshot(1), 120_000, {
+        stallAfterMs: 90_000,
+        cooldownMs: 600_000,
+        notifyOnStall: true,
+        notifyOnRecovery: true
+      }, async (event) => { recoveredEvents.push(event); });
+      await restarted.sample(healthSnapshot(2), 135_000, {
+        stallAfterMs: 90_000,
+        cooldownMs: 600_000,
+        notifyOnStall: true,
+        notifyOnRecovery: true
+      }, async (event) => { recoveredEvents.push(event); });
+
+      expect(recoveredEvents).toEqual([]);
+    } finally {
+      fs.rmSync(root, { recursive: true, force: true });
+    }
+  });
 });