feat(notifications): detect persistent download stalls
This commit is contained in:
Sucukdeluxe
@@ -0,0 +1,452 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
DownloadHealthMonitor,
|
||||
createDownloadHealthState,
|
||||
evaluateDownloadHealth,
|
||||
loadDownloadHealthState,
|
||||
saveDownloadHealthState,
|
||||
type DownloadHealthSnapshot,
|
||||
type DownloadHealthState
|
||||
} from "../src/main/download-health-monitor";
|
||||
|
||||
const RUN_FINGERPRINT = "a".repeat(64);
|
||||
const QUEUE_FINGERPRINT = "b".repeat(64);
|
||||
const OTHER_RUN_FINGERPRINT = "c".repeat(64);
|
||||
const OTHER_QUEUE_FINGERPRINT = "d".repeat(64);
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function snapshot(overrides: Partial<DownloadHealthSnapshot> = {}): DownloadHealthSnapshot {
|
||||
return {
|
||||
runActive: true,
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
openItems: 2,
|
||||
openPackages: 1,
|
||||
knownDownloadedBytes: 4096,
|
||||
activeTasks: 1,
|
||||
startableItems: 0,
|
||||
lastSchedulerTickAt: 0,
|
||||
downloadProgressSequence: 0,
|
||||
itemCompletionSequence: 0,
|
||||
lastPositiveByteAt: 0,
|
||||
technicalRecoveryCount: 0,
|
||||
paused: false,
|
||||
reconnectUntil: 0,
|
||||
nextRetryAt: 0,
|
||||
providerCooldownUntil: 0,
|
||||
blockedOnDisk: false,
|
||||
blockedOnThrottleUntil: 0,
|
||||
activePhaseDeadlineAt: 0,
|
||||
terminalFailure: false,
|
||||
manualStop: false,
|
||||
shuttingDown: false,
|
||||
currentSpeedBps: 0,
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function evaluate(
|
||||
state: DownloadHealthState,
|
||||
current: DownloadHealthSnapshot,
|
||||
now: number,
|
||||
overrides: Partial<Parameters<typeof evaluateDownloadHealth>[3]> = {}
|
||||
) {
|
||||
return evaluateDownloadHealth(state, current, now, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true,
|
||||
...overrides
|
||||
});
|
||||
}
|
||||
|
||||
function sampleTimes(
|
||||
initial: DownloadHealthState,
|
||||
times: number[],
|
||||
current: DownloadHealthSnapshot = snapshot()
|
||||
) {
|
||||
let state = initial;
|
||||
const events = [];
|
||||
for (const now of times) {
|
||||
const result = evaluate(state, current, now);
|
||||
state = result.state;
|
||||
events.push(...result.events);
|
||||
}
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
function alertedState(now = 90_000): DownloadHealthState {
|
||||
return sampleTimes(createDownloadHealthState(), [0, 45_000, now]).state;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("evaluateDownloadHealth", () => {
|
||||
it("keeps a 20 to 30 second silent interval below the alert boundary", () => {
|
||||
const result = sampleTimes(createDownloadHealthState(), [0, 15_000, 30_000]);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("suspect_no_data");
|
||||
expect(result.state.suspiciousDurationMs).toBe(30_000);
|
||||
expect(result.state.suspiciousSamples).toBe(3);
|
||||
});
|
||||
|
||||
it("confirms a no-data stall only after 90 seconds and at least three suspicious samples", () => {
|
||||
const before = sampleTimes(createDownloadHealthState(), [0, 45_000]);
|
||||
const result = evaluate(before.state, snapshot(), 90_000);
|
||||
|
||||
expect(before.events).toEqual([]);
|
||||
expect(result.state.status).toBe("alerted");
|
||||
expect(result.events).toEqual([
|
||||
expect.objectContaining({ type: "download_stalled", priority: "error" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not alert from elapsed time until the third suspicious sample", () => {
|
||||
const result = sampleTimes(createDownloadHealthState(), [0, 90_000]);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.suspiciousDurationMs).toBe(90_000);
|
||||
expect(result.state.suspiciousSamples).toBe(2);
|
||||
});
|
||||
|
||||
it("classifies a startable queue with no scheduler as a scheduler suspicion", () => {
|
||||
const result = evaluate(createDownloadHealthState(), snapshot({
|
||||
activeTasks: 0,
|
||||
startableItems: 2,
|
||||
lastSchedulerTickAt: 0
|
||||
}), 45_000);
|
||||
|
||||
expect(result.state.status).toBe("suspect_scheduler");
|
||||
expect(result.events).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats a recent scheduler tick without an active task as healthy startup activity", () => {
|
||||
const result = evaluate(createDownloadHealthState(), snapshot({
|
||||
activeTasks: 0,
|
||||
startableItems: 2,
|
||||
lastSchedulerTickAt: 29_000
|
||||
}), 30_000);
|
||||
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.state.suspiciousSamples).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["pause", { paused: true }],
|
||||
["reconnect", { reconnectUntil: 120_000 }],
|
||||
["future retry", { activeTasks: 0, startableItems: 0, nextRetryAt: 120_000 }],
|
||||
["provider cooldown", { activeTasks: 0, startableItems: 0, providerCooldownUntil: 120_000 }],
|
||||
["disk wait", { blockedOnDisk: true }],
|
||||
["bandwidth throttle", { blockedOnThrottleUntil: 120_000 }],
|
||||
["valid phase deadline", { activePhaseDeadlineAt: 120_000 }]
|
||||
])("freezes accumulated suspicion during %s", (_name, waitState) => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||
const waiting = evaluate(suspicious.state, snapshot(waitState), 60_000);
|
||||
const resumed = evaluate(waiting.state, snapshot(), 90_000);
|
||||
|
||||
expect(waiting.state.status).toBe("expected_wait");
|
||||
expect(waiting.state.suspiciousDurationMs).toBe(30_000);
|
||||
expect(waiting.state.suspiciousSamples).toBe(2);
|
||||
expect(resumed.events).toEqual([]);
|
||||
expect(resumed.state.suspiciousDurationMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("resets suspicion after a positive byte sequence", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||
const result = evaluate(suspicious.state, snapshot({
|
||||
downloadProgressSequence: 1,
|
||||
lastPositiveByteAt: 45_000
|
||||
}), 45_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||
expect(result.state.suspiciousSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("resets suspicion after a successful item completion sequence", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||
const result = evaluate(suspicious.state, snapshot({ itemCompletionSequence: 1 }), 45_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||
});
|
||||
|
||||
it("ignores speed, progress, item timestamps and global totals as progress evidence", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]);
|
||||
const current = {
|
||||
...snapshot({ currentSpeedBps: 900_000_000 }),
|
||||
progressPercent: 99,
|
||||
updatedAt: 90_000,
|
||||
totalDownloadedBytes: 10_000_000_000
|
||||
} as DownloadHealthSnapshot;
|
||||
const result = evaluate(suspicious.state, current, 90_000);
|
||||
|
||||
expect(result.state.status).toBe("alerted");
|
||||
expect(result.events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not treat sequence decreases or a technical recovery attempt as progress", () => {
|
||||
const initial = createDownloadHealthState({
|
||||
downloadProgressSequence: 8,
|
||||
itemCompletionSequence: 3
|
||||
});
|
||||
const result = sampleTimes(initial, [0, 45_000, 90_000], snapshot({
|
||||
downloadProgressSequence: 2,
|
||||
itemCompletionSequence: 1,
|
||||
technicalRecoveryCount: 1
|
||||
}));
|
||||
|
||||
expect(result.state.status).toBe("alerted");
|
||||
expect(result.events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("requires two positive-byte samples before recovering an alerted incident", () => {
|
||||
const alerted = alertedState();
|
||||
const first = evaluate(alerted, snapshot({
|
||||
downloadProgressSequence: 1,
|
||||
lastPositiveByteAt: 105_000
|
||||
}), 105_000);
|
||||
const second = evaluate(first.state, snapshot({
|
||||
downloadProgressSequence: 2,
|
||||
lastPositiveByteAt: 120_000
|
||||
}), 120_000);
|
||||
|
||||
expect(first.state.status).toBe("recovering");
|
||||
expect(first.events).toEqual([]);
|
||||
expect(second.state.status).toBe("healthy");
|
||||
expect(second.events).toEqual([
|
||||
expect.objectContaining({ type: "download_recovered", priority: "success" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("recovers immediately after a successful item completion", () => {
|
||||
const result = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000);
|
||||
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.events).toEqual([
|
||||
expect.objectContaining({ type: "download_recovered" })
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["terminal failure", { terminalFailure: true }],
|
||||
["manual stop", { runActive: false, manualStop: true }],
|
||||
["shutdown", { runActive: false, shuttingDown: true }]
|
||||
])("closes an alerted incident without recovery after %s", (_name, endState) => {
|
||||
const result = evaluate(alertedState(), snapshot(endState), 105_000);
|
||||
|
||||
expect(result.state.status).toBe("idle");
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.alertedAt).toBe(0);
|
||||
});
|
||||
|
||||
it("applies a ten-minute cooldown before another incident event", () => {
|
||||
const firstAlert = alertedState();
|
||||
const recovered = evaluate(firstAlert, snapshot({ itemCompletionSequence: 1 }), 105_000).state;
|
||||
const duringCooldown = sampleTimes(recovered, [120_000, 165_000, 210_000]);
|
||||
const afterCooldown = evaluate(duringCooldown.state, snapshot(), 690_000);
|
||||
|
||||
expect(duringCooldown.events).toEqual([]);
|
||||
expect(duringCooldown.state.status).toBe("suspect_no_data");
|
||||
expect(afterCooldown.events).toEqual([
|
||||
expect.objectContaining({ type: "download_stalled" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps the incident event id stable when outbox persistence rejects the state transition", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const firstAttempt = evaluate(suspicious, snapshot(), 90_000);
|
||||
const retryAttempt = evaluate(suspicious, snapshot(), 105_000);
|
||||
|
||||
expect(firstAttempt.events[0].id).toBe(retryAttempt.events[0].id);
|
||||
});
|
||||
|
||||
it("omits identifiers, paths, URLs, providers and accounts from incident and recovery payloads", () => {
|
||||
const incident = evaluate(sampleTimes(createDownloadHealthState(), [0, 45_000]).state, snapshot(), 90_000).events[0];
|
||||
const recovered = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000).events[0];
|
||||
const serialized = JSON.stringify([incident, recovered]);
|
||||
|
||||
expect(serialized).not.toMatch(/https?:|\\|\/downloads\/|provider|account|item-|package-/i);
|
||||
expect(incident.payload.fields).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ name: "Offene Dateien", value: "2" }),
|
||||
expect.objectContaining({ name: "Technische Wiederherstellungen", value: "0" })
|
||||
]));
|
||||
});
|
||||
|
||||
it("honors disabled incident and recovery settings independently", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const disabledIncident = evaluate(suspicious, snapshot(), 90_000, { notifyOnStall: false });
|
||||
const disabledRecovery = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000, { notifyOnRecovery: false });
|
||||
|
||||
expect(disabledIncident.events).toEqual([]);
|
||||
expect(disabledIncident.state.status).toBe("suspect_no_data");
|
||||
expect(disabledRecovery.events).toEqual([]);
|
||||
expect(disabledRecovery.state.status).toBe("healthy");
|
||||
});
|
||||
});
|
||||
|
||||
describe("download health restart persistence", () => {
|
||||
it("preserves a persisted incident while startup is still idle", () => {
|
||||
const persisted = createDownloadHealthState({
|
||||
status: "alerted",
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
suspiciousDurationMs: 90_000,
|
||||
suspiciousSamples: 3,
|
||||
incidentStartedAt: 10_000,
|
||||
alertedAt: 90_000,
|
||||
restartPending: true
|
||||
});
|
||||
|
||||
const result = evaluate(persisted, snapshot({ runActive: false, openItems: 0 }), 100_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("suspended");
|
||||
expect(result.state.runFingerprint).toBe(RUN_FINGERPRINT);
|
||||
expect(result.state.queueFingerprint).toBe(QUEUE_FINGERPRINT);
|
||||
expect(result.state.restartPending).toBe(true);
|
||||
});
|
||||
|
||||
it("requires two fresh samples before re-alerting the same persisted fingerprint", () => {
|
||||
const persisted = createDownloadHealthState({
|
||||
status: "suspect_no_data",
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
suspiciousDurationMs: 90_000,
|
||||
suspiciousSamples: 3,
|
||||
incidentStartedAt: 10_000,
|
||||
restartPending: true
|
||||
});
|
||||
const first = evaluate(persisted, snapshot(), 100_000);
|
||||
const second = evaluate(first.state, snapshot(), 115_000);
|
||||
|
||||
expect(first.events).toEqual([]);
|
||||
expect(first.state.restartFreshSamples).toBe(1);
|
||||
expect(second.events).toEqual([
|
||||
expect.objectContaining({ type: "download_stalled" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("discards a persisted incident when the queue fingerprint changes", () => {
|
||||
const persisted = createDownloadHealthState({
|
||||
status: "alerted",
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
suspiciousDurationMs: 90_000,
|
||||
suspiciousSamples: 4,
|
||||
incidentStartedAt: 10_000,
|
||||
alertedAt: 90_000,
|
||||
restartPending: true
|
||||
});
|
||||
const result = evaluate(persisted, snapshot({
|
||||
runFingerprint: OTHER_RUN_FINGERPRINT,
|
||||
queueFingerprint: OTHER_QUEUE_FINGERPRINT
|
||||
}), 100_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("suspect_no_data");
|
||||
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||
expect(result.state.alertedAt).toBe(0);
|
||||
});
|
||||
|
||||
it("writes an allowlisted atomic state and reloads it with a fresh-sample gate", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-state-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
const state = {
|
||||
...alertedState(),
|
||||
privateUrl: "https://private.example.test/file",
|
||||
privatePath: "C:\\private\\download.bin",
|
||||
privateAccount: "private@example.test"
|
||||
} as DownloadHealthState;
|
||||
|
||||
saveDownloadHealthState(filePath, state);
|
||||
const persisted = fs.readFileSync(filePath, "utf8");
|
||||
const loaded = loadDownloadHealthState(filePath);
|
||||
|
||||
expect(persisted).not.toMatch(/private|example\.test|download\.bin/i);
|
||||
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
|
||||
expect(loaded.runFingerprint).toBe(RUN_FINGERPRINT);
|
||||
expect(loaded.queueFingerprint).toBe(QUEUE_FINGERPRINT);
|
||||
expect(loaded.restartPending).toBe(true);
|
||||
expect(loaded.restartFreshSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects malformed queue fingerprints instead of restoring an incident", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-invalid-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
fs.writeFileSync(filePath, JSON.stringify({
|
||||
...alertedState(),
|
||||
queueFingerprint: "https://private.example.test/queue"
|
||||
}), "utf8");
|
||||
|
||||
const loaded = loadDownloadHealthState(filePath);
|
||||
|
||||
expect(loaded).toEqual(createDownloadHealthState());
|
||||
});
|
||||
|
||||
it("does not commit an alert until the outbox accepts the event", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-failure-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const monitor = new DownloadHealthMonitor(filePath, suspicious);
|
||||
const eventIds: string[] = [];
|
||||
|
||||
await expect(monitor.sample(snapshot(), 90_000, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true
|
||||
}, async (event) => {
|
||||
eventIds.push(event.id);
|
||||
throw new Error("outbox unavailable");
|
||||
})).rejects.toThrow("outbox unavailable");
|
||||
await expect(monitor.sample(snapshot(), 105_000, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true
|
||||
}, async (event) => {
|
||||
eventIds.push(event.id);
|
||||
throw new Error("outbox unavailable");
|
||||
})).rejects.toThrow("outbox unavailable");
|
||||
|
||||
expect(eventIds[0]).toBe(eventIds[1]);
|
||||
expect(monitor.getState().status).toBe("suspect_no_data");
|
||||
expect(fs.existsSync(filePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("persists the alerted state after the outbox accepts the event", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-success-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const monitor = new DownloadHealthMonitor(filePath, suspicious);
|
||||
|
||||
const result = await monitor.sample(snapshot(), 90_000, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true
|
||||
}, async () => undefined);
|
||||
|
||||
expect(result.events).toHaveLength(1);
|
||||
expect(monitor.getState().status).toBe("alerted");
|
||||
expect(loadDownloadHealthState(filePath)).toEqual(expect.objectContaining({
|
||||
status: "alerted",
|
||||
restartPending: true
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -14696,3 +14696,245 @@ describe("package lifecycle telemetry boundaries", () => {
|
||||
expect(pkg.terminalAt).toBe(pkg.postProcessCompletedAt);
|
||||
});
|
||||
});
|
||||
|
||||
describe("download health snapshot", () => {
|
||||
function createHealthManager(root: string) {
|
||||
const session = emptySession();
|
||||
const packageId = "private-package-id";
|
||||
const itemId = "private-item-id";
|
||||
const now = 100_000;
|
||||
session.running = true;
|
||||
session.runStartedAt = 10_000;
|
||||
session.packageOrder = [packageId];
|
||||
session.packages[packageId] = {
|
||||
id: packageId,
|
||||
name: "Private package name",
|
||||
outputDir: path.join(root, "private-output"),
|
||||
extractDir: path.join(root, "private-extract"),
|
||||
status: "downloading",
|
||||
itemIds: [itemId],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
createdAt: 1_000,
|
||||
updatedAt: now
|
||||
};
|
||||
session.items[itemId] = {
|
||||
id: itemId,
|
||||
packageId,
|
||||
url: "https://private.example.test/file",
|
||||
provider: "realdebrid",
|
||||
providerLabel: "Private provider label",
|
||||
providerAccountId: "private-account-id",
|
||||
providerAccountLabel: "private@example.test",
|
||||
status: "downloading",
|
||||
retries: 0,
|
||||
speedBps: 8192,
|
||||
downloadedBytes: 4096,
|
||||
totalBytes: 16384,
|
||||
progressPercent: 25,
|
||||
fileName: "private-file.bin",
|
||||
targetPath: path.join(root, "private-output", "private-file.bin"),
|
||||
resumable: true,
|
||||
attempts: 1,
|
||||
lastError: "",
|
||||
fullStatus: "Download läuft",
|
||||
createdAt: 2_000,
|
||||
updatedAt: now
|
||||
};
|
||||
const manager = new DownloadManager(defaultSettings(), session, createStoragePaths(path.join(root, "state")));
|
||||
const state = manager as any;
|
||||
session.running = true;
|
||||
session.items[itemId].status = "downloading";
|
||||
session.items[itemId].speedBps = 8192;
|
||||
session.packages[packageId].status = "downloading";
|
||||
state.runItemIds.add(itemId);
|
||||
state.runPackageIds.add(packageId);
|
||||
return { manager, session, state, packageId, itemId };
|
||||
}
|
||||
|
||||
it("exposes only aggregate run-scope health data and opaque fingerprints", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-snapshot-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, state, packageId, itemId } = createHealthManager(root);
|
||||
state.lastSchedulerTickAt = 99_000;
|
||||
state.activeTasks.set(itemId, {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "downloading",
|
||||
phaseStartedAt: 90_000,
|
||||
phaseDeadlineAt: 0,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0
|
||||
});
|
||||
|
||||
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||
const serialized = JSON.stringify(health);
|
||||
|
||||
expect(health).toMatchObject({
|
||||
runActive: true,
|
||||
openItems: 1,
|
||||
openPackages: 1,
|
||||
knownDownloadedBytes: 4096,
|
||||
activeTasks: 1,
|
||||
startableItems: 0,
|
||||
lastSchedulerTickAt: 99_000,
|
||||
currentSpeedBps: 8192
|
||||
});
|
||||
expect(health.runFingerprint).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(health.queueFingerprint).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(serialized).not.toMatch(/private|example\.test|realdebrid|file\.bin/i);
|
||||
});
|
||||
|
||||
it("keeps the run fingerprint stable across restart time changes for the same generation scope", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-fingerprint-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session } = createHealthManager(root);
|
||||
session.packages[session.packageOrder[0]].resultGeneration = 4;
|
||||
|
||||
const before = manager.getDownloadHealthSnapshot(100_000);
|
||||
session.runStartedAt = 200_000;
|
||||
const after = manager.getDownloadHealthSnapshot(210_000);
|
||||
|
||||
expect(after.runFingerprint).toBe(before.runFingerprint);
|
||||
expect(after.queueFingerprint).toBe(before.queueFingerprint);
|
||||
});
|
||||
|
||||
it("advances progress sequences only for positive byte events and successful item completions", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-sequences-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||
|
||||
state.recordSpeed(1024, packageId);
|
||||
state.recordSpeed(0, packageId);
|
||||
state.recordSpeed(-512, packageId);
|
||||
session.totalDownloadedBytes = 10_000_000;
|
||||
session.totalDownloadedBytes = 0;
|
||||
state.recordRunOutcome(itemId, "completed");
|
||||
state.recordRunOutcome(itemId, "completed");
|
||||
|
||||
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(health.downloadProgressSequence).toBe(1);
|
||||
expect(health.lastPositiveByteAt).toBeGreaterThan(0);
|
||||
expect(health.itemCompletionSequence).toBe(1);
|
||||
});
|
||||
|
||||
it("projects retry, cooldown, disk, throttle and valid phase waits without stale disk events", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-waits-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||
const active = {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "validating",
|
||||
phaseStartedAt: 90_000,
|
||||
phaseDeadlineAt: 130_000,
|
||||
blockedOnDiskWrite: true,
|
||||
blockedOnDiskSince: 95_000,
|
||||
blockedOnThrottleUntil: 120_000
|
||||
};
|
||||
state.activeTasks.set(itemId, active);
|
||||
state.diskWaitEvents = [{
|
||||
phase: "download",
|
||||
targetPath: "C:\\private\\disk",
|
||||
requiredBytes: 100,
|
||||
availableBytes: 0,
|
||||
reserveBytes: 0,
|
||||
retryAt: 80_000,
|
||||
itemId,
|
||||
packageId
|
||||
}];
|
||||
|
||||
const activeWait = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(activeWait.blockedOnDisk).toBe(true);
|
||||
expect(activeWait.blockedOnThrottleUntil).toBe(120_000);
|
||||
expect(activeWait.activePhaseDeadlineAt).toBe(130_000);
|
||||
|
||||
state.activeTasks.clear();
|
||||
session.items[itemId].status = "queued";
|
||||
state.retryAfterByItem.set(itemId, 140_000);
|
||||
state.providerFailures.set("realdebrid", { count: 20, lastFailAt: 99_000, cooldownUntil: 150_000 });
|
||||
|
||||
const queuedWait = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(queuedWait.startableItems).toBe(0);
|
||||
expect(queuedWait.nextRetryAt).toBe(140_000);
|
||||
expect(queuedWait.providerCooldownUntil).toBe(150_000);
|
||||
expect(queuedWait.blockedOnDisk).toBe(false);
|
||||
});
|
||||
|
||||
it("does not project a partial retry as a global wait while another download is active", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-partial-wait-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||
const queuedId = "private-queued-id";
|
||||
session.items[queuedId] = {
|
||||
...session.items[itemId],
|
||||
id: queuedId,
|
||||
status: "queued",
|
||||
downloadedBytes: 0,
|
||||
speedBps: 0
|
||||
};
|
||||
session.packages[packageId].itemIds.push(queuedId);
|
||||
state.runItemIds.add(queuedId);
|
||||
state.activeTasks.set(itemId, {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "downloading",
|
||||
phaseStartedAt: 90_000,
|
||||
phaseDeadlineAt: 0,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0
|
||||
});
|
||||
state.retryAfterByItem.set(queuedId, 140_000);
|
||||
|
||||
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(health.activeTasks).toBe(1);
|
||||
expect(health.nextRetryAt).toBe(0);
|
||||
});
|
||||
|
||||
it("counts one technical recovery when the existing global watchdog restarts stalled tasks", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-watchdog-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, state, packageId, itemId } = createHealthManager(root);
|
||||
const active = {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "downloading",
|
||||
phaseStartedAt: 1,
|
||||
phaseDeadlineAt: 0,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0
|
||||
};
|
||||
state.activeTasks.set(itemId, active);
|
||||
state.lastGlobalProgressBytes = 0;
|
||||
state.lastGlobalProgressAt = 1;
|
||||
|
||||
state.runGlobalStallWatchdog(90_000);
|
||||
|
||||
expect(active.abortController.signal.aborted).toBe(true);
|
||||
expect(active.abortReason).toBe("stall");
|
||||
expect(manager.getDownloadHealthSnapshot(90_000).technicalRecoveryCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import AdmZip from "adm-zip";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { defaultSettings } from "../src/main/constants";
|
||||
import { buildAccountSummary, buildStatsPayload } from "../src/main/support-data";
|
||||
import { buildSupportBundle } from "../src/main/support-bundle";
|
||||
import { createStoragePaths } from "../src/main/storage";
|
||||
import { serializeRealDebridApiAccounts } from "../src/shared/real-debrid-accounts";
|
||||
import { createVisualFixture } from "./visual/fixtures";
|
||||
|
||||
@@ -53,4 +59,28 @@ describe("Real-Debrid support summary", () => {
|
||||
expect(serialized).toContain("realdebrid");
|
||||
expect(serialized).toContain("4096");
|
||||
});
|
||||
|
||||
it("keeps the persisted notification health incident outside support bundles", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-support-health-"));
|
||||
try {
|
||||
const paths = createStoragePaths(root);
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
fs.writeFileSync(paths.notificationHealthFile, "PRIVATE_HEALTH_INCIDENT_PAYLOAD", "utf8");
|
||||
const snapshot = structuredClone(createVisualFixture("empty").snapshot);
|
||||
const manager = {
|
||||
getSnapshot: () => snapshot,
|
||||
getPackageLogPath: () => null,
|
||||
getItemLogPath: () => null
|
||||
};
|
||||
|
||||
const buffer = await buildSupportBundle(manager as any, root, { hostDiagnosticsMode: "none" });
|
||||
const zip = new AdmZip(buffer);
|
||||
const entries = zip.getEntries().map((entry) => entry.entryName);
|
||||
|
||||
expect(entries).not.toContain(path.basename(paths.notificationHealthFile));
|
||||
expect(buffer.toString("utf8")).not.toContain("PRIVATE_HEALTH_INCIDENT_PAYLOAD");
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user