fix(notifications): harden final delivery lifecycle

Enforce one shared shutdown deadline while persisting late digests during blocked sends. Start stall cooldowns from serialized delivery acknowledgements, atomically rewrite cleaned legacy outboxes, retain run summaries for 24 hours, and prevent duplicate history or synthetic post-process start times.
This commit is contained in:
Sucukdeluxe
2026-08-22 08:58:36 +02:00
parent 6715f42ff8
commit 3ec9085444
10 changed files with 733 additions and 77 deletions
+87 -3
View File
@@ -11,6 +11,7 @@ import {
type DownloadHealthSnapshot,
type DownloadHealthState
} from "../src/main/download-health-monitor";
import { NotificationOutbox } from "../src/main/notification-outbox";
const RUN_FINGERPRINT = "a".repeat(64);
const QUEUE_FINGERPRINT = "b".repeat(64);
@@ -250,8 +251,13 @@ describe("evaluateDownloadHealth", () => {
expect(result.state.alertedAt).toBe(0);
});
it("applies a ten-minute cooldown before another incident event", () => {
const firstAlert = alertedState();
it("applies a ten-minute cooldown after a delivered incident event", () => {
const firstAlert = createDownloadHealthState({
...alertedState(),
lastAlertAt: 90_000,
cooldownUntil: 690_000,
lastDeliveredStallEventId: `health:stall:${RUN_FINGERPRINT.slice(0, 16)}:0`
});
const recovered = evaluate(firstAlert, snapshot({ itemCompletionSequence: 1 }), 105_000).state;
const duringCooldown = sampleTimes(recovered, [120_000, 165_000, 210_000]);
const afterCooldown = evaluate(duringCooldown.state, snapshot(), 690_000);
@@ -263,6 +269,83 @@ describe("evaluateDownloadHealth", () => {
]);
});
it("starts the cooldown at actual Discord delivery after a buffered outage", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-delivery-ack-"));
tempDirs.push(root);
const healthFile = path.join(root, "health.json");
const outboxFile = path.join(root, "outbox.json");
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
const monitor = new DownloadHealthMonitor(healthFile, suspicious);
let now = 90_000;
let deliveryAvailable = false;
const deliveredIds: string[] = [];
const outbox = new NotificationOutbox({
filePath: outboxFile,
now: () => now,
send: async (queuedEvent) => {
if (deliveryAvailable) deliveredIds.push(queuedEvent.id);
return deliveryAvailable;
},
onDelivered: (queuedEvent, deliveredAt) => {
return monitor.acknowledgeDelivery(queuedEvent, deliveredAt, 600_000);
}
});
const confirmed = await monitor.sample(snapshot(), now, {
stallAfterMs: 90_000,
cooldownMs: 600_000,
notifyOnStall: true,
notifyOnRecovery: true
}, (event) => outbox.enqueue(event));
await outbox.drain();
now = 300_000;
const repeated = await monitor.sample(snapshot(), now, {
stallAfterMs: 90_000,
cooldownMs: 600_000,
notifyOnStall: true,
notifyOnRecovery: true
}, (event) => outbox.enqueue(event));
expect(confirmed.events).toHaveLength(1);
expect(repeated.events).toEqual([]);
expect(outbox.getStatus().queued).toBe(1);
expect(monitor.getState().cooldownUntil).toBe(0);
deliveryAvailable = true;
now = 420_000;
await outbox.drain(now);
expect(deliveredIds).toEqual([confirmed.events[0].id]);
expect(monitor.getState().lastAlertAt).toBe(420_000);
expect(monitor.getState().cooldownUntil).toBe(1_020_000);
});
it("serializes a delivery acknowledgement behind a concurrent recovery sample", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-ack-race-"));
tempDirs.push(root);
const filePath = path.join(root, "health.json");
const confirmed = sampleTimes(createDownloadHealthState(), [0, 45_000, 90_000]);
const monitor = new DownloadHealthMonitor(filePath, confirmed.state);
let releaseEnqueue = () => {};
const enqueueBlocked = new Promise<void>((resolve) => { releaseEnqueue = resolve; });
const recovery = monitor.sample(snapshot({ itemCompletionSequence: 1 }), 105_000, {
stallAfterMs: 90_000,
cooldownMs: 600_000,
notifyOnStall: true,
notifyOnRecovery: true
}, async () => enqueueBlocked);
await Promise.resolve();
const acknowledgement = monitor.acknowledgeDelivery(confirmed.events[0], 300_000, 600_000);
releaseEnqueue();
await Promise.all([recovery, acknowledgement]);
expect(monitor.getState().status).toBe("healthy");
expect(monitor.getState().lastAlertAt).toBe(300_000);
expect(monitor.getState().cooldownUntil).toBe(900_000);
expect(monitor.getState().lastDeliveredStallEventId).toBe(confirmed.events[0].id);
});
it("keeps the incident event id stable when outbox persistence rejects the state transition", () => {
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
const firstAttempt = evaluate(suspicious, snapshot(), 90_000);
@@ -367,7 +450,8 @@ describe("download health restart persistence", () => {
...alertedState(),
privateUrl: "https://private.example.test/file",
privatePath: "C:\\private\\download.bin",
privateAccount: "private@example.test"
privateAccount: "private@example.test",
lastDeliveredStallEventId: "https://private.example.test/stall"
} as DownloadHealthState;
saveDownloadHealthState(filePath, state);