feat(notifications): detect persistent download stalls
This commit is contained in:
Sucukdeluxe
@@ -80,6 +80,7 @@ import { getLegacyDesktopLogDirectory, migrateLogDirectories, prepareLogDirector
|
||||
import { normalizeStatisticsLedger, saveStatisticsLedger } from "./statistics-ledger";
|
||||
import { NotificationOutbox } from "./notification-outbox";
|
||||
import { sendNotification } from "./notify";
|
||||
import { DownloadHealthMonitor } from "./download-health-monitor";
|
||||
|
||||
function sanitizeSettingsPatch(partial: Partial<AppSettings>): Partial<AppSettings> {
|
||||
const entries = Object.entries(partial || {}).filter(([, value]) => value !== undefined);
|
||||
@@ -122,6 +123,12 @@ export class AppController {
|
||||
|
||||
private notificationOutbox: NotificationOutbox;
|
||||
|
||||
private downloadHealthMonitor: DownloadHealthMonitor;
|
||||
|
||||
private downloadHealthTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
private downloadHealthEvaluation: Promise<void> | null = null;
|
||||
|
||||
private logDirectory = this.storagePaths.baseDir;
|
||||
|
||||
private onStateHandler: ((snapshot: UiSnapshot) => void) | null = null;
|
||||
@@ -168,6 +175,7 @@ export class AppController {
|
||||
timestamp: event.createdAt
|
||||
})
|
||||
});
|
||||
this.downloadHealthMonitor = new DownloadHealthMonitor(this.storagePaths.notificationHealthFile);
|
||||
void this.notificationOutbox.drain().catch((error) => {
|
||||
logger.warn(`Notification-Outbox konnte nicht gestartet werden: ${String(error)}`);
|
||||
});
|
||||
@@ -189,9 +197,14 @@ export class AppController {
|
||||
this.recordHistoryEntry(entry);
|
||||
}
|
||||
});
|
||||
this.manager.on("state", (snapshot: UiSnapshot) => {
|
||||
this.onStateHandler?.(snapshot);
|
||||
});
|
||||
this.manager.on("state", (snapshot: UiSnapshot) => {
|
||||
this.onStateHandler?.(snapshot);
|
||||
});
|
||||
void this.evaluateDownloadHealth();
|
||||
this.downloadHealthTimer = setInterval(() => {
|
||||
void this.evaluateDownloadHealth();
|
||||
}, 15_000);
|
||||
this.downloadHealthTimer.unref?.();
|
||||
logger.info(`App gestartet v${APP_VERSION}`);
|
||||
logger.info(`Log-Datei: ${getLogFilePath()}`);
|
||||
logAuditEvent("INFO", "App gestartet", {
|
||||
@@ -1284,12 +1297,46 @@ export class AppController {
|
||||
return this.manager.getPackageLogPath(packageId) || getPackageLogPath(packageId);
|
||||
}
|
||||
|
||||
public getItemLogPath(itemId: string): string | null {
|
||||
return this.manager.getItemLogPath(itemId) || getItemLogPath(itemId);
|
||||
}
|
||||
|
||||
public getItemLogPath(itemId: string): string | null {
|
||||
return this.manager.getItemLogPath(itemId) || getItemLogPath(itemId);
|
||||
}
|
||||
|
||||
private evaluateDownloadHealth(): Promise<void> {
|
||||
if (this.downloadHealthEvaluation) {
|
||||
return this.downloadHealthEvaluation;
|
||||
}
|
||||
const now = Date.now();
|
||||
const task = this.downloadHealthMonitor.sample(
|
||||
this.manager.getDownloadHealthSnapshot(now),
|
||||
now,
|
||||
{
|
||||
stallAfterMs: this.settings.notifyStallAfterSeconds * 1000,
|
||||
cooldownMs: this.settings.notifyStallCooldownMinutes * 60_000,
|
||||
notifyOnStall: this.settings.notifyOnDownloadStall && Boolean(String(this.settings.notifyUrl || "").trim()),
|
||||
notifyOnRecovery: this.settings.notifyOnDownloadRecovery && Boolean(String(this.settings.notifyUrl || "").trim())
|
||||
},
|
||||
(event) => this.notificationOutbox.enqueue(event)
|
||||
).then(() => undefined).catch((error) => {
|
||||
logger.warn(`Download-Health-Monitor konnte nicht ausgewertet werden: ${String(error)}`);
|
||||
}).finally(() => {
|
||||
if (this.downloadHealthEvaluation === task) {
|
||||
this.downloadHealthEvaluation = null;
|
||||
}
|
||||
});
|
||||
this.downloadHealthEvaluation = task;
|
||||
return task;
|
||||
}
|
||||
|
||||
public async shutdown(): Promise<void> {
|
||||
if (this.runtimeStatsTimer) {
|
||||
if (this.downloadHealthTimer) {
|
||||
clearInterval(this.downloadHealthTimer);
|
||||
this.downloadHealthTimer = null;
|
||||
}
|
||||
this.manager.suspendDownloadHealthMonitoring?.();
|
||||
if (this.downloadHealthEvaluation) {
|
||||
await this.downloadHealthEvaluation;
|
||||
}
|
||||
if (this.runtimeStatsTimer) {
|
||||
clearInterval(this.runtimeStatsTimer);
|
||||
this.runtimeStatsTimer = null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { NotificationEvent } from "./notification-outbox";
|
||||
|
||||
export type DownloadHealthStatus =
|
||||
| "idle"
|
||||
| "suspended"
|
||||
| "expected_wait"
|
||||
| "healthy"
|
||||
| "suspect_scheduler"
|
||||
| "suspect_no_data"
|
||||
| "alerted"
|
||||
| "recovering";
|
||||
|
||||
export type DownloadHealthIncidentType = "scheduler" | "no_data";
|
||||
|
||||
export interface DownloadHealthSnapshot {
|
||||
runActive: boolean;
|
||||
runFingerprint: string;
|
||||
queueFingerprint: string;
|
||||
openItems: number;
|
||||
openPackages: number;
|
||||
knownDownloadedBytes: number;
|
||||
activeTasks: number;
|
||||
startableItems: number;
|
||||
lastSchedulerTickAt: number;
|
||||
downloadProgressSequence: number;
|
||||
itemCompletionSequence: number;
|
||||
lastPositiveByteAt: number;
|
||||
technicalRecoveryCount: number;
|
||||
paused: boolean;
|
||||
reconnectUntil: number;
|
||||
nextRetryAt: number;
|
||||
providerCooldownUntil: number;
|
||||
blockedOnDisk: boolean;
|
||||
blockedOnThrottleUntil: number;
|
||||
activePhaseDeadlineAt: number;
|
||||
terminalFailure: boolean;
|
||||
manualStop: boolean;
|
||||
shuttingDown: boolean;
|
||||
currentSpeedBps: number;
|
||||
}
|
||||
|
||||
export interface DownloadHealthState {
|
||||
version: 1;
|
||||
status: DownloadHealthStatus;
|
||||
runFingerprint: string | null;
|
||||
queueFingerprint: string | null;
|
||||
suspiciousDurationMs: number;
|
||||
suspiciousSamples: number;
|
||||
incidentStartedAt: number;
|
||||
incidentType: DownloadHealthIncidentType | null;
|
||||
alertedAt: number;
|
||||
lastAlertAt: number;
|
||||
cooldownUntil: number;
|
||||
recoverySamples: number;
|
||||
lastSampleAt: number | null;
|
||||
downloadProgressSequence: number;
|
||||
itemCompletionSequence: number;
|
||||
lastPositiveByteAt: number;
|
||||
technicalRecoveryCount: number;
|
||||
restartPending: boolean;
|
||||
restartFreshSamples: number;
|
||||
}
|
||||
|
||||
export interface DownloadHealthOptions {
|
||||
stallAfterMs: number;
|
||||
cooldownMs: number;
|
||||
notifyOnStall: boolean;
|
||||
notifyOnRecovery: boolean;
|
||||
}
|
||||
|
||||
export interface DownloadHealthEvaluation {
|
||||
state: DownloadHealthState;
|
||||
events: NotificationEvent[];
|
||||
}
|
||||
|
||||
const HEALTH_STATUSES = new Set<DownloadHealthStatus>([
|
||||
"idle",
|
||||
"suspended",
|
||||
"expected_wait",
|
||||
"healthy",
|
||||
"suspect_scheduler",
|
||||
"suspect_no_data",
|
||||
"alerted",
|
||||
"recovering"
|
||||
]);
|
||||
const INCIDENT_TYPES = new Set<DownloadHealthIncidentType>(["scheduler", "no_data"]);
|
||||
const FINGERPRINT_PATTERN = /^[a-f0-9]{64}$/;
|
||||
const MIN_SUSPICIOUS_SAMPLES = 3;
|
||||
const SCHEDULER_STALE_AFTER_MS = 30_000;
|
||||
const ERROR_EVENT_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const SUCCESS_EVENT_TTL_MS = 6 * 60 * 60 * 1000;
|
||||
|
||||
function finiteInteger(value: unknown, fallback = 0): number {
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) ? Math.max(0, Math.floor(numeric)) : fallback;
|
||||
}
|
||||
|
||||
function validFingerprint(value: unknown): string | null {
|
||||
return typeof value === "string" && FINGERPRINT_PATTERN.test(value) ? value : null;
|
||||
}
|
||||
|
||||
function durationText(durationMs: number): string {
|
||||
const totalSeconds = Math.max(0, Math.floor(durationMs / 1000));
|
||||
if (totalSeconds < 60) {
|
||||
return `${totalSeconds} s`;
|
||||
}
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
return seconds > 0 ? `${minutes} min ${seconds} s` : `${minutes} min`;
|
||||
}
|
||||
|
||||
function byteText(bytes: number): string {
|
||||
const value = Math.max(0, finiteInteger(bytes));
|
||||
if (value < 1024) {
|
||||
return `${value} B`;
|
||||
}
|
||||
const units = ["KB", "MB", "GB", "TB"];
|
||||
let amount = value / 1024;
|
||||
let unit = units[0];
|
||||
for (let index = 1; index < units.length && amount >= 1024; index += 1) {
|
||||
amount /= 1024;
|
||||
unit = units[index];
|
||||
}
|
||||
return `${amount >= 10 ? amount.toFixed(0) : amount.toFixed(1)} ${unit}`;
|
||||
}
|
||||
|
||||
function incidentEvent(
|
||||
state: DownloadHealthState,
|
||||
snapshot: DownloadHealthSnapshot,
|
||||
now: number
|
||||
): NotificationEvent {
|
||||
const durationMs = Math.max(state.suspiciousDurationMs, now - state.incidentStartedAt);
|
||||
return {
|
||||
id: `health:stall:${snapshot.runFingerprint.slice(0, 16)}:${state.incidentStartedAt}`,
|
||||
type: "download_stalled",
|
||||
priority: "error",
|
||||
createdAt: now,
|
||||
expiresAt: now + ERROR_EVENT_TTL_MS,
|
||||
attempts: 0,
|
||||
nextAttemptAt: now,
|
||||
payload: {
|
||||
title: "Downloadstillstand bestätigt",
|
||||
description: `Seit ${durationText(durationMs)} wurde kein bestätigter Downloadfortschritt erkannt.`,
|
||||
color: 0xe67e22,
|
||||
fields: [
|
||||
{ name: "Dauer", value: durationText(durationMs), inline: true },
|
||||
{ name: "Offene Pakete", value: String(snapshot.openPackages), inline: true },
|
||||
{ name: "Offene Dateien", value: String(snapshot.openItems), inline: true },
|
||||
{ name: "Bekannte Mindestmenge", value: byteText(snapshot.knownDownloadedBytes), inline: true },
|
||||
{ name: "Aktiv / startfähig", value: `${snapshot.activeTasks} / ${snapshot.startableItems}`, inline: true },
|
||||
{ name: "Technische Wiederherstellungen", value: String(snapshot.technicalRecoveryCount), inline: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function recoveryEvent(
|
||||
state: DownloadHealthState,
|
||||
snapshot: DownloadHealthSnapshot,
|
||||
now: number
|
||||
): NotificationEvent {
|
||||
return {
|
||||
id: `health:recovery:${snapshot.runFingerprint.slice(0, 16)}:${state.alertedAt}`,
|
||||
type: "download_recovered",
|
||||
priority: "success",
|
||||
createdAt: now,
|
||||
expiresAt: now + SUCCESS_EVENT_TTL_MS,
|
||||
attempts: 0,
|
||||
nextAttemptAt: now,
|
||||
payload: {
|
||||
title: "Download läuft wieder",
|
||||
description: "Nach dem bestätigten Stillstand wurde neuer Fortschritt erkannt.",
|
||||
color: 0x2ecc71,
|
||||
fields: [
|
||||
{ name: "Incident-Dauer", value: durationText(Math.max(0, now - state.incidentStartedAt)), inline: true },
|
||||
{ name: "Aktive Downloads", value: String(snapshot.activeTasks), inline: true },
|
||||
{ name: "Aktuelle Geschwindigkeit", value: `${byteText(snapshot.currentSpeedBps)}/s`, inline: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function createDownloadHealthState(overrides: Partial<DownloadHealthState> = {}): DownloadHealthState {
|
||||
return {
|
||||
version: 1,
|
||||
status: "idle",
|
||||
runFingerprint: null,
|
||||
queueFingerprint: null,
|
||||
suspiciousDurationMs: 0,
|
||||
suspiciousSamples: 0,
|
||||
incidentStartedAt: 0,
|
||||
incidentType: null,
|
||||
alertedAt: 0,
|
||||
lastAlertAt: 0,
|
||||
cooldownUntil: 0,
|
||||
recoverySamples: 0,
|
||||
lastSampleAt: null,
|
||||
downloadProgressSequence: 0,
|
||||
itemCompletionSequence: 0,
|
||||
lastPositiveByteAt: 0,
|
||||
technicalRecoveryCount: 0,
|
||||
restartPending: false,
|
||||
restartFreshSamples: 0,
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function resetForSnapshot(
|
||||
state: DownloadHealthState,
|
||||
snapshot: DownloadHealthSnapshot,
|
||||
now: number
|
||||
): DownloadHealthState {
|
||||
return createDownloadHealthState({
|
||||
runFingerprint: snapshot.runFingerprint,
|
||||
queueFingerprint: snapshot.queueFingerprint,
|
||||
lastAlertAt: state.lastAlertAt,
|
||||
cooldownUntil: state.cooldownUntil,
|
||||
lastSampleAt: now,
|
||||
downloadProgressSequence: finiteInteger(snapshot.downloadProgressSequence),
|
||||
itemCompletionSequence: finiteInteger(snapshot.itemCompletionSequence),
|
||||
lastPositiveByteAt: finiteInteger(snapshot.lastPositiveByteAt),
|
||||
technicalRecoveryCount: finiteInteger(snapshot.technicalRecoveryCount)
|
||||
});
|
||||
}
|
||||
|
||||
function endIncident(state: DownloadHealthState): DownloadHealthState {
|
||||
return createDownloadHealthState({
|
||||
lastAlertAt: state.lastAlertAt,
|
||||
cooldownUntil: state.cooldownUntil,
|
||||
downloadProgressSequence: state.downloadProgressSequence,
|
||||
itemCompletionSequence: state.itemCompletionSequence,
|
||||
lastPositiveByteAt: state.lastPositiveByteAt,
|
||||
technicalRecoveryCount: state.technicalRecoveryCount
|
||||
});
|
||||
}
|
||||
|
||||
function expectedWait(snapshot: DownloadHealthSnapshot, now: number): boolean {
|
||||
return snapshot.paused
|
||||
|| snapshot.reconnectUntil > now
|
||||
|| snapshot.nextRetryAt > now
|
||||
|| snapshot.providerCooldownUntil > now
|
||||
|| snapshot.blockedOnDisk
|
||||
|| snapshot.blockedOnThrottleUntil > now
|
||||
|| snapshot.activePhaseDeadlineAt > now;
|
||||
}
|
||||
|
||||
function suspicionType(snapshot: DownloadHealthSnapshot, now: number): DownloadHealthIncidentType | null {
|
||||
if (snapshot.activeTasks === 0 && snapshot.startableItems > 0) {
|
||||
if (snapshot.lastSchedulerTickAt <= 0 || now - snapshot.lastSchedulerTickAt >= SCHEDULER_STALE_AFTER_MS) {
|
||||
return "scheduler";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return "no_data";
|
||||
}
|
||||
|
||||
export function evaluateDownloadHealth(
|
||||
previous: DownloadHealthState,
|
||||
snapshot: DownloadHealthSnapshot,
|
||||
nowValue: number,
|
||||
options: DownloadHealthOptions
|
||||
): DownloadHealthEvaluation {
|
||||
const now = finiteInteger(nowValue);
|
||||
let state = createDownloadHealthState(previous);
|
||||
const events: NotificationEvent[] = [];
|
||||
const runFingerprint = validFingerprint(snapshot.runFingerprint);
|
||||
const queueFingerprint = validFingerprint(snapshot.queueFingerprint);
|
||||
|
||||
if (state.restartPending && !snapshot.runActive && !snapshot.terminalFailure && !snapshot.manualStop && !snapshot.shuttingDown) {
|
||||
state.status = "suspended";
|
||||
state.lastSampleAt = null;
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
if (!snapshot.runActive || snapshot.openItems <= 0 || snapshot.terminalFailure || snapshot.manualStop || snapshot.shuttingDown || !runFingerprint || !queueFingerprint) {
|
||||
state.downloadProgressSequence = Math.max(state.downloadProgressSequence, finiteInteger(snapshot.downloadProgressSequence));
|
||||
state.itemCompletionSequence = Math.max(state.itemCompletionSequence, finiteInteger(snapshot.itemCompletionSequence));
|
||||
state.lastPositiveByteAt = Math.max(state.lastPositiveByteAt, finiteInteger(snapshot.lastPositiveByteAt));
|
||||
state.technicalRecoveryCount = Math.max(state.technicalRecoveryCount, finiteInteger(snapshot.technicalRecoveryCount));
|
||||
return { state: endIncident(state), events };
|
||||
}
|
||||
|
||||
if (state.runFingerprint !== runFingerprint || state.queueFingerprint !== queueFingerprint) {
|
||||
state = resetForSnapshot(state, snapshot, now);
|
||||
}
|
||||
|
||||
state.runFingerprint = runFingerprint;
|
||||
state.queueFingerprint = queueFingerprint;
|
||||
|
||||
if (state.restartPending) {
|
||||
state.restartFreshSamples += 1;
|
||||
state.lastSampleAt = now;
|
||||
if (state.restartFreshSamples < 2) {
|
||||
state.downloadProgressSequence = finiteInteger(snapshot.downloadProgressSequence);
|
||||
state.itemCompletionSequence = finiteInteger(snapshot.itemCompletionSequence);
|
||||
state.lastPositiveByteAt = finiteInteger(snapshot.lastPositiveByteAt);
|
||||
state.technicalRecoveryCount = finiteInteger(snapshot.technicalRecoveryCount);
|
||||
return { state, events };
|
||||
}
|
||||
state.restartPending = false;
|
||||
}
|
||||
|
||||
const progressSequence = finiteInteger(snapshot.downloadProgressSequence);
|
||||
const completionSequence = finiteInteger(snapshot.itemCompletionSequence);
|
||||
const positiveByteProgress = progressSequence > state.downloadProgressSequence;
|
||||
const completionProgress = completionSequence > state.itemCompletionSequence;
|
||||
const progress = positiveByteProgress || completionProgress;
|
||||
state.downloadProgressSequence = Math.max(state.downloadProgressSequence, progressSequence);
|
||||
state.itemCompletionSequence = Math.max(state.itemCompletionSequence, completionSequence);
|
||||
state.lastPositiveByteAt = Math.max(state.lastPositiveByteAt, finiteInteger(snapshot.lastPositiveByteAt));
|
||||
state.technicalRecoveryCount = Math.max(state.technicalRecoveryCount, finiteInteger(snapshot.technicalRecoveryCount));
|
||||
|
||||
if (state.alertedAt > 0 && progress) {
|
||||
state.recoverySamples = completionProgress ? 2 : state.recoverySamples + 1;
|
||||
if (state.recoverySamples >= 2) {
|
||||
if (options.notifyOnRecovery) {
|
||||
events.push(recoveryEvent(state, snapshot, now));
|
||||
}
|
||||
const recovered = resetForSnapshot(state, snapshot, now);
|
||||
recovered.status = "healthy";
|
||||
recovered.lastAlertAt = state.lastAlertAt;
|
||||
recovered.cooldownUntil = state.cooldownUntil;
|
||||
return { state: recovered, events };
|
||||
}
|
||||
state.status = "recovering";
|
||||
state.lastSampleAt = now;
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
if (state.alertedAt > 0) {
|
||||
if (expectedWait(snapshot, now)) {
|
||||
state.status = "expected_wait";
|
||||
} else {
|
||||
state.status = "alerted";
|
||||
}
|
||||
state.lastSampleAt = now;
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
if (expectedWait(snapshot, now)) {
|
||||
state.status = "expected_wait";
|
||||
state.lastSampleAt = now;
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
if (progress) {
|
||||
state.status = "healthy";
|
||||
state.suspiciousDurationMs = 0;
|
||||
state.suspiciousSamples = 0;
|
||||
state.incidentStartedAt = 0;
|
||||
state.incidentType = null;
|
||||
state.recoverySamples = 0;
|
||||
state.lastSampleAt = now;
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
const incidentType = suspicionType(snapshot, now);
|
||||
if (!incidentType) {
|
||||
state.status = "healthy";
|
||||
state.suspiciousDurationMs = 0;
|
||||
state.suspiciousSamples = 0;
|
||||
state.incidentStartedAt = 0;
|
||||
state.incidentType = null;
|
||||
state.lastSampleAt = now;
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
const elapsed = state.lastSampleAt === null ? 0 : Math.max(0, now - state.lastSampleAt);
|
||||
if (state.incidentStartedAt <= 0) {
|
||||
state.incidentStartedAt = now;
|
||||
}
|
||||
state.suspiciousDurationMs += elapsed;
|
||||
state.suspiciousSamples += 1;
|
||||
state.incidentType = incidentType;
|
||||
state.status = incidentType === "scheduler" ? "suspect_scheduler" : "suspect_no_data";
|
||||
state.lastSampleAt = now;
|
||||
|
||||
const stallAfterMs = Math.max(0, finiteInteger(options.stallAfterMs, 90_000));
|
||||
const cooldownMs = Math.max(0, finiteInteger(options.cooldownMs, 600_000));
|
||||
const confirmed = state.suspiciousDurationMs >= stallAfterMs
|
||||
&& state.suspiciousSamples >= MIN_SUSPICIOUS_SAMPLES;
|
||||
if (confirmed && options.notifyOnStall && now >= state.cooldownUntil) {
|
||||
state.status = "alerted";
|
||||
state.alertedAt = now;
|
||||
state.lastAlertAt = now;
|
||||
state.cooldownUntil = now + cooldownMs;
|
||||
state.recoverySamples = 0;
|
||||
events.push(incidentEvent(state, snapshot, now));
|
||||
}
|
||||
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
function persistedState(state: DownloadHealthState): Omit<DownloadHealthState, "restartPending" | "restartFreshSamples"> {
|
||||
return {
|
||||
version: 1,
|
||||
status: state.status,
|
||||
runFingerprint: state.runFingerprint,
|
||||
queueFingerprint: state.queueFingerprint,
|
||||
suspiciousDurationMs: finiteInteger(state.suspiciousDurationMs),
|
||||
suspiciousSamples: finiteInteger(state.suspiciousSamples),
|
||||
incidentStartedAt: finiteInteger(state.incidentStartedAt),
|
||||
incidentType: state.incidentType,
|
||||
alertedAt: finiteInteger(state.alertedAt),
|
||||
lastAlertAt: finiteInteger(state.lastAlertAt),
|
||||
cooldownUntil: finiteInteger(state.cooldownUntil),
|
||||
recoverySamples: finiteInteger(state.recoverySamples),
|
||||
lastSampleAt: state.lastSampleAt === null ? null : finiteInteger(state.lastSampleAt),
|
||||
downloadProgressSequence: finiteInteger(state.downloadProgressSequence),
|
||||
itemCompletionSequence: finiteInteger(state.itemCompletionSequence),
|
||||
lastPositiveByteAt: finiteInteger(state.lastPositiveByteAt),
|
||||
technicalRecoveryCount: finiteInteger(state.technicalRecoveryCount)
|
||||
};
|
||||
}
|
||||
|
||||
export function saveDownloadHealthState(filePath: string, state: DownloadHealthState): void {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
const tempPath = `${filePath}.tmp`;
|
||||
try {
|
||||
fs.writeFileSync(tempPath, JSON.stringify(persistedState(state)), "utf8");
|
||||
fs.renameSync(tempPath, filePath);
|
||||
} catch (error) {
|
||||
try {
|
||||
fs.rmSync(tempPath, { force: true });
|
||||
} catch {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function loadDownloadHealthState(filePath: string): DownloadHealthState {
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(filePath, "utf8")) as Record<string, unknown>;
|
||||
const runFingerprint = raw.runFingerprint === null ? null : validFingerprint(raw.runFingerprint);
|
||||
const queueFingerprint = raw.queueFingerprint === null ? null : validFingerprint(raw.queueFingerprint);
|
||||
if (raw.version !== 1 || !HEALTH_STATUSES.has(raw.status as DownloadHealthStatus)) {
|
||||
return createDownloadHealthState();
|
||||
}
|
||||
if ((raw.runFingerprint !== null && !runFingerprint) || (raw.queueFingerprint !== null && !queueFingerprint)) {
|
||||
return createDownloadHealthState();
|
||||
}
|
||||
const incidentType = INCIDENT_TYPES.has(raw.incidentType as DownloadHealthIncidentType)
|
||||
? raw.incidentType as DownloadHealthIncidentType
|
||||
: null;
|
||||
return createDownloadHealthState({
|
||||
status: raw.status as DownloadHealthStatus,
|
||||
runFingerprint,
|
||||
queueFingerprint,
|
||||
suspiciousDurationMs: finiteInteger(raw.suspiciousDurationMs),
|
||||
suspiciousSamples: finiteInteger(raw.suspiciousSamples),
|
||||
incidentStartedAt: finiteInteger(raw.incidentStartedAt),
|
||||
incidentType,
|
||||
alertedAt: finiteInteger(raw.alertedAt),
|
||||
lastAlertAt: finiteInteger(raw.lastAlertAt),
|
||||
cooldownUntil: finiteInteger(raw.cooldownUntil),
|
||||
recoverySamples: finiteInteger(raw.recoverySamples),
|
||||
lastSampleAt: null,
|
||||
downloadProgressSequence: finiteInteger(raw.downloadProgressSequence),
|
||||
itemCompletionSequence: finiteInteger(raw.itemCompletionSequence),
|
||||
lastPositiveByteAt: finiteInteger(raw.lastPositiveByteAt),
|
||||
technicalRecoveryCount: finiteInteger(raw.technicalRecoveryCount),
|
||||
restartPending: Boolean(runFingerprint && queueFingerprint),
|
||||
restartFreshSamples: 0
|
||||
});
|
||||
} catch {
|
||||
return createDownloadHealthState();
|
||||
}
|
||||
}
|
||||
|
||||
export class DownloadHealthMonitor {
|
||||
private state: DownloadHealthState;
|
||||
|
||||
public constructor(private readonly filePath: string, initialState?: DownloadHealthState) {
|
||||
this.state = initialState ? createDownloadHealthState(initialState) : loadDownloadHealthState(filePath);
|
||||
}
|
||||
|
||||
public getState(): DownloadHealthState {
|
||||
return createDownloadHealthState(this.state);
|
||||
}
|
||||
|
||||
public async sample(
|
||||
snapshot: DownloadHealthSnapshot,
|
||||
now: number,
|
||||
options: DownloadHealthOptions,
|
||||
enqueue: (event: NotificationEvent) => Promise<void>
|
||||
): Promise<DownloadHealthEvaluation> {
|
||||
const evaluation = evaluateDownloadHealth(this.state, snapshot, now, options);
|
||||
for (const event of evaluation.events) {
|
||||
await enqueue(event);
|
||||
}
|
||||
saveDownloadHealthState(this.filePath, evaluation.state);
|
||||
this.state = evaluation.state;
|
||||
return evaluation;
|
||||
}
|
||||
}
|
||||
+281
-99
@@ -1,6 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { createHash } from "node:crypto";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import {
|
||||
@@ -93,6 +94,7 @@ import {
|
||||
} from "./statistics-ledger";
|
||||
import { finalizePackageResult } from "./package-telemetry";
|
||||
import type { NotificationEvent } from "./notification-outbox";
|
||||
import type { DownloadHealthSnapshot } from "./download-health-monitor";
|
||||
import {
|
||||
buildHistoryEntry,
|
||||
buildPackageDigestEvents,
|
||||
@@ -118,8 +120,12 @@ type ActiveTask = {
|
||||
stallRetries?: number;
|
||||
genericErrorRetries?: number;
|
||||
unrestrictRetries?: number;
|
||||
blockedOnDiskWrite?: boolean;
|
||||
blockedOnDiskSince?: number;
|
||||
blockedOnDiskWrite?: boolean;
|
||||
blockedOnDiskSince?: number;
|
||||
blockedOnThrottleUntil?: number;
|
||||
phase?: "validating" | "downloading" | "integrity_check";
|
||||
phaseStartedAt?: number;
|
||||
phaseDeadlineAt?: number;
|
||||
};
|
||||
|
||||
const DOWNLOAD_ACCOUNT_PROVIDERS: readonly DebridProvider[] = [
|
||||
@@ -1960,7 +1966,23 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
private itemCount = 0;
|
||||
|
||||
private lastSchedulerHeartbeatAt = 0;
|
||||
private lastSchedulerHeartbeatAt = 0;
|
||||
|
||||
private lastSchedulerTickAt = 0;
|
||||
|
||||
private downloadProgressSequence = 0;
|
||||
|
||||
private itemCompletionSequence = 0;
|
||||
|
||||
private lastPositiveByteAt = 0;
|
||||
|
||||
private technicalRecoveryCount = 0;
|
||||
|
||||
private healthManualStop = false;
|
||||
|
||||
private healthShuttingDown = false;
|
||||
|
||||
private healthTerminalFailure = false;
|
||||
|
||||
private lastReconnectMarkAt = 0;
|
||||
|
||||
@@ -2478,11 +2500,118 @@ export class DownloadManager extends EventEmitter {
|
||||
return cloneSession(this.session);
|
||||
}
|
||||
|
||||
public getSummary(): DownloadSummary | null {
|
||||
return this.summary;
|
||||
}
|
||||
|
||||
public isSessionRunning(): boolean {
|
||||
public getSummary(): DownloadSummary | null {
|
||||
return this.summary;
|
||||
}
|
||||
|
||||
public getDownloadHealthSnapshot(now = nowMs()): DownloadHealthSnapshot {
|
||||
const openPackages = new Set<string>();
|
||||
const queueParts: string[] = [];
|
||||
const activeTasks: ActiveTask[] = [];
|
||||
let openItems = 0;
|
||||
let knownDownloadedBytes = 0;
|
||||
let currentSpeedBps = 0;
|
||||
let startableItems = 0;
|
||||
let nextRetryAt = 0;
|
||||
let providerCooldownUntil = 0;
|
||||
|
||||
for (const itemId of this.runItemIds) {
|
||||
queueParts.push(itemId);
|
||||
const item = this.session.items[itemId];
|
||||
if (!item || isFinishedStatus(item.status)) {
|
||||
continue;
|
||||
}
|
||||
const pkg = this.session.packages[item.packageId];
|
||||
if (!pkg || pkg.cancelled || !pkg.enabled) {
|
||||
continue;
|
||||
}
|
||||
openItems += 1;
|
||||
openPackages.add(pkg.id);
|
||||
knownDownloadedBytes += Math.max(0, Math.floor(Number(item.downloadedBytes) || 0));
|
||||
currentSpeedBps += Math.max(0, Math.floor(Number(item.speedBps) || 0));
|
||||
const active = this.activeTasks.get(itemId);
|
||||
if (active && !active.abortController.signal.aborted) {
|
||||
activeTasks.push(active);
|
||||
continue;
|
||||
}
|
||||
if (item.status !== "queued" && item.status !== "reconnect_wait") {
|
||||
continue;
|
||||
}
|
||||
const retryAt = this.retryAfterByItem.get(itemId) || 0;
|
||||
const failureKey = this.getProviderFailureKeyForItem(item);
|
||||
const cooldownAt = this.providerFailures.get(failureKey)?.cooldownUntil || 0;
|
||||
if (retryAt > now) {
|
||||
nextRetryAt = nextRetryAt === 0 ? retryAt : Math.min(nextRetryAt, retryAt);
|
||||
}
|
||||
if (cooldownAt > now) {
|
||||
providerCooldownUntil = providerCooldownUntil === 0
|
||||
? cooldownAt
|
||||
: Math.min(providerCooldownUntil, cooldownAt);
|
||||
}
|
||||
if (retryAt <= now && cooldownAt <= now) {
|
||||
startableItems += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const blockedOnDisk = activeTasks.length > 0 && activeTasks.every((active) => Boolean(active.blockedOnDiskWrite));
|
||||
const throttleDeadlines = activeTasks.map((active) => active.blockedOnThrottleUntil || 0);
|
||||
const blockedOnThrottleUntil = throttleDeadlines.length > 0 && throttleDeadlines.every((deadline) => deadline > now)
|
||||
? Math.min(...throttleDeadlines)
|
||||
: 0;
|
||||
const phaseDeadlines = activeTasks.map((active) => active.phaseDeadlineAt || 0);
|
||||
const activePhaseDeadlineAt = phaseDeadlines.length > 0 && phaseDeadlines.every((deadline) => deadline > now)
|
||||
? Math.min(...phaseDeadlines)
|
||||
: 0;
|
||||
const queueFingerprint = createHash("sha256")
|
||||
.update([...queueParts].sort().join("\n"))
|
||||
.digest("hex");
|
||||
const runParts = [...this.runPackageIds].map((packageId) => {
|
||||
const generation = Math.max(1, Math.floor(Number(this.session.packages[packageId]?.resultGeneration) || 1));
|
||||
return `${packageId}:${generation}`;
|
||||
});
|
||||
const runFingerprint = createHash("sha256")
|
||||
.update(`${runParts.sort().join("\n")}|${queueFingerprint}`)
|
||||
.digest("hex");
|
||||
|
||||
return {
|
||||
runActive: this.session.running && openItems > 0,
|
||||
runFingerprint,
|
||||
queueFingerprint,
|
||||
openItems,
|
||||
openPackages: openPackages.size,
|
||||
knownDownloadedBytes,
|
||||
activeTasks: activeTasks.length,
|
||||
startableItems,
|
||||
lastSchedulerTickAt: this.lastSchedulerTickAt,
|
||||
downloadProgressSequence: this.downloadProgressSequence,
|
||||
itemCompletionSequence: this.itemCompletionSequence,
|
||||
lastPositiveByteAt: this.lastPositiveByteAt,
|
||||
technicalRecoveryCount: this.technicalRecoveryCount,
|
||||
paused: this.session.paused,
|
||||
reconnectUntil: this.session.reconnectUntil,
|
||||
nextRetryAt: activeTasks.length === 0 && startableItems === 0 ? nextRetryAt : 0,
|
||||
providerCooldownUntil: activeTasks.length === 0 && startableItems === 0 ? providerCooldownUntil : 0,
|
||||
blockedOnDisk,
|
||||
blockedOnThrottleUntil,
|
||||
activePhaseDeadlineAt,
|
||||
terminalFailure: this.healthTerminalFailure,
|
||||
manualStop: this.healthManualStop,
|
||||
shuttingDown: this.healthShuttingDown,
|
||||
currentSpeedBps: this.session.running && !this.session.paused ? currentSpeedBps : 0
|
||||
};
|
||||
}
|
||||
|
||||
private beginHealthRun(): void {
|
||||
this.healthManualStop = false;
|
||||
this.healthShuttingDown = false;
|
||||
this.healthTerminalFailure = false;
|
||||
}
|
||||
|
||||
public suspendDownloadHealthMonitoring(): void {
|
||||
this.healthShuttingDown = true;
|
||||
}
|
||||
|
||||
public isSessionRunning(): boolean {
|
||||
return this.session.running;
|
||||
}
|
||||
|
||||
@@ -5978,6 +6107,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
public async startPackages(packageIds: string[]): Promise<void> {
|
||||
this.beginHealthRun();
|
||||
this.ensureUsableDownloadAccount();
|
||||
const targetSet = new Set(packageIds);
|
||||
for (const packageId of this.packagePostProcessTasks.keys()) {
|
||||
@@ -6080,6 +6210,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
public async startItems(itemIds: string[]): Promise<void> {
|
||||
this.beginHealthRun();
|
||||
this.ensureUsableDownloadAccount();
|
||||
const targetSet = new Set(itemIds);
|
||||
|
||||
@@ -6196,6 +6327,7 @@ export class DownloadManager extends EventEmitter {
|
||||
if (this.session.running) {
|
||||
return;
|
||||
}
|
||||
this.beginHealthRun();
|
||||
this.ensureUsableDownloadAccount();
|
||||
this.schedulerGeneration += 1;
|
||||
|
||||
@@ -6347,8 +6479,10 @@ export class DownloadManager extends EventEmitter {
|
||||
});
|
||||
}
|
||||
|
||||
public stop(options?: { parkForRestart?: boolean }): void {
|
||||
const parkForRestart = options?.parkForRestart === true;
|
||||
public stop(options?: { parkForRestart?: boolean }): void {
|
||||
const parkForRestart = options?.parkForRestart === true;
|
||||
this.healthManualStop = !parkForRestart;
|
||||
this.healthShuttingDown = parkForRestart;
|
||||
const abortReason: "stop" | "shutdown" = parkForRestart ? "shutdown" : "stop";
|
||||
const keepExtraction = this.settings.autoExtractWhenStopped;
|
||||
const wasRunning = this.session.running;
|
||||
@@ -6426,6 +6560,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
public prepareForShutdown(): void {
|
||||
this.healthShuttingDown = true;
|
||||
logger.info(`Shutdown-Vorbereitung gestartet: active=${this.activeTasks.size}, running=${this.session.running}, paused=${this.session.paused}`);
|
||||
this.updateStatisticsActivity(nowMs());
|
||||
this.rotationListenerActive = false;
|
||||
@@ -6922,9 +7057,14 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
private lastSpeedPruneAt = 0;
|
||||
|
||||
private recordSpeed(bytes: number, packageId: string = ""): void {
|
||||
const now = nowMs();
|
||||
if (bytes > 0 && this.consecutiveReconnects > 0) {
|
||||
private recordSpeed(bytes: number, packageId: string = ""): void {
|
||||
const now = nowMs();
|
||||
if (!Number.isFinite(bytes) || bytes <= 0) {
|
||||
return;
|
||||
}
|
||||
this.downloadProgressSequence += 1;
|
||||
this.lastPositiveByteAt = now;
|
||||
if (bytes > 0 && this.consecutiveReconnects > 0) {
|
||||
this.consecutiveReconnects = 0;
|
||||
}
|
||||
const bucket = now - (now % 120);
|
||||
@@ -6955,8 +7095,9 @@ export class DownloadManager extends EventEmitter {
|
||||
this.statisticsDirty = true;
|
||||
this.statisticsUrgent = true;
|
||||
}
|
||||
if (status === "completed" && previous !== "completed") {
|
||||
this.sessionCompletedFiles += 1;
|
||||
if (status === "completed" && previous !== "completed") {
|
||||
this.itemCompletionSequence += 1;
|
||||
this.sessionCompletedFiles += 1;
|
||||
this.settings.totalCompletedFilesAllTime = Math.max(0, Number(this.settings.totalCompletedFilesAllTime || 0)) + 1;
|
||||
this.invalidateStatsCache();
|
||||
}
|
||||
@@ -9108,6 +9249,7 @@ export class DownloadManager extends EventEmitter {
|
||||
try {
|
||||
while (this.session.running && this.schedulerGeneration === myGeneration) {
|
||||
const now = nowMs();
|
||||
this.lastSchedulerTickAt = now;
|
||||
this.updateStatisticsActivity(now);
|
||||
if (now - this.lastSchedulerHeartbeatAt >= 60000) {
|
||||
this.lastSchedulerHeartbeatAt = now;
|
||||
@@ -9246,8 +9388,9 @@ export class DownloadManager extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`);
|
||||
for (const active of this.activeTasks.values()) {
|
||||
logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`);
|
||||
this.technicalRecoveryCount += 1;
|
||||
for (const active of this.activeTasks.values()) {
|
||||
if (active.abortController.signal.aborted) {
|
||||
continue;
|
||||
}
|
||||
@@ -9576,10 +9719,14 @@ export class DownloadManager extends EventEmitter {
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0
|
||||
};
|
||||
nonResumableCounted: false,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0,
|
||||
phase: "validating",
|
||||
phaseStartedAt: item.updatedAt,
|
||||
phaseDeadlineAt: item.updatedAt + getUnrestrictTimeoutMs() + 15_000
|
||||
};
|
||||
this.activeTasks.set(itemId, active);
|
||||
this.notePacedStartForItem(item, nowMs());
|
||||
this.emitState();
|
||||
@@ -9701,6 +9848,9 @@ export class DownloadManager extends EventEmitter {
|
||||
this.settings,
|
||||
item.url
|
||||
);
|
||||
active.phase = "validating";
|
||||
active.phaseStartedAt = nowMs();
|
||||
active.phaseDeadlineAt = active.phaseStartedAt + unrestrictTimeoutMs;
|
||||
const unrestrictTimeoutSignal = AbortSignal.timeout(unrestrictTimeoutMs);
|
||||
const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]);
|
||||
let unrestricted;
|
||||
@@ -9805,6 +9955,9 @@ export class DownloadManager extends EventEmitter {
|
||||
throw error;
|
||||
}
|
||||
item.status = "downloading";
|
||||
active.phase = "downloading";
|
||||
active.phaseStartedAt = nowMs();
|
||||
active.phaseDeadlineAt = 0;
|
||||
const pLabel = unrestricted.providerLabel;
|
||||
item.fullStatus = "Starte...";
|
||||
item.updatedAt = nowMs();
|
||||
@@ -9855,10 +10008,18 @@ export class DownloadManager extends EventEmitter {
|
||||
item.status = "integrity_check";
|
||||
item.fullStatus = "CRC-Check läuft";
|
||||
item.updatedAt = nowMs();
|
||||
this.emitState();
|
||||
|
||||
const integrityStartedAt = nowMs();
|
||||
const validation = await validateFileAgainstManifest(item.targetPath, pkg.outputDir);
|
||||
this.emitState();
|
||||
|
||||
const integrityStartedAt = nowMs();
|
||||
const integrityBytes = Math.max(0, Number(item.downloadedBytes || item.totalBytes || 0));
|
||||
const integrityBudgetMs = Math.max(
|
||||
120_000,
|
||||
Math.min(30 * 60 * 1000, 60_000 + Math.ceil(integrityBytes / (25 * 1024 * 1024)) * 1000)
|
||||
);
|
||||
active.phase = "integrity_check";
|
||||
active.phaseStartedAt = integrityStartedAt;
|
||||
active.phaseDeadlineAt = integrityStartedAt + integrityBudgetMs;
|
||||
const validation = await validateFileAgainstManifest(item.targetPath, pkg.outputDir);
|
||||
if (active.abortController.signal.aborted) {
|
||||
throw new Error(`aborted:${active.abortReason}`);
|
||||
}
|
||||
@@ -9881,9 +10042,12 @@ export class DownloadManager extends EventEmitter {
|
||||
item.progressPercent = 0;
|
||||
item.downloadedBytes = 0;
|
||||
item.totalBytes = mergeKnownTotalBytes(item.totalBytes, unrestricted.fileSize);
|
||||
this.emitState();
|
||||
await sleep(300);
|
||||
continue;
|
||||
this.emitState();
|
||||
await sleep(300);
|
||||
active.phase = "downloading";
|
||||
active.phaseStartedAt = nowMs();
|
||||
active.phaseDeadlineAt = 0;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Integritätsprüfung fehlgeschlagen (${validation.message})`);
|
||||
}
|
||||
@@ -11178,7 +11342,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
||||
await this.applySpeedLimit(buffer.length, windowBytes, windowStarted, active.abortController.signal);
|
||||
await this.applySpeedLimit(buffer.length, windowBytes, windowStarted, active);
|
||||
if (active.abortController.signal.aborted) {
|
||||
throw new Error(`aborted:${active.abortReason}`);
|
||||
}
|
||||
@@ -12241,45 +12405,53 @@ export class DownloadManager extends EventEmitter {
|
||||
return 0;
|
||||
}
|
||||
|
||||
private async applyGlobalSpeedLimit(chunkBytes: number, bytesPerSecond: number, signal?: AbortSignal): Promise<void> {
|
||||
const task = this.globalSpeedLimitQueue
|
||||
private async applyGlobalSpeedLimit(chunkBytes: number, bytesPerSecond: number, active?: ActiveTask): Promise<void> {
|
||||
const signal = active?.abortController.signal;
|
||||
const task = this.globalSpeedLimitQueue
|
||||
.catch(() => undefined)
|
||||
.then(async () => {
|
||||
if (signal?.aborted) {
|
||||
throw new Error("aborted:speed_limit");
|
||||
}
|
||||
const now = nowMs();
|
||||
const waitMs = Math.max(0, this.globalSpeedLimitNextAt - now);
|
||||
if (waitMs > 0) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
||||
timer = null;
|
||||
if (signal) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
resolve();
|
||||
}, waitMs);
|
||||
|
||||
const onAbort = (): void => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(new Error("aborted:speed_limit"));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (signal?.aborted) {
|
||||
const now = nowMs();
|
||||
const waitMs = Math.max(0, this.globalSpeedLimitNextAt - now);
|
||||
if (waitMs > 0) {
|
||||
if (active) {
|
||||
active.blockedOnThrottleUntil = now + waitMs;
|
||||
}
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
||||
timer = null;
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, waitMs);
|
||||
|
||||
const onAbort = (): void => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(new Error("aborted:speed_limit"));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
if (active) {
|
||||
active.blockedOnThrottleUntil = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (signal?.aborted) {
|
||||
throw new Error("aborted:speed_limit");
|
||||
}
|
||||
|
||||
@@ -12292,7 +12464,8 @@ export class DownloadManager extends EventEmitter {
|
||||
await task;
|
||||
}
|
||||
|
||||
private async applySpeedLimit(chunkBytes: number, localWindowBytes: number, localWindowStarted: number, signal?: AbortSignal): Promise<void> {
|
||||
private async applySpeedLimit(chunkBytes: number, localWindowBytes: number, localWindowStarted: number, active?: ActiveTask): Promise<void> {
|
||||
const signal = active?.abortController.signal;
|
||||
const limitKbps = this.getEffectiveSpeedLimitKbps();
|
||||
if (limitKbps <= 0) {
|
||||
return;
|
||||
@@ -12304,40 +12477,48 @@ export class DownloadManager extends EventEmitter {
|
||||
const projected = localWindowBytes + chunkBytes;
|
||||
const allowed = bytesPerSecond * elapsed;
|
||||
if (projected > allowed) {
|
||||
const sleepMs = Math.ceil(((projected - allowed) / bytesPerSecond) * 1000);
|
||||
if (sleepMs > 0) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
||||
timer = null;
|
||||
if (signal) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
resolve();
|
||||
}, Math.min(300, sleepMs));
|
||||
|
||||
const onAbort = (): void => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(new Error("aborted:speed_limit"));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
const sleepMs = Math.ceil(((projected - allowed) / bytesPerSecond) * 1000);
|
||||
if (sleepMs > 0) {
|
||||
const boundedSleepMs = Math.min(300, sleepMs);
|
||||
if (active) {
|
||||
active.blockedOnThrottleUntil = nowMs() + boundedSleepMs;
|
||||
}
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
||||
timer = null;
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, boundedSleepMs);
|
||||
|
||||
const onAbort = (): void => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(new Error("aborted:speed_limit"));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
if (active) {
|
||||
active.blockedOnThrottleUntil = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await this.applyGlobalSpeedLimit(chunkBytes, bytesPerSecond, signal);
|
||||
await this.applyGlobalSpeedLimit(chunkBytes, bytesPerSecond, active);
|
||||
}
|
||||
|
||||
private async findReadyArchiveSets(pkg: PackageEntry): Promise<Set<string>> {
|
||||
@@ -13882,9 +14063,10 @@ export class DownloadManager extends EventEmitter {
|
||||
this.session.runStartedAt = 0;
|
||||
const total = this.runItemIds.size;
|
||||
const outcomes = Array.from(this.runOutcomes.values());
|
||||
const success = outcomes.filter((status) => status === "completed").length;
|
||||
const failed = outcomes.filter((status) => status === "failed").length;
|
||||
const cancelled = outcomes.filter((status) => status === "cancelled").length;
|
||||
const success = outcomes.filter((status) => status === "completed").length;
|
||||
const failed = outcomes.filter((status) => status === "failed").length;
|
||||
const cancelled = outcomes.filter((status) => status === "cancelled").length;
|
||||
this.healthTerminalFailure = failed > 0;
|
||||
const extracted = this.runCompletedPackages.size;
|
||||
const duration = runStartedAt > 0 ? Math.max(1, Math.floor((completedAt - runStartedAt) / 1000)) : 1;
|
||||
const avgSpeed = Math.floor(this.session.totalDownloadedBytes / duration);
|
||||
|
||||
+3
-1
@@ -702,6 +702,7 @@ export interface StoragePaths {
|
||||
historyFile: string;
|
||||
statisticsFile: string;
|
||||
notificationOutboxFile: string;
|
||||
notificationHealthFile: string;
|
||||
}
|
||||
|
||||
export function createStoragePaths(baseDir: string): StoragePaths {
|
||||
@@ -711,7 +712,8 @@ export function createStoragePaths(baseDir: string): StoragePaths {
|
||||
sessionFile: path.join(baseDir, "rd_session_state.json"),
|
||||
historyFile: path.join(baseDir, "rd_history.json"),
|
||||
statisticsFile: path.join(baseDir, "rd_statistics.json"),
|
||||
notificationOutboxFile: path.join(baseDir, "rd_notification_outbox.json")
|
||||
notificationOutboxFile: path.join(baseDir, "rd_notification_outbox.json"),
|
||||
notificationHealthFile: path.join(baseDir, "rd_notification_health.json")
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
DownloadHealthMonitor,
|
||||
createDownloadHealthState,
|
||||
evaluateDownloadHealth,
|
||||
loadDownloadHealthState,
|
||||
saveDownloadHealthState,
|
||||
type DownloadHealthSnapshot,
|
||||
type DownloadHealthState
|
||||
} from "../src/main/download-health-monitor";
|
||||
|
||||
const RUN_FINGERPRINT = "a".repeat(64);
|
||||
const QUEUE_FINGERPRINT = "b".repeat(64);
|
||||
const OTHER_RUN_FINGERPRINT = "c".repeat(64);
|
||||
const OTHER_QUEUE_FINGERPRINT = "d".repeat(64);
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function snapshot(overrides: Partial<DownloadHealthSnapshot> = {}): DownloadHealthSnapshot {
|
||||
return {
|
||||
runActive: true,
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
openItems: 2,
|
||||
openPackages: 1,
|
||||
knownDownloadedBytes: 4096,
|
||||
activeTasks: 1,
|
||||
startableItems: 0,
|
||||
lastSchedulerTickAt: 0,
|
||||
downloadProgressSequence: 0,
|
||||
itemCompletionSequence: 0,
|
||||
lastPositiveByteAt: 0,
|
||||
technicalRecoveryCount: 0,
|
||||
paused: false,
|
||||
reconnectUntil: 0,
|
||||
nextRetryAt: 0,
|
||||
providerCooldownUntil: 0,
|
||||
blockedOnDisk: false,
|
||||
blockedOnThrottleUntil: 0,
|
||||
activePhaseDeadlineAt: 0,
|
||||
terminalFailure: false,
|
||||
manualStop: false,
|
||||
shuttingDown: false,
|
||||
currentSpeedBps: 0,
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function evaluate(
|
||||
state: DownloadHealthState,
|
||||
current: DownloadHealthSnapshot,
|
||||
now: number,
|
||||
overrides: Partial<Parameters<typeof evaluateDownloadHealth>[3]> = {}
|
||||
) {
|
||||
return evaluateDownloadHealth(state, current, now, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true,
|
||||
...overrides
|
||||
});
|
||||
}
|
||||
|
||||
function sampleTimes(
|
||||
initial: DownloadHealthState,
|
||||
times: number[],
|
||||
current: DownloadHealthSnapshot = snapshot()
|
||||
) {
|
||||
let state = initial;
|
||||
const events = [];
|
||||
for (const now of times) {
|
||||
const result = evaluate(state, current, now);
|
||||
state = result.state;
|
||||
events.push(...result.events);
|
||||
}
|
||||
return { state, events };
|
||||
}
|
||||
|
||||
function alertedState(now = 90_000): DownloadHealthState {
|
||||
return sampleTimes(createDownloadHealthState(), [0, 45_000, now]).state;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("evaluateDownloadHealth", () => {
|
||||
it("keeps a 20 to 30 second silent interval below the alert boundary", () => {
|
||||
const result = sampleTimes(createDownloadHealthState(), [0, 15_000, 30_000]);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("suspect_no_data");
|
||||
expect(result.state.suspiciousDurationMs).toBe(30_000);
|
||||
expect(result.state.suspiciousSamples).toBe(3);
|
||||
});
|
||||
|
||||
it("confirms a no-data stall only after 90 seconds and at least three suspicious samples", () => {
|
||||
const before = sampleTimes(createDownloadHealthState(), [0, 45_000]);
|
||||
const result = evaluate(before.state, snapshot(), 90_000);
|
||||
|
||||
expect(before.events).toEqual([]);
|
||||
expect(result.state.status).toBe("alerted");
|
||||
expect(result.events).toEqual([
|
||||
expect.objectContaining({ type: "download_stalled", priority: "error" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not alert from elapsed time until the third suspicious sample", () => {
|
||||
const result = sampleTimes(createDownloadHealthState(), [0, 90_000]);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.suspiciousDurationMs).toBe(90_000);
|
||||
expect(result.state.suspiciousSamples).toBe(2);
|
||||
});
|
||||
|
||||
it("classifies a startable queue with no scheduler as a scheduler suspicion", () => {
|
||||
const result = evaluate(createDownloadHealthState(), snapshot({
|
||||
activeTasks: 0,
|
||||
startableItems: 2,
|
||||
lastSchedulerTickAt: 0
|
||||
}), 45_000);
|
||||
|
||||
expect(result.state.status).toBe("suspect_scheduler");
|
||||
expect(result.events).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats a recent scheduler tick without an active task as healthy startup activity", () => {
|
||||
const result = evaluate(createDownloadHealthState(), snapshot({
|
||||
activeTasks: 0,
|
||||
startableItems: 2,
|
||||
lastSchedulerTickAt: 29_000
|
||||
}), 30_000);
|
||||
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.state.suspiciousSamples).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["pause", { paused: true }],
|
||||
["reconnect", { reconnectUntil: 120_000 }],
|
||||
["future retry", { activeTasks: 0, startableItems: 0, nextRetryAt: 120_000 }],
|
||||
["provider cooldown", { activeTasks: 0, startableItems: 0, providerCooldownUntil: 120_000 }],
|
||||
["disk wait", { blockedOnDisk: true }],
|
||||
["bandwidth throttle", { blockedOnThrottleUntil: 120_000 }],
|
||||
["valid phase deadline", { activePhaseDeadlineAt: 120_000 }]
|
||||
])("freezes accumulated suspicion during %s", (_name, waitState) => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||
const waiting = evaluate(suspicious.state, snapshot(waitState), 60_000);
|
||||
const resumed = evaluate(waiting.state, snapshot(), 90_000);
|
||||
|
||||
expect(waiting.state.status).toBe("expected_wait");
|
||||
expect(waiting.state.suspiciousDurationMs).toBe(30_000);
|
||||
expect(waiting.state.suspiciousSamples).toBe(2);
|
||||
expect(resumed.events).toEqual([]);
|
||||
expect(resumed.state.suspiciousDurationMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("resets suspicion after a positive byte sequence", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||
const result = evaluate(suspicious.state, snapshot({
|
||||
downloadProgressSequence: 1,
|
||||
lastPositiveByteAt: 45_000
|
||||
}), 45_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||
expect(result.state.suspiciousSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("resets suspicion after a successful item completion sequence", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||
const result = evaluate(suspicious.state, snapshot({ itemCompletionSequence: 1 }), 45_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||
});
|
||||
|
||||
it("ignores speed, progress, item timestamps and global totals as progress evidence", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]);
|
||||
const current = {
|
||||
...snapshot({ currentSpeedBps: 900_000_000 }),
|
||||
progressPercent: 99,
|
||||
updatedAt: 90_000,
|
||||
totalDownloadedBytes: 10_000_000_000
|
||||
} as DownloadHealthSnapshot;
|
||||
const result = evaluate(suspicious.state, current, 90_000);
|
||||
|
||||
expect(result.state.status).toBe("alerted");
|
||||
expect(result.events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not treat sequence decreases or a technical recovery attempt as progress", () => {
|
||||
const initial = createDownloadHealthState({
|
||||
downloadProgressSequence: 8,
|
||||
itemCompletionSequence: 3
|
||||
});
|
||||
const result = sampleTimes(initial, [0, 45_000, 90_000], snapshot({
|
||||
downloadProgressSequence: 2,
|
||||
itemCompletionSequence: 1,
|
||||
technicalRecoveryCount: 1
|
||||
}));
|
||||
|
||||
expect(result.state.status).toBe("alerted");
|
||||
expect(result.events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("requires two positive-byte samples before recovering an alerted incident", () => {
|
||||
const alerted = alertedState();
|
||||
const first = evaluate(alerted, snapshot({
|
||||
downloadProgressSequence: 1,
|
||||
lastPositiveByteAt: 105_000
|
||||
}), 105_000);
|
||||
const second = evaluate(first.state, snapshot({
|
||||
downloadProgressSequence: 2,
|
||||
lastPositiveByteAt: 120_000
|
||||
}), 120_000);
|
||||
|
||||
expect(first.state.status).toBe("recovering");
|
||||
expect(first.events).toEqual([]);
|
||||
expect(second.state.status).toBe("healthy");
|
||||
expect(second.events).toEqual([
|
||||
expect.objectContaining({ type: "download_recovered", priority: "success" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("recovers immediately after a successful item completion", () => {
|
||||
const result = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000);
|
||||
|
||||
expect(result.state.status).toBe("healthy");
|
||||
expect(result.events).toEqual([
|
||||
expect.objectContaining({ type: "download_recovered" })
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["terminal failure", { terminalFailure: true }],
|
||||
["manual stop", { runActive: false, manualStop: true }],
|
||||
["shutdown", { runActive: false, shuttingDown: true }]
|
||||
])("closes an alerted incident without recovery after %s", (_name, endState) => {
|
||||
const result = evaluate(alertedState(), snapshot(endState), 105_000);
|
||||
|
||||
expect(result.state.status).toBe("idle");
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.alertedAt).toBe(0);
|
||||
});
|
||||
|
||||
it("applies a ten-minute cooldown before another incident event", () => {
|
||||
const firstAlert = alertedState();
|
||||
const recovered = evaluate(firstAlert, snapshot({ itemCompletionSequence: 1 }), 105_000).state;
|
||||
const duringCooldown = sampleTimes(recovered, [120_000, 165_000, 210_000]);
|
||||
const afterCooldown = evaluate(duringCooldown.state, snapshot(), 690_000);
|
||||
|
||||
expect(duringCooldown.events).toEqual([]);
|
||||
expect(duringCooldown.state.status).toBe("suspect_no_data");
|
||||
expect(afterCooldown.events).toEqual([
|
||||
expect.objectContaining({ type: "download_stalled" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps the incident event id stable when outbox persistence rejects the state transition", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const firstAttempt = evaluate(suspicious, snapshot(), 90_000);
|
||||
const retryAttempt = evaluate(suspicious, snapshot(), 105_000);
|
||||
|
||||
expect(firstAttempt.events[0].id).toBe(retryAttempt.events[0].id);
|
||||
});
|
||||
|
||||
it("omits identifiers, paths, URLs, providers and accounts from incident and recovery payloads", () => {
|
||||
const incident = evaluate(sampleTimes(createDownloadHealthState(), [0, 45_000]).state, snapshot(), 90_000).events[0];
|
||||
const recovered = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000).events[0];
|
||||
const serialized = JSON.stringify([incident, recovered]);
|
||||
|
||||
expect(serialized).not.toMatch(/https?:|\\|\/downloads\/|provider|account|item-|package-/i);
|
||||
expect(incident.payload.fields).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ name: "Offene Dateien", value: "2" }),
|
||||
expect.objectContaining({ name: "Technische Wiederherstellungen", value: "0" })
|
||||
]));
|
||||
});
|
||||
|
||||
it("honors disabled incident and recovery settings independently", () => {
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const disabledIncident = evaluate(suspicious, snapshot(), 90_000, { notifyOnStall: false });
|
||||
const disabledRecovery = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000, { notifyOnRecovery: false });
|
||||
|
||||
expect(disabledIncident.events).toEqual([]);
|
||||
expect(disabledIncident.state.status).toBe("suspect_no_data");
|
||||
expect(disabledRecovery.events).toEqual([]);
|
||||
expect(disabledRecovery.state.status).toBe("healthy");
|
||||
});
|
||||
});
|
||||
|
||||
describe("download health restart persistence", () => {
|
||||
it("preserves a persisted incident while startup is still idle", () => {
|
||||
const persisted = createDownloadHealthState({
|
||||
status: "alerted",
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
suspiciousDurationMs: 90_000,
|
||||
suspiciousSamples: 3,
|
||||
incidentStartedAt: 10_000,
|
||||
alertedAt: 90_000,
|
||||
restartPending: true
|
||||
});
|
||||
|
||||
const result = evaluate(persisted, snapshot({ runActive: false, openItems: 0 }), 100_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("suspended");
|
||||
expect(result.state.runFingerprint).toBe(RUN_FINGERPRINT);
|
||||
expect(result.state.queueFingerprint).toBe(QUEUE_FINGERPRINT);
|
||||
expect(result.state.restartPending).toBe(true);
|
||||
});
|
||||
|
||||
it("requires two fresh samples before re-alerting the same persisted fingerprint", () => {
|
||||
const persisted = createDownloadHealthState({
|
||||
status: "suspect_no_data",
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
suspiciousDurationMs: 90_000,
|
||||
suspiciousSamples: 3,
|
||||
incidentStartedAt: 10_000,
|
||||
restartPending: true
|
||||
});
|
||||
const first = evaluate(persisted, snapshot(), 100_000);
|
||||
const second = evaluate(first.state, snapshot(), 115_000);
|
||||
|
||||
expect(first.events).toEqual([]);
|
||||
expect(first.state.restartFreshSamples).toBe(1);
|
||||
expect(second.events).toEqual([
|
||||
expect.objectContaining({ type: "download_stalled" })
|
||||
]);
|
||||
});
|
||||
|
||||
it("discards a persisted incident when the queue fingerprint changes", () => {
|
||||
const persisted = createDownloadHealthState({
|
||||
status: "alerted",
|
||||
runFingerprint: RUN_FINGERPRINT,
|
||||
queueFingerprint: QUEUE_FINGERPRINT,
|
||||
suspiciousDurationMs: 90_000,
|
||||
suspiciousSamples: 4,
|
||||
incidentStartedAt: 10_000,
|
||||
alertedAt: 90_000,
|
||||
restartPending: true
|
||||
});
|
||||
const result = evaluate(persisted, snapshot({
|
||||
runFingerprint: OTHER_RUN_FINGERPRINT,
|
||||
queueFingerprint: OTHER_QUEUE_FINGERPRINT
|
||||
}), 100_000);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.state.status).toBe("suspect_no_data");
|
||||
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||
expect(result.state.alertedAt).toBe(0);
|
||||
});
|
||||
|
||||
it("writes an allowlisted atomic state and reloads it with a fresh-sample gate", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-state-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
const state = {
|
||||
...alertedState(),
|
||||
privateUrl: "https://private.example.test/file",
|
||||
privatePath: "C:\\private\\download.bin",
|
||||
privateAccount: "private@example.test"
|
||||
} as DownloadHealthState;
|
||||
|
||||
saveDownloadHealthState(filePath, state);
|
||||
const persisted = fs.readFileSync(filePath, "utf8");
|
||||
const loaded = loadDownloadHealthState(filePath);
|
||||
|
||||
expect(persisted).not.toMatch(/private|example\.test|download\.bin/i);
|
||||
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
|
||||
expect(loaded.runFingerprint).toBe(RUN_FINGERPRINT);
|
||||
expect(loaded.queueFingerprint).toBe(QUEUE_FINGERPRINT);
|
||||
expect(loaded.restartPending).toBe(true);
|
||||
expect(loaded.restartFreshSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects malformed queue fingerprints instead of restoring an incident", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-invalid-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
fs.writeFileSync(filePath, JSON.stringify({
|
||||
...alertedState(),
|
||||
queueFingerprint: "https://private.example.test/queue"
|
||||
}), "utf8");
|
||||
|
||||
const loaded = loadDownloadHealthState(filePath);
|
||||
|
||||
expect(loaded).toEqual(createDownloadHealthState());
|
||||
});
|
||||
|
||||
it("does not commit an alert until the outbox accepts the event", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-failure-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const monitor = new DownloadHealthMonitor(filePath, suspicious);
|
||||
const eventIds: string[] = [];
|
||||
|
||||
await expect(monitor.sample(snapshot(), 90_000, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true
|
||||
}, async (event) => {
|
||||
eventIds.push(event.id);
|
||||
throw new Error("outbox unavailable");
|
||||
})).rejects.toThrow("outbox unavailable");
|
||||
await expect(monitor.sample(snapshot(), 105_000, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true
|
||||
}, async (event) => {
|
||||
eventIds.push(event.id);
|
||||
throw new Error("outbox unavailable");
|
||||
})).rejects.toThrow("outbox unavailable");
|
||||
|
||||
expect(eventIds[0]).toBe(eventIds[1]);
|
||||
expect(monitor.getState().status).toBe("suspect_no_data");
|
||||
expect(fs.existsSync(filePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("persists the alerted state after the outbox accepts the event", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-success-"));
|
||||
tempDirs.push(root);
|
||||
const filePath = path.join(root, "health.json");
|
||||
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||
const monitor = new DownloadHealthMonitor(filePath, suspicious);
|
||||
|
||||
const result = await monitor.sample(snapshot(), 90_000, {
|
||||
stallAfterMs: 90_000,
|
||||
cooldownMs: 600_000,
|
||||
notifyOnStall: true,
|
||||
notifyOnRecovery: true
|
||||
}, async () => undefined);
|
||||
|
||||
expect(result.events).toHaveLength(1);
|
||||
expect(monitor.getState().status).toBe("alerted");
|
||||
expect(loadDownloadHealthState(filePath)).toEqual(expect.objectContaining({
|
||||
status: "alerted",
|
||||
restartPending: true
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -14696,3 +14696,245 @@ describe("package lifecycle telemetry boundaries", () => {
|
||||
expect(pkg.terminalAt).toBe(pkg.postProcessCompletedAt);
|
||||
});
|
||||
});
|
||||
|
||||
describe("download health snapshot", () => {
|
||||
function createHealthManager(root: string) {
|
||||
const session = emptySession();
|
||||
const packageId = "private-package-id";
|
||||
const itemId = "private-item-id";
|
||||
const now = 100_000;
|
||||
session.running = true;
|
||||
session.runStartedAt = 10_000;
|
||||
session.packageOrder = [packageId];
|
||||
session.packages[packageId] = {
|
||||
id: packageId,
|
||||
name: "Private package name",
|
||||
outputDir: path.join(root, "private-output"),
|
||||
extractDir: path.join(root, "private-extract"),
|
||||
status: "downloading",
|
||||
itemIds: [itemId],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
createdAt: 1_000,
|
||||
updatedAt: now
|
||||
};
|
||||
session.items[itemId] = {
|
||||
id: itemId,
|
||||
packageId,
|
||||
url: "https://private.example.test/file",
|
||||
provider: "realdebrid",
|
||||
providerLabel: "Private provider label",
|
||||
providerAccountId: "private-account-id",
|
||||
providerAccountLabel: "private@example.test",
|
||||
status: "downloading",
|
||||
retries: 0,
|
||||
speedBps: 8192,
|
||||
downloadedBytes: 4096,
|
||||
totalBytes: 16384,
|
||||
progressPercent: 25,
|
||||
fileName: "private-file.bin",
|
||||
targetPath: path.join(root, "private-output", "private-file.bin"),
|
||||
resumable: true,
|
||||
attempts: 1,
|
||||
lastError: "",
|
||||
fullStatus: "Download läuft",
|
||||
createdAt: 2_000,
|
||||
updatedAt: now
|
||||
};
|
||||
const manager = new DownloadManager(defaultSettings(), session, createStoragePaths(path.join(root, "state")));
|
||||
const state = manager as any;
|
||||
session.running = true;
|
||||
session.items[itemId].status = "downloading";
|
||||
session.items[itemId].speedBps = 8192;
|
||||
session.packages[packageId].status = "downloading";
|
||||
state.runItemIds.add(itemId);
|
||||
state.runPackageIds.add(packageId);
|
||||
return { manager, session, state, packageId, itemId };
|
||||
}
|
||||
|
||||
it("exposes only aggregate run-scope health data and opaque fingerprints", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-snapshot-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, state, packageId, itemId } = createHealthManager(root);
|
||||
state.lastSchedulerTickAt = 99_000;
|
||||
state.activeTasks.set(itemId, {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "downloading",
|
||||
phaseStartedAt: 90_000,
|
||||
phaseDeadlineAt: 0,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0
|
||||
});
|
||||
|
||||
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||
const serialized = JSON.stringify(health);
|
||||
|
||||
expect(health).toMatchObject({
|
||||
runActive: true,
|
||||
openItems: 1,
|
||||
openPackages: 1,
|
||||
knownDownloadedBytes: 4096,
|
||||
activeTasks: 1,
|
||||
startableItems: 0,
|
||||
lastSchedulerTickAt: 99_000,
|
||||
currentSpeedBps: 8192
|
||||
});
|
||||
expect(health.runFingerprint).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(health.queueFingerprint).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(serialized).not.toMatch(/private|example\.test|realdebrid|file\.bin/i);
|
||||
});
|
||||
|
||||
it("keeps the run fingerprint stable across restart time changes for the same generation scope", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-fingerprint-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session } = createHealthManager(root);
|
||||
session.packages[session.packageOrder[0]].resultGeneration = 4;
|
||||
|
||||
const before = manager.getDownloadHealthSnapshot(100_000);
|
||||
session.runStartedAt = 200_000;
|
||||
const after = manager.getDownloadHealthSnapshot(210_000);
|
||||
|
||||
expect(after.runFingerprint).toBe(before.runFingerprint);
|
||||
expect(after.queueFingerprint).toBe(before.queueFingerprint);
|
||||
});
|
||||
|
||||
it("advances progress sequences only for positive byte events and successful item completions", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-sequences-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||
|
||||
state.recordSpeed(1024, packageId);
|
||||
state.recordSpeed(0, packageId);
|
||||
state.recordSpeed(-512, packageId);
|
||||
session.totalDownloadedBytes = 10_000_000;
|
||||
session.totalDownloadedBytes = 0;
|
||||
state.recordRunOutcome(itemId, "completed");
|
||||
state.recordRunOutcome(itemId, "completed");
|
||||
|
||||
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(health.downloadProgressSequence).toBe(1);
|
||||
expect(health.lastPositiveByteAt).toBeGreaterThan(0);
|
||||
expect(health.itemCompletionSequence).toBe(1);
|
||||
});
|
||||
|
||||
it("projects retry, cooldown, disk, throttle and valid phase waits without stale disk events", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-waits-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||
const active = {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "validating",
|
||||
phaseStartedAt: 90_000,
|
||||
phaseDeadlineAt: 130_000,
|
||||
blockedOnDiskWrite: true,
|
||||
blockedOnDiskSince: 95_000,
|
||||
blockedOnThrottleUntil: 120_000
|
||||
};
|
||||
state.activeTasks.set(itemId, active);
|
||||
state.diskWaitEvents = [{
|
||||
phase: "download",
|
||||
targetPath: "C:\\private\\disk",
|
||||
requiredBytes: 100,
|
||||
availableBytes: 0,
|
||||
reserveBytes: 0,
|
||||
retryAt: 80_000,
|
||||
itemId,
|
||||
packageId
|
||||
}];
|
||||
|
||||
const activeWait = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(activeWait.blockedOnDisk).toBe(true);
|
||||
expect(activeWait.blockedOnThrottleUntil).toBe(120_000);
|
||||
expect(activeWait.activePhaseDeadlineAt).toBe(130_000);
|
||||
|
||||
state.activeTasks.clear();
|
||||
session.items[itemId].status = "queued";
|
||||
state.retryAfterByItem.set(itemId, 140_000);
|
||||
state.providerFailures.set("realdebrid", { count: 20, lastFailAt: 99_000, cooldownUntil: 150_000 });
|
||||
|
||||
const queuedWait = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(queuedWait.startableItems).toBe(0);
|
||||
expect(queuedWait.nextRetryAt).toBe(140_000);
|
||||
expect(queuedWait.providerCooldownUntil).toBe(150_000);
|
||||
expect(queuedWait.blockedOnDisk).toBe(false);
|
||||
});
|
||||
|
||||
it("does not project a partial retry as a global wait while another download is active", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-partial-wait-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||
const queuedId = "private-queued-id";
|
||||
session.items[queuedId] = {
|
||||
...session.items[itemId],
|
||||
id: queuedId,
|
||||
status: "queued",
|
||||
downloadedBytes: 0,
|
||||
speedBps: 0
|
||||
};
|
||||
session.packages[packageId].itemIds.push(queuedId);
|
||||
state.runItemIds.add(queuedId);
|
||||
state.activeTasks.set(itemId, {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "downloading",
|
||||
phaseStartedAt: 90_000,
|
||||
phaseDeadlineAt: 0,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0
|
||||
});
|
||||
state.retryAfterByItem.set(queuedId, 140_000);
|
||||
|
||||
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||
|
||||
expect(health.activeTasks).toBe(1);
|
||||
expect(health.nextRetryAt).toBe(0);
|
||||
});
|
||||
|
||||
it("counts one technical recovery when the existing global watchdog restarts stalled tasks", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-watchdog-"));
|
||||
tempDirs.push(root);
|
||||
const { manager, state, packageId, itemId } = createHealthManager(root);
|
||||
const active = {
|
||||
itemId,
|
||||
packageId,
|
||||
abortController: new AbortController(),
|
||||
abortReason: "none",
|
||||
resumable: true,
|
||||
nonResumableCounted: false,
|
||||
phase: "downloading",
|
||||
phaseStartedAt: 1,
|
||||
phaseDeadlineAt: 0,
|
||||
blockedOnDiskWrite: false,
|
||||
blockedOnDiskSince: 0,
|
||||
blockedOnThrottleUntil: 0
|
||||
};
|
||||
state.activeTasks.set(itemId, active);
|
||||
state.lastGlobalProgressBytes = 0;
|
||||
state.lastGlobalProgressAt = 1;
|
||||
|
||||
state.runGlobalStallWatchdog(90_000);
|
||||
|
||||
expect(active.abortController.signal.aborted).toBe(true);
|
||||
expect(active.abortReason).toBe("stall");
|
||||
expect(manager.getDownloadHealthSnapshot(90_000).technicalRecoveryCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import AdmZip from "adm-zip";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { defaultSettings } from "../src/main/constants";
|
||||
import { buildAccountSummary, buildStatsPayload } from "../src/main/support-data";
|
||||
import { buildSupportBundle } from "../src/main/support-bundle";
|
||||
import { createStoragePaths } from "../src/main/storage";
|
||||
import { serializeRealDebridApiAccounts } from "../src/shared/real-debrid-accounts";
|
||||
import { createVisualFixture } from "./visual/fixtures";
|
||||
|
||||
@@ -53,4 +59,28 @@ describe("Real-Debrid support summary", () => {
|
||||
expect(serialized).toContain("realdebrid");
|
||||
expect(serialized).toContain("4096");
|
||||
});
|
||||
|
||||
it("keeps the persisted notification health incident outside support bundles", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-support-health-"));
|
||||
try {
|
||||
const paths = createStoragePaths(root);
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
fs.writeFileSync(paths.notificationHealthFile, "PRIVATE_HEALTH_INCIDENT_PAYLOAD", "utf8");
|
||||
const snapshot = structuredClone(createVisualFixture("empty").snapshot);
|
||||
const manager = {
|
||||
getSnapshot: () => snapshot,
|
||||
getPackageLogPath: () => null,
|
||||
getItemLogPath: () => null
|
||||
};
|
||||
|
||||
const buffer = await buildSupportBundle(manager as any, root, { hostDiagnosticsMode: "none" });
|
||||
const zip = new AdmZip(buffer);
|
||||
const entries = zip.getEntries().map((entry) => entry.entryName);
|
||||
|
||||
expect(entries).not.toContain(path.basename(paths.notificationHealthFile));
|
||||
expect(buffer.toString("utf8")).not.toContain("PRIVATE_HEALTH_INCIDENT_PAYLOAD");
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user