feat(notifications): detect persistent download stalls
This commit is contained in:
@@ -80,6 +80,7 @@ import { getLegacyDesktopLogDirectory, migrateLogDirectories, prepareLogDirector
|
|||||||
import { normalizeStatisticsLedger, saveStatisticsLedger } from "./statistics-ledger";
|
import { normalizeStatisticsLedger, saveStatisticsLedger } from "./statistics-ledger";
|
||||||
import { NotificationOutbox } from "./notification-outbox";
|
import { NotificationOutbox } from "./notification-outbox";
|
||||||
import { sendNotification } from "./notify";
|
import { sendNotification } from "./notify";
|
||||||
|
import { DownloadHealthMonitor } from "./download-health-monitor";
|
||||||
|
|
||||||
function sanitizeSettingsPatch(partial: Partial<AppSettings>): Partial<AppSettings> {
|
function sanitizeSettingsPatch(partial: Partial<AppSettings>): Partial<AppSettings> {
|
||||||
const entries = Object.entries(partial || {}).filter(([, value]) => value !== undefined);
|
const entries = Object.entries(partial || {}).filter(([, value]) => value !== undefined);
|
||||||
@@ -122,6 +123,12 @@ export class AppController {
|
|||||||
|
|
||||||
private notificationOutbox: NotificationOutbox;
|
private notificationOutbox: NotificationOutbox;
|
||||||
|
|
||||||
|
private downloadHealthMonitor: DownloadHealthMonitor;
|
||||||
|
|
||||||
|
private downloadHealthTimer: NodeJS.Timeout | null = null;
|
||||||
|
|
||||||
|
private downloadHealthEvaluation: Promise<void> | null = null;
|
||||||
|
|
||||||
private logDirectory = this.storagePaths.baseDir;
|
private logDirectory = this.storagePaths.baseDir;
|
||||||
|
|
||||||
private onStateHandler: ((snapshot: UiSnapshot) => void) | null = null;
|
private onStateHandler: ((snapshot: UiSnapshot) => void) | null = null;
|
||||||
@@ -168,6 +175,7 @@ export class AppController {
|
|||||||
timestamp: event.createdAt
|
timestamp: event.createdAt
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
|
this.downloadHealthMonitor = new DownloadHealthMonitor(this.storagePaths.notificationHealthFile);
|
||||||
void this.notificationOutbox.drain().catch((error) => {
|
void this.notificationOutbox.drain().catch((error) => {
|
||||||
logger.warn(`Notification-Outbox konnte nicht gestartet werden: ${String(error)}`);
|
logger.warn(`Notification-Outbox konnte nicht gestartet werden: ${String(error)}`);
|
||||||
});
|
});
|
||||||
@@ -189,9 +197,14 @@ export class AppController {
|
|||||||
this.recordHistoryEntry(entry);
|
this.recordHistoryEntry(entry);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
this.manager.on("state", (snapshot: UiSnapshot) => {
|
this.manager.on("state", (snapshot: UiSnapshot) => {
|
||||||
this.onStateHandler?.(snapshot);
|
this.onStateHandler?.(snapshot);
|
||||||
});
|
});
|
||||||
|
void this.evaluateDownloadHealth();
|
||||||
|
this.downloadHealthTimer = setInterval(() => {
|
||||||
|
void this.evaluateDownloadHealth();
|
||||||
|
}, 15_000);
|
||||||
|
this.downloadHealthTimer.unref?.();
|
||||||
logger.info(`App gestartet v${APP_VERSION}`);
|
logger.info(`App gestartet v${APP_VERSION}`);
|
||||||
logger.info(`Log-Datei: ${getLogFilePath()}`);
|
logger.info(`Log-Datei: ${getLogFilePath()}`);
|
||||||
logAuditEvent("INFO", "App gestartet", {
|
logAuditEvent("INFO", "App gestartet", {
|
||||||
@@ -1284,12 +1297,46 @@ export class AppController {
|
|||||||
return this.manager.getPackageLogPath(packageId) || getPackageLogPath(packageId);
|
return this.manager.getPackageLogPath(packageId) || getPackageLogPath(packageId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getItemLogPath(itemId: string): string | null {
|
public getItemLogPath(itemId: string): string | null {
|
||||||
return this.manager.getItemLogPath(itemId) || getItemLogPath(itemId);
|
return this.manager.getItemLogPath(itemId) || getItemLogPath(itemId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private evaluateDownloadHealth(): Promise<void> {
|
||||||
|
if (this.downloadHealthEvaluation) {
|
||||||
|
return this.downloadHealthEvaluation;
|
||||||
|
}
|
||||||
|
const now = Date.now();
|
||||||
|
const task = this.downloadHealthMonitor.sample(
|
||||||
|
this.manager.getDownloadHealthSnapshot(now),
|
||||||
|
now,
|
||||||
|
{
|
||||||
|
stallAfterMs: this.settings.notifyStallAfterSeconds * 1000,
|
||||||
|
cooldownMs: this.settings.notifyStallCooldownMinutes * 60_000,
|
||||||
|
notifyOnStall: this.settings.notifyOnDownloadStall && Boolean(String(this.settings.notifyUrl || "").trim()),
|
||||||
|
notifyOnRecovery: this.settings.notifyOnDownloadRecovery && Boolean(String(this.settings.notifyUrl || "").trim())
|
||||||
|
},
|
||||||
|
(event) => this.notificationOutbox.enqueue(event)
|
||||||
|
).then(() => undefined).catch((error) => {
|
||||||
|
logger.warn(`Download-Health-Monitor konnte nicht ausgewertet werden: ${String(error)}`);
|
||||||
|
}).finally(() => {
|
||||||
|
if (this.downloadHealthEvaluation === task) {
|
||||||
|
this.downloadHealthEvaluation = null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
this.downloadHealthEvaluation = task;
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
public async shutdown(): Promise<void> {
|
public async shutdown(): Promise<void> {
|
||||||
if (this.runtimeStatsTimer) {
|
if (this.downloadHealthTimer) {
|
||||||
|
clearInterval(this.downloadHealthTimer);
|
||||||
|
this.downloadHealthTimer = null;
|
||||||
|
}
|
||||||
|
this.manager.suspendDownloadHealthMonitoring?.();
|
||||||
|
if (this.downloadHealthEvaluation) {
|
||||||
|
await this.downloadHealthEvaluation;
|
||||||
|
}
|
||||||
|
if (this.runtimeStatsTimer) {
|
||||||
clearInterval(this.runtimeStatsTimer);
|
clearInterval(this.runtimeStatsTimer);
|
||||||
this.runtimeStatsTimer = null;
|
this.runtimeStatsTimer = null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,497 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { NotificationEvent } from "./notification-outbox";
|
||||||
|
|
||||||
|
export type DownloadHealthStatus =
|
||||||
|
| "idle"
|
||||||
|
| "suspended"
|
||||||
|
| "expected_wait"
|
||||||
|
| "healthy"
|
||||||
|
| "suspect_scheduler"
|
||||||
|
| "suspect_no_data"
|
||||||
|
| "alerted"
|
||||||
|
| "recovering";
|
||||||
|
|
||||||
|
export type DownloadHealthIncidentType = "scheduler" | "no_data";
|
||||||
|
|
||||||
|
export interface DownloadHealthSnapshot {
|
||||||
|
runActive: boolean;
|
||||||
|
runFingerprint: string;
|
||||||
|
queueFingerprint: string;
|
||||||
|
openItems: number;
|
||||||
|
openPackages: number;
|
||||||
|
knownDownloadedBytes: number;
|
||||||
|
activeTasks: number;
|
||||||
|
startableItems: number;
|
||||||
|
lastSchedulerTickAt: number;
|
||||||
|
downloadProgressSequence: number;
|
||||||
|
itemCompletionSequence: number;
|
||||||
|
lastPositiveByteAt: number;
|
||||||
|
technicalRecoveryCount: number;
|
||||||
|
paused: boolean;
|
||||||
|
reconnectUntil: number;
|
||||||
|
nextRetryAt: number;
|
||||||
|
providerCooldownUntil: number;
|
||||||
|
blockedOnDisk: boolean;
|
||||||
|
blockedOnThrottleUntil: number;
|
||||||
|
activePhaseDeadlineAt: number;
|
||||||
|
terminalFailure: boolean;
|
||||||
|
manualStop: boolean;
|
||||||
|
shuttingDown: boolean;
|
||||||
|
currentSpeedBps: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadHealthState {
|
||||||
|
version: 1;
|
||||||
|
status: DownloadHealthStatus;
|
||||||
|
runFingerprint: string | null;
|
||||||
|
queueFingerprint: string | null;
|
||||||
|
suspiciousDurationMs: number;
|
||||||
|
suspiciousSamples: number;
|
||||||
|
incidentStartedAt: number;
|
||||||
|
incidentType: DownloadHealthIncidentType | null;
|
||||||
|
alertedAt: number;
|
||||||
|
lastAlertAt: number;
|
||||||
|
cooldownUntil: number;
|
||||||
|
recoverySamples: number;
|
||||||
|
lastSampleAt: number | null;
|
||||||
|
downloadProgressSequence: number;
|
||||||
|
itemCompletionSequence: number;
|
||||||
|
lastPositiveByteAt: number;
|
||||||
|
technicalRecoveryCount: number;
|
||||||
|
restartPending: boolean;
|
||||||
|
restartFreshSamples: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadHealthOptions {
|
||||||
|
stallAfterMs: number;
|
||||||
|
cooldownMs: number;
|
||||||
|
notifyOnStall: boolean;
|
||||||
|
notifyOnRecovery: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadHealthEvaluation {
|
||||||
|
state: DownloadHealthState;
|
||||||
|
events: NotificationEvent[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const HEALTH_STATUSES = new Set<DownloadHealthStatus>([
|
||||||
|
"idle",
|
||||||
|
"suspended",
|
||||||
|
"expected_wait",
|
||||||
|
"healthy",
|
||||||
|
"suspect_scheduler",
|
||||||
|
"suspect_no_data",
|
||||||
|
"alerted",
|
||||||
|
"recovering"
|
||||||
|
]);
|
||||||
|
const INCIDENT_TYPES = new Set<DownloadHealthIncidentType>(["scheduler", "no_data"]);
|
||||||
|
const FINGERPRINT_PATTERN = /^[a-f0-9]{64}$/;
|
||||||
|
const MIN_SUSPICIOUS_SAMPLES = 3;
|
||||||
|
const SCHEDULER_STALE_AFTER_MS = 30_000;
|
||||||
|
const ERROR_EVENT_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
|
const SUCCESS_EVENT_TTL_MS = 6 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
function finiteInteger(value: unknown, fallback = 0): number {
|
||||||
|
const numeric = Number(value);
|
||||||
|
return Number.isFinite(numeric) ? Math.max(0, Math.floor(numeric)) : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validFingerprint(value: unknown): string | null {
|
||||||
|
return typeof value === "string" && FINGERPRINT_PATTERN.test(value) ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function durationText(durationMs: number): string {
|
||||||
|
const totalSeconds = Math.max(0, Math.floor(durationMs / 1000));
|
||||||
|
if (totalSeconds < 60) {
|
||||||
|
return `${totalSeconds} s`;
|
||||||
|
}
|
||||||
|
const minutes = Math.floor(totalSeconds / 60);
|
||||||
|
const seconds = totalSeconds % 60;
|
||||||
|
return seconds > 0 ? `${minutes} min ${seconds} s` : `${minutes} min`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function byteText(bytes: number): string {
|
||||||
|
const value = Math.max(0, finiteInteger(bytes));
|
||||||
|
if (value < 1024) {
|
||||||
|
return `${value} B`;
|
||||||
|
}
|
||||||
|
const units = ["KB", "MB", "GB", "TB"];
|
||||||
|
let amount = value / 1024;
|
||||||
|
let unit = units[0];
|
||||||
|
for (let index = 1; index < units.length && amount >= 1024; index += 1) {
|
||||||
|
amount /= 1024;
|
||||||
|
unit = units[index];
|
||||||
|
}
|
||||||
|
return `${amount >= 10 ? amount.toFixed(0) : amount.toFixed(1)} ${unit}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function incidentEvent(
|
||||||
|
state: DownloadHealthState,
|
||||||
|
snapshot: DownloadHealthSnapshot,
|
||||||
|
now: number
|
||||||
|
): NotificationEvent {
|
||||||
|
const durationMs = Math.max(state.suspiciousDurationMs, now - state.incidentStartedAt);
|
||||||
|
return {
|
||||||
|
id: `health:stall:${snapshot.runFingerprint.slice(0, 16)}:${state.incidentStartedAt}`,
|
||||||
|
type: "download_stalled",
|
||||||
|
priority: "error",
|
||||||
|
createdAt: now,
|
||||||
|
expiresAt: now + ERROR_EVENT_TTL_MS,
|
||||||
|
attempts: 0,
|
||||||
|
nextAttemptAt: now,
|
||||||
|
payload: {
|
||||||
|
title: "Downloadstillstand bestätigt",
|
||||||
|
description: `Seit ${durationText(durationMs)} wurde kein bestätigter Downloadfortschritt erkannt.`,
|
||||||
|
color: 0xe67e22,
|
||||||
|
fields: [
|
||||||
|
{ name: "Dauer", value: durationText(durationMs), inline: true },
|
||||||
|
{ name: "Offene Pakete", value: String(snapshot.openPackages), inline: true },
|
||||||
|
{ name: "Offene Dateien", value: String(snapshot.openItems), inline: true },
|
||||||
|
{ name: "Bekannte Mindestmenge", value: byteText(snapshot.knownDownloadedBytes), inline: true },
|
||||||
|
{ name: "Aktiv / startfähig", value: `${snapshot.activeTasks} / ${snapshot.startableItems}`, inline: true },
|
||||||
|
{ name: "Technische Wiederherstellungen", value: String(snapshot.technicalRecoveryCount), inline: true }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function recoveryEvent(
|
||||||
|
state: DownloadHealthState,
|
||||||
|
snapshot: DownloadHealthSnapshot,
|
||||||
|
now: number
|
||||||
|
): NotificationEvent {
|
||||||
|
return {
|
||||||
|
id: `health:recovery:${snapshot.runFingerprint.slice(0, 16)}:${state.alertedAt}`,
|
||||||
|
type: "download_recovered",
|
||||||
|
priority: "success",
|
||||||
|
createdAt: now,
|
||||||
|
expiresAt: now + SUCCESS_EVENT_TTL_MS,
|
||||||
|
attempts: 0,
|
||||||
|
nextAttemptAt: now,
|
||||||
|
payload: {
|
||||||
|
title: "Download läuft wieder",
|
||||||
|
description: "Nach dem bestätigten Stillstand wurde neuer Fortschritt erkannt.",
|
||||||
|
color: 0x2ecc71,
|
||||||
|
fields: [
|
||||||
|
{ name: "Incident-Dauer", value: durationText(Math.max(0, now - state.incidentStartedAt)), inline: true },
|
||||||
|
{ name: "Aktive Downloads", value: String(snapshot.activeTasks), inline: true },
|
||||||
|
{ name: "Aktuelle Geschwindigkeit", value: `${byteText(snapshot.currentSpeedBps)}/s`, inline: true }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createDownloadHealthState(overrides: Partial<DownloadHealthState> = {}): DownloadHealthState {
|
||||||
|
return {
|
||||||
|
version: 1,
|
||||||
|
status: "idle",
|
||||||
|
runFingerprint: null,
|
||||||
|
queueFingerprint: null,
|
||||||
|
suspiciousDurationMs: 0,
|
||||||
|
suspiciousSamples: 0,
|
||||||
|
incidentStartedAt: 0,
|
||||||
|
incidentType: null,
|
||||||
|
alertedAt: 0,
|
||||||
|
lastAlertAt: 0,
|
||||||
|
cooldownUntil: 0,
|
||||||
|
recoverySamples: 0,
|
||||||
|
lastSampleAt: null,
|
||||||
|
downloadProgressSequence: 0,
|
||||||
|
itemCompletionSequence: 0,
|
||||||
|
lastPositiveByteAt: 0,
|
||||||
|
technicalRecoveryCount: 0,
|
||||||
|
restartPending: false,
|
||||||
|
restartFreshSamples: 0,
|
||||||
|
...overrides
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetForSnapshot(
|
||||||
|
state: DownloadHealthState,
|
||||||
|
snapshot: DownloadHealthSnapshot,
|
||||||
|
now: number
|
||||||
|
): DownloadHealthState {
|
||||||
|
return createDownloadHealthState({
|
||||||
|
runFingerprint: snapshot.runFingerprint,
|
||||||
|
queueFingerprint: snapshot.queueFingerprint,
|
||||||
|
lastAlertAt: state.lastAlertAt,
|
||||||
|
cooldownUntil: state.cooldownUntil,
|
||||||
|
lastSampleAt: now,
|
||||||
|
downloadProgressSequence: finiteInteger(snapshot.downloadProgressSequence),
|
||||||
|
itemCompletionSequence: finiteInteger(snapshot.itemCompletionSequence),
|
||||||
|
lastPositiveByteAt: finiteInteger(snapshot.lastPositiveByteAt),
|
||||||
|
technicalRecoveryCount: finiteInteger(snapshot.technicalRecoveryCount)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function endIncident(state: DownloadHealthState): DownloadHealthState {
|
||||||
|
return createDownloadHealthState({
|
||||||
|
lastAlertAt: state.lastAlertAt,
|
||||||
|
cooldownUntil: state.cooldownUntil,
|
||||||
|
downloadProgressSequence: state.downloadProgressSequence,
|
||||||
|
itemCompletionSequence: state.itemCompletionSequence,
|
||||||
|
lastPositiveByteAt: state.lastPositiveByteAt,
|
||||||
|
technicalRecoveryCount: state.technicalRecoveryCount
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function expectedWait(snapshot: DownloadHealthSnapshot, now: number): boolean {
|
||||||
|
return snapshot.paused
|
||||||
|
|| snapshot.reconnectUntil > now
|
||||||
|
|| snapshot.nextRetryAt > now
|
||||||
|
|| snapshot.providerCooldownUntil > now
|
||||||
|
|| snapshot.blockedOnDisk
|
||||||
|
|| snapshot.blockedOnThrottleUntil > now
|
||||||
|
|| snapshot.activePhaseDeadlineAt > now;
|
||||||
|
}
|
||||||
|
|
||||||
|
function suspicionType(snapshot: DownloadHealthSnapshot, now: number): DownloadHealthIncidentType | null {
|
||||||
|
if (snapshot.activeTasks === 0 && snapshot.startableItems > 0) {
|
||||||
|
if (snapshot.lastSchedulerTickAt <= 0 || now - snapshot.lastSchedulerTickAt >= SCHEDULER_STALE_AFTER_MS) {
|
||||||
|
return "scheduler";
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return "no_data";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateDownloadHealth(
|
||||||
|
previous: DownloadHealthState,
|
||||||
|
snapshot: DownloadHealthSnapshot,
|
||||||
|
nowValue: number,
|
||||||
|
options: DownloadHealthOptions
|
||||||
|
): DownloadHealthEvaluation {
|
||||||
|
const now = finiteInteger(nowValue);
|
||||||
|
let state = createDownloadHealthState(previous);
|
||||||
|
const events: NotificationEvent[] = [];
|
||||||
|
const runFingerprint = validFingerprint(snapshot.runFingerprint);
|
||||||
|
const queueFingerprint = validFingerprint(snapshot.queueFingerprint);
|
||||||
|
|
||||||
|
if (state.restartPending && !snapshot.runActive && !snapshot.terminalFailure && !snapshot.manualStop && !snapshot.shuttingDown) {
|
||||||
|
state.status = "suspended";
|
||||||
|
state.lastSampleAt = null;
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!snapshot.runActive || snapshot.openItems <= 0 || snapshot.terminalFailure || snapshot.manualStop || snapshot.shuttingDown || !runFingerprint || !queueFingerprint) {
|
||||||
|
state.downloadProgressSequence = Math.max(state.downloadProgressSequence, finiteInteger(snapshot.downloadProgressSequence));
|
||||||
|
state.itemCompletionSequence = Math.max(state.itemCompletionSequence, finiteInteger(snapshot.itemCompletionSequence));
|
||||||
|
state.lastPositiveByteAt = Math.max(state.lastPositiveByteAt, finiteInteger(snapshot.lastPositiveByteAt));
|
||||||
|
state.technicalRecoveryCount = Math.max(state.technicalRecoveryCount, finiteInteger(snapshot.technicalRecoveryCount));
|
||||||
|
return { state: endIncident(state), events };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.runFingerprint !== runFingerprint || state.queueFingerprint !== queueFingerprint) {
|
||||||
|
state = resetForSnapshot(state, snapshot, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
state.runFingerprint = runFingerprint;
|
||||||
|
state.queueFingerprint = queueFingerprint;
|
||||||
|
|
||||||
|
if (state.restartPending) {
|
||||||
|
state.restartFreshSamples += 1;
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
if (state.restartFreshSamples < 2) {
|
||||||
|
state.downloadProgressSequence = finiteInteger(snapshot.downloadProgressSequence);
|
||||||
|
state.itemCompletionSequence = finiteInteger(snapshot.itemCompletionSequence);
|
||||||
|
state.lastPositiveByteAt = finiteInteger(snapshot.lastPositiveByteAt);
|
||||||
|
state.technicalRecoveryCount = finiteInteger(snapshot.technicalRecoveryCount);
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
state.restartPending = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const progressSequence = finiteInteger(snapshot.downloadProgressSequence);
|
||||||
|
const completionSequence = finiteInteger(snapshot.itemCompletionSequence);
|
||||||
|
const positiveByteProgress = progressSequence > state.downloadProgressSequence;
|
||||||
|
const completionProgress = completionSequence > state.itemCompletionSequence;
|
||||||
|
const progress = positiveByteProgress || completionProgress;
|
||||||
|
state.downloadProgressSequence = Math.max(state.downloadProgressSequence, progressSequence);
|
||||||
|
state.itemCompletionSequence = Math.max(state.itemCompletionSequence, completionSequence);
|
||||||
|
state.lastPositiveByteAt = Math.max(state.lastPositiveByteAt, finiteInteger(snapshot.lastPositiveByteAt));
|
||||||
|
state.technicalRecoveryCount = Math.max(state.technicalRecoveryCount, finiteInteger(snapshot.technicalRecoveryCount));
|
||||||
|
|
||||||
|
if (state.alertedAt > 0 && progress) {
|
||||||
|
state.recoverySamples = completionProgress ? 2 : state.recoverySamples + 1;
|
||||||
|
if (state.recoverySamples >= 2) {
|
||||||
|
if (options.notifyOnRecovery) {
|
||||||
|
events.push(recoveryEvent(state, snapshot, now));
|
||||||
|
}
|
||||||
|
const recovered = resetForSnapshot(state, snapshot, now);
|
||||||
|
recovered.status = "healthy";
|
||||||
|
recovered.lastAlertAt = state.lastAlertAt;
|
||||||
|
recovered.cooldownUntil = state.cooldownUntil;
|
||||||
|
return { state: recovered, events };
|
||||||
|
}
|
||||||
|
state.status = "recovering";
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.alertedAt > 0) {
|
||||||
|
if (expectedWait(snapshot, now)) {
|
||||||
|
state.status = "expected_wait";
|
||||||
|
} else {
|
||||||
|
state.status = "alerted";
|
||||||
|
}
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expectedWait(snapshot, now)) {
|
||||||
|
state.status = "expected_wait";
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (progress) {
|
||||||
|
state.status = "healthy";
|
||||||
|
state.suspiciousDurationMs = 0;
|
||||||
|
state.suspiciousSamples = 0;
|
||||||
|
state.incidentStartedAt = 0;
|
||||||
|
state.incidentType = null;
|
||||||
|
state.recoverySamples = 0;
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
const incidentType = suspicionType(snapshot, now);
|
||||||
|
if (!incidentType) {
|
||||||
|
state.status = "healthy";
|
||||||
|
state.suspiciousDurationMs = 0;
|
||||||
|
state.suspiciousSamples = 0;
|
||||||
|
state.incidentStartedAt = 0;
|
||||||
|
state.incidentType = null;
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
const elapsed = state.lastSampleAt === null ? 0 : Math.max(0, now - state.lastSampleAt);
|
||||||
|
if (state.incidentStartedAt <= 0) {
|
||||||
|
state.incidentStartedAt = now;
|
||||||
|
}
|
||||||
|
state.suspiciousDurationMs += elapsed;
|
||||||
|
state.suspiciousSamples += 1;
|
||||||
|
state.incidentType = incidentType;
|
||||||
|
state.status = incidentType === "scheduler" ? "suspect_scheduler" : "suspect_no_data";
|
||||||
|
state.lastSampleAt = now;
|
||||||
|
|
||||||
|
const stallAfterMs = Math.max(0, finiteInteger(options.stallAfterMs, 90_000));
|
||||||
|
const cooldownMs = Math.max(0, finiteInteger(options.cooldownMs, 600_000));
|
||||||
|
const confirmed = state.suspiciousDurationMs >= stallAfterMs
|
||||||
|
&& state.suspiciousSamples >= MIN_SUSPICIOUS_SAMPLES;
|
||||||
|
if (confirmed && options.notifyOnStall && now >= state.cooldownUntil) {
|
||||||
|
state.status = "alerted";
|
||||||
|
state.alertedAt = now;
|
||||||
|
state.lastAlertAt = now;
|
||||||
|
state.cooldownUntil = now + cooldownMs;
|
||||||
|
state.recoverySamples = 0;
|
||||||
|
events.push(incidentEvent(state, snapshot, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
function persistedState(state: DownloadHealthState): Omit<DownloadHealthState, "restartPending" | "restartFreshSamples"> {
|
||||||
|
return {
|
||||||
|
version: 1,
|
||||||
|
status: state.status,
|
||||||
|
runFingerprint: state.runFingerprint,
|
||||||
|
queueFingerprint: state.queueFingerprint,
|
||||||
|
suspiciousDurationMs: finiteInteger(state.suspiciousDurationMs),
|
||||||
|
suspiciousSamples: finiteInteger(state.suspiciousSamples),
|
||||||
|
incidentStartedAt: finiteInteger(state.incidentStartedAt),
|
||||||
|
incidentType: state.incidentType,
|
||||||
|
alertedAt: finiteInteger(state.alertedAt),
|
||||||
|
lastAlertAt: finiteInteger(state.lastAlertAt),
|
||||||
|
cooldownUntil: finiteInteger(state.cooldownUntil),
|
||||||
|
recoverySamples: finiteInteger(state.recoverySamples),
|
||||||
|
lastSampleAt: state.lastSampleAt === null ? null : finiteInteger(state.lastSampleAt),
|
||||||
|
downloadProgressSequence: finiteInteger(state.downloadProgressSequence),
|
||||||
|
itemCompletionSequence: finiteInteger(state.itemCompletionSequence),
|
||||||
|
lastPositiveByteAt: finiteInteger(state.lastPositiveByteAt),
|
||||||
|
technicalRecoveryCount: finiteInteger(state.technicalRecoveryCount)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function saveDownloadHealthState(filePath: string, state: DownloadHealthState): void {
|
||||||
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||||
|
const tempPath = `${filePath}.tmp`;
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(tempPath, JSON.stringify(persistedState(state)), "utf8");
|
||||||
|
fs.renameSync(tempPath, filePath);
|
||||||
|
} catch (error) {
|
||||||
|
try {
|
||||||
|
fs.rmSync(tempPath, { force: true });
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadDownloadHealthState(filePath: string): DownloadHealthState {
|
||||||
|
try {
|
||||||
|
const raw = JSON.parse(fs.readFileSync(filePath, "utf8")) as Record<string, unknown>;
|
||||||
|
const runFingerprint = raw.runFingerprint === null ? null : validFingerprint(raw.runFingerprint);
|
||||||
|
const queueFingerprint = raw.queueFingerprint === null ? null : validFingerprint(raw.queueFingerprint);
|
||||||
|
if (raw.version !== 1 || !HEALTH_STATUSES.has(raw.status as DownloadHealthStatus)) {
|
||||||
|
return createDownloadHealthState();
|
||||||
|
}
|
||||||
|
if ((raw.runFingerprint !== null && !runFingerprint) || (raw.queueFingerprint !== null && !queueFingerprint)) {
|
||||||
|
return createDownloadHealthState();
|
||||||
|
}
|
||||||
|
const incidentType = INCIDENT_TYPES.has(raw.incidentType as DownloadHealthIncidentType)
|
||||||
|
? raw.incidentType as DownloadHealthIncidentType
|
||||||
|
: null;
|
||||||
|
return createDownloadHealthState({
|
||||||
|
status: raw.status as DownloadHealthStatus,
|
||||||
|
runFingerprint,
|
||||||
|
queueFingerprint,
|
||||||
|
suspiciousDurationMs: finiteInteger(raw.suspiciousDurationMs),
|
||||||
|
suspiciousSamples: finiteInteger(raw.suspiciousSamples),
|
||||||
|
incidentStartedAt: finiteInteger(raw.incidentStartedAt),
|
||||||
|
incidentType,
|
||||||
|
alertedAt: finiteInteger(raw.alertedAt),
|
||||||
|
lastAlertAt: finiteInteger(raw.lastAlertAt),
|
||||||
|
cooldownUntil: finiteInteger(raw.cooldownUntil),
|
||||||
|
recoverySamples: finiteInteger(raw.recoverySamples),
|
||||||
|
lastSampleAt: null,
|
||||||
|
downloadProgressSequence: finiteInteger(raw.downloadProgressSequence),
|
||||||
|
itemCompletionSequence: finiteInteger(raw.itemCompletionSequence),
|
||||||
|
lastPositiveByteAt: finiteInteger(raw.lastPositiveByteAt),
|
||||||
|
technicalRecoveryCount: finiteInteger(raw.technicalRecoveryCount),
|
||||||
|
restartPending: Boolean(runFingerprint && queueFingerprint),
|
||||||
|
restartFreshSamples: 0
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return createDownloadHealthState();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DownloadHealthMonitor {
|
||||||
|
private state: DownloadHealthState;
|
||||||
|
|
||||||
|
public constructor(private readonly filePath: string, initialState?: DownloadHealthState) {
|
||||||
|
this.state = initialState ? createDownloadHealthState(initialState) : loadDownloadHealthState(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getState(): DownloadHealthState {
|
||||||
|
return createDownloadHealthState(this.state);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async sample(
|
||||||
|
snapshot: DownloadHealthSnapshot,
|
||||||
|
now: number,
|
||||||
|
options: DownloadHealthOptions,
|
||||||
|
enqueue: (event: NotificationEvent) => Promise<void>
|
||||||
|
): Promise<DownloadHealthEvaluation> {
|
||||||
|
const evaluation = evaluateDownloadHealth(this.state, snapshot, now, options);
|
||||||
|
for (const event of evaluation.events) {
|
||||||
|
await enqueue(event);
|
||||||
|
}
|
||||||
|
saveDownloadHealthState(this.filePath, evaluation.state);
|
||||||
|
this.state = evaluation.state;
|
||||||
|
return evaluation;
|
||||||
|
}
|
||||||
|
}
|
||||||
+281
-99
@@ -1,6 +1,7 @@
|
|||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
import { EventEmitter } from "node:events";
|
import { EventEmitter } from "node:events";
|
||||||
import { v4 as uuidv4 } from "uuid";
|
import { v4 as uuidv4 } from "uuid";
|
||||||
import {
|
import {
|
||||||
@@ -93,6 +94,7 @@ import {
|
|||||||
} from "./statistics-ledger";
|
} from "./statistics-ledger";
|
||||||
import { finalizePackageResult } from "./package-telemetry";
|
import { finalizePackageResult } from "./package-telemetry";
|
||||||
import type { NotificationEvent } from "./notification-outbox";
|
import type { NotificationEvent } from "./notification-outbox";
|
||||||
|
import type { DownloadHealthSnapshot } from "./download-health-monitor";
|
||||||
import {
|
import {
|
||||||
buildHistoryEntry,
|
buildHistoryEntry,
|
||||||
buildPackageDigestEvents,
|
buildPackageDigestEvents,
|
||||||
@@ -118,8 +120,12 @@ type ActiveTask = {
|
|||||||
stallRetries?: number;
|
stallRetries?: number;
|
||||||
genericErrorRetries?: number;
|
genericErrorRetries?: number;
|
||||||
unrestrictRetries?: number;
|
unrestrictRetries?: number;
|
||||||
blockedOnDiskWrite?: boolean;
|
blockedOnDiskWrite?: boolean;
|
||||||
blockedOnDiskSince?: number;
|
blockedOnDiskSince?: number;
|
||||||
|
blockedOnThrottleUntil?: number;
|
||||||
|
phase?: "validating" | "downloading" | "integrity_check";
|
||||||
|
phaseStartedAt?: number;
|
||||||
|
phaseDeadlineAt?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
const DOWNLOAD_ACCOUNT_PROVIDERS: readonly DebridProvider[] = [
|
const DOWNLOAD_ACCOUNT_PROVIDERS: readonly DebridProvider[] = [
|
||||||
@@ -1960,7 +1966,23 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|
|
||||||
private itemCount = 0;
|
private itemCount = 0;
|
||||||
|
|
||||||
private lastSchedulerHeartbeatAt = 0;
|
private lastSchedulerHeartbeatAt = 0;
|
||||||
|
|
||||||
|
private lastSchedulerTickAt = 0;
|
||||||
|
|
||||||
|
private downloadProgressSequence = 0;
|
||||||
|
|
||||||
|
private itemCompletionSequence = 0;
|
||||||
|
|
||||||
|
private lastPositiveByteAt = 0;
|
||||||
|
|
||||||
|
private technicalRecoveryCount = 0;
|
||||||
|
|
||||||
|
private healthManualStop = false;
|
||||||
|
|
||||||
|
private healthShuttingDown = false;
|
||||||
|
|
||||||
|
private healthTerminalFailure = false;
|
||||||
|
|
||||||
private lastReconnectMarkAt = 0;
|
private lastReconnectMarkAt = 0;
|
||||||
|
|
||||||
@@ -2478,11 +2500,118 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return cloneSession(this.session);
|
return cloneSession(this.session);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getSummary(): DownloadSummary | null {
|
public getSummary(): DownloadSummary | null {
|
||||||
return this.summary;
|
return this.summary;
|
||||||
}
|
}
|
||||||
|
|
||||||
public isSessionRunning(): boolean {
|
public getDownloadHealthSnapshot(now = nowMs()): DownloadHealthSnapshot {
|
||||||
|
const openPackages = new Set<string>();
|
||||||
|
const queueParts: string[] = [];
|
||||||
|
const activeTasks: ActiveTask[] = [];
|
||||||
|
let openItems = 0;
|
||||||
|
let knownDownloadedBytes = 0;
|
||||||
|
let currentSpeedBps = 0;
|
||||||
|
let startableItems = 0;
|
||||||
|
let nextRetryAt = 0;
|
||||||
|
let providerCooldownUntil = 0;
|
||||||
|
|
||||||
|
for (const itemId of this.runItemIds) {
|
||||||
|
queueParts.push(itemId);
|
||||||
|
const item = this.session.items[itemId];
|
||||||
|
if (!item || isFinishedStatus(item.status)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const pkg = this.session.packages[item.packageId];
|
||||||
|
if (!pkg || pkg.cancelled || !pkg.enabled) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
openItems += 1;
|
||||||
|
openPackages.add(pkg.id);
|
||||||
|
knownDownloadedBytes += Math.max(0, Math.floor(Number(item.downloadedBytes) || 0));
|
||||||
|
currentSpeedBps += Math.max(0, Math.floor(Number(item.speedBps) || 0));
|
||||||
|
const active = this.activeTasks.get(itemId);
|
||||||
|
if (active && !active.abortController.signal.aborted) {
|
||||||
|
activeTasks.push(active);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (item.status !== "queued" && item.status !== "reconnect_wait") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const retryAt = this.retryAfterByItem.get(itemId) || 0;
|
||||||
|
const failureKey = this.getProviderFailureKeyForItem(item);
|
||||||
|
const cooldownAt = this.providerFailures.get(failureKey)?.cooldownUntil || 0;
|
||||||
|
if (retryAt > now) {
|
||||||
|
nextRetryAt = nextRetryAt === 0 ? retryAt : Math.min(nextRetryAt, retryAt);
|
||||||
|
}
|
||||||
|
if (cooldownAt > now) {
|
||||||
|
providerCooldownUntil = providerCooldownUntil === 0
|
||||||
|
? cooldownAt
|
||||||
|
: Math.min(providerCooldownUntil, cooldownAt);
|
||||||
|
}
|
||||||
|
if (retryAt <= now && cooldownAt <= now) {
|
||||||
|
startableItems += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const blockedOnDisk = activeTasks.length > 0 && activeTasks.every((active) => Boolean(active.blockedOnDiskWrite));
|
||||||
|
const throttleDeadlines = activeTasks.map((active) => active.blockedOnThrottleUntil || 0);
|
||||||
|
const blockedOnThrottleUntil = throttleDeadlines.length > 0 && throttleDeadlines.every((deadline) => deadline > now)
|
||||||
|
? Math.min(...throttleDeadlines)
|
||||||
|
: 0;
|
||||||
|
const phaseDeadlines = activeTasks.map((active) => active.phaseDeadlineAt || 0);
|
||||||
|
const activePhaseDeadlineAt = phaseDeadlines.length > 0 && phaseDeadlines.every((deadline) => deadline > now)
|
||||||
|
? Math.min(...phaseDeadlines)
|
||||||
|
: 0;
|
||||||
|
const queueFingerprint = createHash("sha256")
|
||||||
|
.update([...queueParts].sort().join("\n"))
|
||||||
|
.digest("hex");
|
||||||
|
const runParts = [...this.runPackageIds].map((packageId) => {
|
||||||
|
const generation = Math.max(1, Math.floor(Number(this.session.packages[packageId]?.resultGeneration) || 1));
|
||||||
|
return `${packageId}:${generation}`;
|
||||||
|
});
|
||||||
|
const runFingerprint = createHash("sha256")
|
||||||
|
.update(`${runParts.sort().join("\n")}|${queueFingerprint}`)
|
||||||
|
.digest("hex");
|
||||||
|
|
||||||
|
return {
|
||||||
|
runActive: this.session.running && openItems > 0,
|
||||||
|
runFingerprint,
|
||||||
|
queueFingerprint,
|
||||||
|
openItems,
|
||||||
|
openPackages: openPackages.size,
|
||||||
|
knownDownloadedBytes,
|
||||||
|
activeTasks: activeTasks.length,
|
||||||
|
startableItems,
|
||||||
|
lastSchedulerTickAt: this.lastSchedulerTickAt,
|
||||||
|
downloadProgressSequence: this.downloadProgressSequence,
|
||||||
|
itemCompletionSequence: this.itemCompletionSequence,
|
||||||
|
lastPositiveByteAt: this.lastPositiveByteAt,
|
||||||
|
technicalRecoveryCount: this.technicalRecoveryCount,
|
||||||
|
paused: this.session.paused,
|
||||||
|
reconnectUntil: this.session.reconnectUntil,
|
||||||
|
nextRetryAt: activeTasks.length === 0 && startableItems === 0 ? nextRetryAt : 0,
|
||||||
|
providerCooldownUntil: activeTasks.length === 0 && startableItems === 0 ? providerCooldownUntil : 0,
|
||||||
|
blockedOnDisk,
|
||||||
|
blockedOnThrottleUntil,
|
||||||
|
activePhaseDeadlineAt,
|
||||||
|
terminalFailure: this.healthTerminalFailure,
|
||||||
|
manualStop: this.healthManualStop,
|
||||||
|
shuttingDown: this.healthShuttingDown,
|
||||||
|
currentSpeedBps: this.session.running && !this.session.paused ? currentSpeedBps : 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private beginHealthRun(): void {
|
||||||
|
this.healthManualStop = false;
|
||||||
|
this.healthShuttingDown = false;
|
||||||
|
this.healthTerminalFailure = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public suspendDownloadHealthMonitoring(): void {
|
||||||
|
this.healthShuttingDown = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public isSessionRunning(): boolean {
|
||||||
return this.session.running;
|
return this.session.running;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5978,6 +6107,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async startPackages(packageIds: string[]): Promise<void> {
|
public async startPackages(packageIds: string[]): Promise<void> {
|
||||||
|
this.beginHealthRun();
|
||||||
this.ensureUsableDownloadAccount();
|
this.ensureUsableDownloadAccount();
|
||||||
const targetSet = new Set(packageIds);
|
const targetSet = new Set(packageIds);
|
||||||
for (const packageId of this.packagePostProcessTasks.keys()) {
|
for (const packageId of this.packagePostProcessTasks.keys()) {
|
||||||
@@ -6080,6 +6210,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async startItems(itemIds: string[]): Promise<void> {
|
public async startItems(itemIds: string[]): Promise<void> {
|
||||||
|
this.beginHealthRun();
|
||||||
this.ensureUsableDownloadAccount();
|
this.ensureUsableDownloadAccount();
|
||||||
const targetSet = new Set(itemIds);
|
const targetSet = new Set(itemIds);
|
||||||
|
|
||||||
@@ -6196,6 +6327,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (this.session.running) {
|
if (this.session.running) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
this.beginHealthRun();
|
||||||
this.ensureUsableDownloadAccount();
|
this.ensureUsableDownloadAccount();
|
||||||
this.schedulerGeneration += 1;
|
this.schedulerGeneration += 1;
|
||||||
|
|
||||||
@@ -6347,8 +6479,10 @@ export class DownloadManager extends EventEmitter {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public stop(options?: { parkForRestart?: boolean }): void {
|
public stop(options?: { parkForRestart?: boolean }): void {
|
||||||
const parkForRestart = options?.parkForRestart === true;
|
const parkForRestart = options?.parkForRestart === true;
|
||||||
|
this.healthManualStop = !parkForRestart;
|
||||||
|
this.healthShuttingDown = parkForRestart;
|
||||||
const abortReason: "stop" | "shutdown" = parkForRestart ? "shutdown" : "stop";
|
const abortReason: "stop" | "shutdown" = parkForRestart ? "shutdown" : "stop";
|
||||||
const keepExtraction = this.settings.autoExtractWhenStopped;
|
const keepExtraction = this.settings.autoExtractWhenStopped;
|
||||||
const wasRunning = this.session.running;
|
const wasRunning = this.session.running;
|
||||||
@@ -6426,6 +6560,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public prepareForShutdown(): void {
|
public prepareForShutdown(): void {
|
||||||
|
this.healthShuttingDown = true;
|
||||||
logger.info(`Shutdown-Vorbereitung gestartet: active=${this.activeTasks.size}, running=${this.session.running}, paused=${this.session.paused}`);
|
logger.info(`Shutdown-Vorbereitung gestartet: active=${this.activeTasks.size}, running=${this.session.running}, paused=${this.session.paused}`);
|
||||||
this.updateStatisticsActivity(nowMs());
|
this.updateStatisticsActivity(nowMs());
|
||||||
this.rotationListenerActive = false;
|
this.rotationListenerActive = false;
|
||||||
@@ -6922,9 +7057,14 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|
|
||||||
private lastSpeedPruneAt = 0;
|
private lastSpeedPruneAt = 0;
|
||||||
|
|
||||||
private recordSpeed(bytes: number, packageId: string = ""): void {
|
private recordSpeed(bytes: number, packageId: string = ""): void {
|
||||||
const now = nowMs();
|
const now = nowMs();
|
||||||
if (bytes > 0 && this.consecutiveReconnects > 0) {
|
if (!Number.isFinite(bytes) || bytes <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.downloadProgressSequence += 1;
|
||||||
|
this.lastPositiveByteAt = now;
|
||||||
|
if (bytes > 0 && this.consecutiveReconnects > 0) {
|
||||||
this.consecutiveReconnects = 0;
|
this.consecutiveReconnects = 0;
|
||||||
}
|
}
|
||||||
const bucket = now - (now % 120);
|
const bucket = now - (now % 120);
|
||||||
@@ -6955,8 +7095,9 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.statisticsDirty = true;
|
this.statisticsDirty = true;
|
||||||
this.statisticsUrgent = true;
|
this.statisticsUrgent = true;
|
||||||
}
|
}
|
||||||
if (status === "completed" && previous !== "completed") {
|
if (status === "completed" && previous !== "completed") {
|
||||||
this.sessionCompletedFiles += 1;
|
this.itemCompletionSequence += 1;
|
||||||
|
this.sessionCompletedFiles += 1;
|
||||||
this.settings.totalCompletedFilesAllTime = Math.max(0, Number(this.settings.totalCompletedFilesAllTime || 0)) + 1;
|
this.settings.totalCompletedFilesAllTime = Math.max(0, Number(this.settings.totalCompletedFilesAllTime || 0)) + 1;
|
||||||
this.invalidateStatsCache();
|
this.invalidateStatsCache();
|
||||||
}
|
}
|
||||||
@@ -9108,6 +9249,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
try {
|
try {
|
||||||
while (this.session.running && this.schedulerGeneration === myGeneration) {
|
while (this.session.running && this.schedulerGeneration === myGeneration) {
|
||||||
const now = nowMs();
|
const now = nowMs();
|
||||||
|
this.lastSchedulerTickAt = now;
|
||||||
this.updateStatisticsActivity(now);
|
this.updateStatisticsActivity(now);
|
||||||
if (now - this.lastSchedulerHeartbeatAt >= 60000) {
|
if (now - this.lastSchedulerHeartbeatAt >= 60000) {
|
||||||
this.lastSchedulerHeartbeatAt = now;
|
this.lastSchedulerHeartbeatAt = now;
|
||||||
@@ -9246,8 +9388,9 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`);
|
logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`);
|
||||||
for (const active of this.activeTasks.values()) {
|
this.technicalRecoveryCount += 1;
|
||||||
|
for (const active of this.activeTasks.values()) {
|
||||||
if (active.abortController.signal.aborted) {
|
if (active.abortController.signal.aborted) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -9576,10 +9719,14 @@ export class DownloadManager extends EventEmitter {
|
|||||||
abortController: new AbortController(),
|
abortController: new AbortController(),
|
||||||
abortReason: "none",
|
abortReason: "none",
|
||||||
resumable: true,
|
resumable: true,
|
||||||
nonResumableCounted: false,
|
nonResumableCounted: false,
|
||||||
blockedOnDiskWrite: false,
|
blockedOnDiskWrite: false,
|
||||||
blockedOnDiskSince: 0
|
blockedOnDiskSince: 0,
|
||||||
};
|
blockedOnThrottleUntil: 0,
|
||||||
|
phase: "validating",
|
||||||
|
phaseStartedAt: item.updatedAt,
|
||||||
|
phaseDeadlineAt: item.updatedAt + getUnrestrictTimeoutMs() + 15_000
|
||||||
|
};
|
||||||
this.activeTasks.set(itemId, active);
|
this.activeTasks.set(itemId, active);
|
||||||
this.notePacedStartForItem(item, nowMs());
|
this.notePacedStartForItem(item, nowMs());
|
||||||
this.emitState();
|
this.emitState();
|
||||||
@@ -9701,6 +9848,9 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.settings,
|
this.settings,
|
||||||
item.url
|
item.url
|
||||||
);
|
);
|
||||||
|
active.phase = "validating";
|
||||||
|
active.phaseStartedAt = nowMs();
|
||||||
|
active.phaseDeadlineAt = active.phaseStartedAt + unrestrictTimeoutMs;
|
||||||
const unrestrictTimeoutSignal = AbortSignal.timeout(unrestrictTimeoutMs);
|
const unrestrictTimeoutSignal = AbortSignal.timeout(unrestrictTimeoutMs);
|
||||||
const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]);
|
const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]);
|
||||||
let unrestricted;
|
let unrestricted;
|
||||||
@@ -9805,6 +9955,9 @@ export class DownloadManager extends EventEmitter {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
item.status = "downloading";
|
item.status = "downloading";
|
||||||
|
active.phase = "downloading";
|
||||||
|
active.phaseStartedAt = nowMs();
|
||||||
|
active.phaseDeadlineAt = 0;
|
||||||
const pLabel = unrestricted.providerLabel;
|
const pLabel = unrestricted.providerLabel;
|
||||||
item.fullStatus = "Starte...";
|
item.fullStatus = "Starte...";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
@@ -9855,10 +10008,18 @@ export class DownloadManager extends EventEmitter {
|
|||||||
item.status = "integrity_check";
|
item.status = "integrity_check";
|
||||||
item.fullStatus = "CRC-Check läuft";
|
item.fullStatus = "CRC-Check läuft";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
this.emitState();
|
this.emitState();
|
||||||
|
|
||||||
const integrityStartedAt = nowMs();
|
const integrityStartedAt = nowMs();
|
||||||
const validation = await validateFileAgainstManifest(item.targetPath, pkg.outputDir);
|
const integrityBytes = Math.max(0, Number(item.downloadedBytes || item.totalBytes || 0));
|
||||||
|
const integrityBudgetMs = Math.max(
|
||||||
|
120_000,
|
||||||
|
Math.min(30 * 60 * 1000, 60_000 + Math.ceil(integrityBytes / (25 * 1024 * 1024)) * 1000)
|
||||||
|
);
|
||||||
|
active.phase = "integrity_check";
|
||||||
|
active.phaseStartedAt = integrityStartedAt;
|
||||||
|
active.phaseDeadlineAt = integrityStartedAt + integrityBudgetMs;
|
||||||
|
const validation = await validateFileAgainstManifest(item.targetPath, pkg.outputDir);
|
||||||
if (active.abortController.signal.aborted) {
|
if (active.abortController.signal.aborted) {
|
||||||
throw new Error(`aborted:${active.abortReason}`);
|
throw new Error(`aborted:${active.abortReason}`);
|
||||||
}
|
}
|
||||||
@@ -9881,9 +10042,12 @@ export class DownloadManager extends EventEmitter {
|
|||||||
item.progressPercent = 0;
|
item.progressPercent = 0;
|
||||||
item.downloadedBytes = 0;
|
item.downloadedBytes = 0;
|
||||||
item.totalBytes = mergeKnownTotalBytes(item.totalBytes, unrestricted.fileSize);
|
item.totalBytes = mergeKnownTotalBytes(item.totalBytes, unrestricted.fileSize);
|
||||||
this.emitState();
|
this.emitState();
|
||||||
await sleep(300);
|
await sleep(300);
|
||||||
continue;
|
active.phase = "downloading";
|
||||||
|
active.phaseStartedAt = nowMs();
|
||||||
|
active.phaseDeadlineAt = 0;
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
throw new Error(`Integritätsprüfung fehlgeschlagen (${validation.message})`);
|
throw new Error(`Integritätsprüfung fehlgeschlagen (${validation.message})`);
|
||||||
}
|
}
|
||||||
@@ -11178,7 +11342,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
||||||
await this.applySpeedLimit(buffer.length, windowBytes, windowStarted, active.abortController.signal);
|
await this.applySpeedLimit(buffer.length, windowBytes, windowStarted, active);
|
||||||
if (active.abortController.signal.aborted) {
|
if (active.abortController.signal.aborted) {
|
||||||
throw new Error(`aborted:${active.abortReason}`);
|
throw new Error(`aborted:${active.abortReason}`);
|
||||||
}
|
}
|
||||||
@@ -12241,45 +12405,53 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async applyGlobalSpeedLimit(chunkBytes: number, bytesPerSecond: number, signal?: AbortSignal): Promise<void> {
|
private async applyGlobalSpeedLimit(chunkBytes: number, bytesPerSecond: number, active?: ActiveTask): Promise<void> {
|
||||||
const task = this.globalSpeedLimitQueue
|
const signal = active?.abortController.signal;
|
||||||
|
const task = this.globalSpeedLimitQueue
|
||||||
.catch(() => undefined)
|
.catch(() => undefined)
|
||||||
.then(async () => {
|
.then(async () => {
|
||||||
if (signal?.aborted) {
|
if (signal?.aborted) {
|
||||||
throw new Error("aborted:speed_limit");
|
throw new Error("aborted:speed_limit");
|
||||||
}
|
}
|
||||||
const now = nowMs();
|
const now = nowMs();
|
||||||
const waitMs = Math.max(0, this.globalSpeedLimitNextAt - now);
|
const waitMs = Math.max(0, this.globalSpeedLimitNextAt - now);
|
||||||
if (waitMs > 0) {
|
if (waitMs > 0) {
|
||||||
await new Promise<void>((resolve, reject) => {
|
if (active) {
|
||||||
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
active.blockedOnThrottleUntil = now + waitMs;
|
||||||
timer = null;
|
}
|
||||||
if (signal) {
|
try {
|
||||||
signal.removeEventListener("abort", onAbort);
|
await new Promise<void>((resolve, reject) => {
|
||||||
}
|
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
||||||
resolve();
|
timer = null;
|
||||||
}, waitMs);
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
const onAbort = (): void => {
|
}, waitMs);
|
||||||
if (timer) {
|
|
||||||
clearTimeout(timer);
|
const onAbort = (): void => {
|
||||||
timer = null;
|
if (timer) {
|
||||||
}
|
clearTimeout(timer);
|
||||||
signal?.removeEventListener("abort", onAbort);
|
timer = null;
|
||||||
reject(new Error("aborted:speed_limit"));
|
}
|
||||||
};
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
reject(new Error("aborted:speed_limit"));
|
||||||
if (signal) {
|
};
|
||||||
if (signal.aborted) {
|
|
||||||
onAbort();
|
if (signal) {
|
||||||
return;
|
if (signal.aborted) {
|
||||||
}
|
onAbort();
|
||||||
signal.addEventListener("abort", onAbort, { once: true });
|
return;
|
||||||
}
|
}
|
||||||
});
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
}
|
}
|
||||||
|
});
|
||||||
if (signal?.aborted) {
|
} finally {
|
||||||
|
if (active) {
|
||||||
|
active.blockedOnThrottleUntil = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (signal?.aborted) {
|
||||||
throw new Error("aborted:speed_limit");
|
throw new Error("aborted:speed_limit");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -12292,7 +12464,8 @@ export class DownloadManager extends EventEmitter {
|
|||||||
await task;
|
await task;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async applySpeedLimit(chunkBytes: number, localWindowBytes: number, localWindowStarted: number, signal?: AbortSignal): Promise<void> {
|
private async applySpeedLimit(chunkBytes: number, localWindowBytes: number, localWindowStarted: number, active?: ActiveTask): Promise<void> {
|
||||||
|
const signal = active?.abortController.signal;
|
||||||
const limitKbps = this.getEffectiveSpeedLimitKbps();
|
const limitKbps = this.getEffectiveSpeedLimitKbps();
|
||||||
if (limitKbps <= 0) {
|
if (limitKbps <= 0) {
|
||||||
return;
|
return;
|
||||||
@@ -12304,40 +12477,48 @@ export class DownloadManager extends EventEmitter {
|
|||||||
const projected = localWindowBytes + chunkBytes;
|
const projected = localWindowBytes + chunkBytes;
|
||||||
const allowed = bytesPerSecond * elapsed;
|
const allowed = bytesPerSecond * elapsed;
|
||||||
if (projected > allowed) {
|
if (projected > allowed) {
|
||||||
const sleepMs = Math.ceil(((projected - allowed) / bytesPerSecond) * 1000);
|
const sleepMs = Math.ceil(((projected - allowed) / bytesPerSecond) * 1000);
|
||||||
if (sleepMs > 0) {
|
if (sleepMs > 0) {
|
||||||
await new Promise<void>((resolve, reject) => {
|
const boundedSleepMs = Math.min(300, sleepMs);
|
||||||
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
if (active) {
|
||||||
timer = null;
|
active.blockedOnThrottleUntil = nowMs() + boundedSleepMs;
|
||||||
if (signal) {
|
}
|
||||||
signal.removeEventListener("abort", onAbort);
|
try {
|
||||||
}
|
await new Promise<void>((resolve, reject) => {
|
||||||
resolve();
|
let timer: NodeJS.Timeout | null = setTimeout(() => {
|
||||||
}, Math.min(300, sleepMs));
|
timer = null;
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
const onAbort = (): void => {
|
resolve();
|
||||||
if (timer) {
|
}, boundedSleepMs);
|
||||||
clearTimeout(timer);
|
|
||||||
timer = null;
|
const onAbort = (): void => {
|
||||||
}
|
if (timer) {
|
||||||
signal?.removeEventListener("abort", onAbort);
|
clearTimeout(timer);
|
||||||
reject(new Error("aborted:speed_limit"));
|
timer = null;
|
||||||
};
|
}
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
if (signal) {
|
reject(new Error("aborted:speed_limit"));
|
||||||
if (signal.aborted) {
|
};
|
||||||
onAbort();
|
|
||||||
return;
|
if (signal) {
|
||||||
}
|
if (signal.aborted) {
|
||||||
signal.addEventListener("abort", onAbort, { once: true });
|
onAbort();
|
||||||
}
|
return;
|
||||||
});
|
}
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
if (active) {
|
||||||
|
active.blockedOnThrottleUntil = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.applyGlobalSpeedLimit(chunkBytes, bytesPerSecond, signal);
|
await this.applyGlobalSpeedLimit(chunkBytes, bytesPerSecond, active);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async findReadyArchiveSets(pkg: PackageEntry): Promise<Set<string>> {
|
private async findReadyArchiveSets(pkg: PackageEntry): Promise<Set<string>> {
|
||||||
@@ -13882,9 +14063,10 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.session.runStartedAt = 0;
|
this.session.runStartedAt = 0;
|
||||||
const total = this.runItemIds.size;
|
const total = this.runItemIds.size;
|
||||||
const outcomes = Array.from(this.runOutcomes.values());
|
const outcomes = Array.from(this.runOutcomes.values());
|
||||||
const success = outcomes.filter((status) => status === "completed").length;
|
const success = outcomes.filter((status) => status === "completed").length;
|
||||||
const failed = outcomes.filter((status) => status === "failed").length;
|
const failed = outcomes.filter((status) => status === "failed").length;
|
||||||
const cancelled = outcomes.filter((status) => status === "cancelled").length;
|
const cancelled = outcomes.filter((status) => status === "cancelled").length;
|
||||||
|
this.healthTerminalFailure = failed > 0;
|
||||||
const extracted = this.runCompletedPackages.size;
|
const extracted = this.runCompletedPackages.size;
|
||||||
const duration = runStartedAt > 0 ? Math.max(1, Math.floor((completedAt - runStartedAt) / 1000)) : 1;
|
const duration = runStartedAt > 0 ? Math.max(1, Math.floor((completedAt - runStartedAt) / 1000)) : 1;
|
||||||
const avgSpeed = Math.floor(this.session.totalDownloadedBytes / duration);
|
const avgSpeed = Math.floor(this.session.totalDownloadedBytes / duration);
|
||||||
|
|||||||
+3
-1
@@ -702,6 +702,7 @@ export interface StoragePaths {
|
|||||||
historyFile: string;
|
historyFile: string;
|
||||||
statisticsFile: string;
|
statisticsFile: string;
|
||||||
notificationOutboxFile: string;
|
notificationOutboxFile: string;
|
||||||
|
notificationHealthFile: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createStoragePaths(baseDir: string): StoragePaths {
|
export function createStoragePaths(baseDir: string): StoragePaths {
|
||||||
@@ -711,7 +712,8 @@ export function createStoragePaths(baseDir: string): StoragePaths {
|
|||||||
sessionFile: path.join(baseDir, "rd_session_state.json"),
|
sessionFile: path.join(baseDir, "rd_session_state.json"),
|
||||||
historyFile: path.join(baseDir, "rd_history.json"),
|
historyFile: path.join(baseDir, "rd_history.json"),
|
||||||
statisticsFile: path.join(baseDir, "rd_statistics.json"),
|
statisticsFile: path.join(baseDir, "rd_statistics.json"),
|
||||||
notificationOutboxFile: path.join(baseDir, "rd_notification_outbox.json")
|
notificationOutboxFile: path.join(baseDir, "rd_notification_outbox.json"),
|
||||||
|
notificationHealthFile: path.join(baseDir, "rd_notification_health.json")
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,452 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
DownloadHealthMonitor,
|
||||||
|
createDownloadHealthState,
|
||||||
|
evaluateDownloadHealth,
|
||||||
|
loadDownloadHealthState,
|
||||||
|
saveDownloadHealthState,
|
||||||
|
type DownloadHealthSnapshot,
|
||||||
|
type DownloadHealthState
|
||||||
|
} from "../src/main/download-health-monitor";
|
||||||
|
|
||||||
|
const RUN_FINGERPRINT = "a".repeat(64);
|
||||||
|
const QUEUE_FINGERPRINT = "b".repeat(64);
|
||||||
|
const OTHER_RUN_FINGERPRINT = "c".repeat(64);
|
||||||
|
const OTHER_QUEUE_FINGERPRINT = "d".repeat(64);
|
||||||
|
const tempDirs: string[] = [];
|
||||||
|
|
||||||
|
function snapshot(overrides: Partial<DownloadHealthSnapshot> = {}): DownloadHealthSnapshot {
|
||||||
|
return {
|
||||||
|
runActive: true,
|
||||||
|
runFingerprint: RUN_FINGERPRINT,
|
||||||
|
queueFingerprint: QUEUE_FINGERPRINT,
|
||||||
|
openItems: 2,
|
||||||
|
openPackages: 1,
|
||||||
|
knownDownloadedBytes: 4096,
|
||||||
|
activeTasks: 1,
|
||||||
|
startableItems: 0,
|
||||||
|
lastSchedulerTickAt: 0,
|
||||||
|
downloadProgressSequence: 0,
|
||||||
|
itemCompletionSequence: 0,
|
||||||
|
lastPositiveByteAt: 0,
|
||||||
|
technicalRecoveryCount: 0,
|
||||||
|
paused: false,
|
||||||
|
reconnectUntil: 0,
|
||||||
|
nextRetryAt: 0,
|
||||||
|
providerCooldownUntil: 0,
|
||||||
|
blockedOnDisk: false,
|
||||||
|
blockedOnThrottleUntil: 0,
|
||||||
|
activePhaseDeadlineAt: 0,
|
||||||
|
terminalFailure: false,
|
||||||
|
manualStop: false,
|
||||||
|
shuttingDown: false,
|
||||||
|
currentSpeedBps: 0,
|
||||||
|
...overrides
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function evaluate(
|
||||||
|
state: DownloadHealthState,
|
||||||
|
current: DownloadHealthSnapshot,
|
||||||
|
now: number,
|
||||||
|
overrides: Partial<Parameters<typeof evaluateDownloadHealth>[3]> = {}
|
||||||
|
) {
|
||||||
|
return evaluateDownloadHealth(state, current, now, {
|
||||||
|
stallAfterMs: 90_000,
|
||||||
|
cooldownMs: 600_000,
|
||||||
|
notifyOnStall: true,
|
||||||
|
notifyOnRecovery: true,
|
||||||
|
...overrides
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function sampleTimes(
|
||||||
|
initial: DownloadHealthState,
|
||||||
|
times: number[],
|
||||||
|
current: DownloadHealthSnapshot = snapshot()
|
||||||
|
) {
|
||||||
|
let state = initial;
|
||||||
|
const events = [];
|
||||||
|
for (const now of times) {
|
||||||
|
const result = evaluate(state, current, now);
|
||||||
|
state = result.state;
|
||||||
|
events.push(...result.events);
|
||||||
|
}
|
||||||
|
return { state, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
function alertedState(now = 90_000): DownloadHealthState {
|
||||||
|
return sampleTimes(createDownloadHealthState(), [0, 45_000, now]).state;
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const dir of tempDirs.splice(0)) {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("evaluateDownloadHealth", () => {
|
||||||
|
it("keeps a 20 to 30 second silent interval below the alert boundary", () => {
|
||||||
|
const result = sampleTimes(createDownloadHealthState(), [0, 15_000, 30_000]);
|
||||||
|
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.status).toBe("suspect_no_data");
|
||||||
|
expect(result.state.suspiciousDurationMs).toBe(30_000);
|
||||||
|
expect(result.state.suspiciousSamples).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("confirms a no-data stall only after 90 seconds and at least three suspicious samples", () => {
|
||||||
|
const before = sampleTimes(createDownloadHealthState(), [0, 45_000]);
|
||||||
|
const result = evaluate(before.state, snapshot(), 90_000);
|
||||||
|
|
||||||
|
expect(before.events).toEqual([]);
|
||||||
|
expect(result.state.status).toBe("alerted");
|
||||||
|
expect(result.events).toEqual([
|
||||||
|
expect.objectContaining({ type: "download_stalled", priority: "error" })
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not alert from elapsed time until the third suspicious sample", () => {
|
||||||
|
const result = sampleTimes(createDownloadHealthState(), [0, 90_000]);
|
||||||
|
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.suspiciousDurationMs).toBe(90_000);
|
||||||
|
expect(result.state.suspiciousSamples).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("classifies a startable queue with no scheduler as a scheduler suspicion", () => {
|
||||||
|
const result = evaluate(createDownloadHealthState(), snapshot({
|
||||||
|
activeTasks: 0,
|
||||||
|
startableItems: 2,
|
||||||
|
lastSchedulerTickAt: 0
|
||||||
|
}), 45_000);
|
||||||
|
|
||||||
|
expect(result.state.status).toBe("suspect_scheduler");
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats a recent scheduler tick without an active task as healthy startup activity", () => {
|
||||||
|
const result = evaluate(createDownloadHealthState(), snapshot({
|
||||||
|
activeTasks: 0,
|
||||||
|
startableItems: 2,
|
||||||
|
lastSchedulerTickAt: 29_000
|
||||||
|
}), 30_000);
|
||||||
|
|
||||||
|
expect(result.state.status).toBe("healthy");
|
||||||
|
expect(result.state.suspiciousSamples).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["pause", { paused: true }],
|
||||||
|
["reconnect", { reconnectUntil: 120_000 }],
|
||||||
|
["future retry", { activeTasks: 0, startableItems: 0, nextRetryAt: 120_000 }],
|
||||||
|
["provider cooldown", { activeTasks: 0, startableItems: 0, providerCooldownUntil: 120_000 }],
|
||||||
|
["disk wait", { blockedOnDisk: true }],
|
||||||
|
["bandwidth throttle", { blockedOnThrottleUntil: 120_000 }],
|
||||||
|
["valid phase deadline", { activePhaseDeadlineAt: 120_000 }]
|
||||||
|
])("freezes accumulated suspicion during %s", (_name, waitState) => {
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||||
|
const waiting = evaluate(suspicious.state, snapshot(waitState), 60_000);
|
||||||
|
const resumed = evaluate(waiting.state, snapshot(), 90_000);
|
||||||
|
|
||||||
|
expect(waiting.state.status).toBe("expected_wait");
|
||||||
|
expect(waiting.state.suspiciousDurationMs).toBe(30_000);
|
||||||
|
expect(waiting.state.suspiciousSamples).toBe(2);
|
||||||
|
expect(resumed.events).toEqual([]);
|
||||||
|
expect(resumed.state.suspiciousDurationMs).toBe(60_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resets suspicion after a positive byte sequence", () => {
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||||
|
const result = evaluate(suspicious.state, snapshot({
|
||||||
|
downloadProgressSequence: 1,
|
||||||
|
lastPositiveByteAt: 45_000
|
||||||
|
}), 45_000);
|
||||||
|
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.status).toBe("healthy");
|
||||||
|
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||||
|
expect(result.state.suspiciousSamples).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resets suspicion after a successful item completion sequence", () => {
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]);
|
||||||
|
const result = evaluate(suspicious.state, snapshot({ itemCompletionSequence: 1 }), 45_000);
|
||||||
|
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.status).toBe("healthy");
|
||||||
|
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores speed, progress, item timestamps and global totals as progress evidence", () => {
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]);
|
||||||
|
const current = {
|
||||||
|
...snapshot({ currentSpeedBps: 900_000_000 }),
|
||||||
|
progressPercent: 99,
|
||||||
|
updatedAt: 90_000,
|
||||||
|
totalDownloadedBytes: 10_000_000_000
|
||||||
|
} as DownloadHealthSnapshot;
|
||||||
|
const result = evaluate(suspicious.state, current, 90_000);
|
||||||
|
|
||||||
|
expect(result.state.status).toBe("alerted");
|
||||||
|
expect(result.events).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat sequence decreases or a technical recovery attempt as progress", () => {
|
||||||
|
const initial = createDownloadHealthState({
|
||||||
|
downloadProgressSequence: 8,
|
||||||
|
itemCompletionSequence: 3
|
||||||
|
});
|
||||||
|
const result = sampleTimes(initial, [0, 45_000, 90_000], snapshot({
|
||||||
|
downloadProgressSequence: 2,
|
||||||
|
itemCompletionSequence: 1,
|
||||||
|
technicalRecoveryCount: 1
|
||||||
|
}));
|
||||||
|
|
||||||
|
expect(result.state.status).toBe("alerted");
|
||||||
|
expect(result.events).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires two positive-byte samples before recovering an alerted incident", () => {
|
||||||
|
const alerted = alertedState();
|
||||||
|
const first = evaluate(alerted, snapshot({
|
||||||
|
downloadProgressSequence: 1,
|
||||||
|
lastPositiveByteAt: 105_000
|
||||||
|
}), 105_000);
|
||||||
|
const second = evaluate(first.state, snapshot({
|
||||||
|
downloadProgressSequence: 2,
|
||||||
|
lastPositiveByteAt: 120_000
|
||||||
|
}), 120_000);
|
||||||
|
|
||||||
|
expect(first.state.status).toBe("recovering");
|
||||||
|
expect(first.events).toEqual([]);
|
||||||
|
expect(second.state.status).toBe("healthy");
|
||||||
|
expect(second.events).toEqual([
|
||||||
|
expect.objectContaining({ type: "download_recovered", priority: "success" })
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("recovers immediately after a successful item completion", () => {
|
||||||
|
const result = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000);
|
||||||
|
|
||||||
|
expect(result.state.status).toBe("healthy");
|
||||||
|
expect(result.events).toEqual([
|
||||||
|
expect.objectContaining({ type: "download_recovered" })
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["terminal failure", { terminalFailure: true }],
|
||||||
|
["manual stop", { runActive: false, manualStop: true }],
|
||||||
|
["shutdown", { runActive: false, shuttingDown: true }]
|
||||||
|
])("closes an alerted incident without recovery after %s", (_name, endState) => {
|
||||||
|
const result = evaluate(alertedState(), snapshot(endState), 105_000);
|
||||||
|
|
||||||
|
expect(result.state.status).toBe("idle");
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.alertedAt).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies a ten-minute cooldown before another incident event", () => {
|
||||||
|
const firstAlert = alertedState();
|
||||||
|
const recovered = evaluate(firstAlert, snapshot({ itemCompletionSequence: 1 }), 105_000).state;
|
||||||
|
const duringCooldown = sampleTimes(recovered, [120_000, 165_000, 210_000]);
|
||||||
|
const afterCooldown = evaluate(duringCooldown.state, snapshot(), 690_000);
|
||||||
|
|
||||||
|
expect(duringCooldown.events).toEqual([]);
|
||||||
|
expect(duringCooldown.state.status).toBe("suspect_no_data");
|
||||||
|
expect(afterCooldown.events).toEqual([
|
||||||
|
expect.objectContaining({ type: "download_stalled" })
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the incident event id stable when outbox persistence rejects the state transition", () => {
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||||
|
const firstAttempt = evaluate(suspicious, snapshot(), 90_000);
|
||||||
|
const retryAttempt = evaluate(suspicious, snapshot(), 105_000);
|
||||||
|
|
||||||
|
expect(firstAttempt.events[0].id).toBe(retryAttempt.events[0].id);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("omits identifiers, paths, URLs, providers and accounts from incident and recovery payloads", () => {
|
||||||
|
const incident = evaluate(sampleTimes(createDownloadHealthState(), [0, 45_000]).state, snapshot(), 90_000).events[0];
|
||||||
|
const recovered = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000).events[0];
|
||||||
|
const serialized = JSON.stringify([incident, recovered]);
|
||||||
|
|
||||||
|
expect(serialized).not.toMatch(/https?:|\\|\/downloads\/|provider|account|item-|package-/i);
|
||||||
|
expect(incident.payload.fields).toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({ name: "Offene Dateien", value: "2" }),
|
||||||
|
expect.objectContaining({ name: "Technische Wiederherstellungen", value: "0" })
|
||||||
|
]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honors disabled incident and recovery settings independently", () => {
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||||
|
const disabledIncident = evaluate(suspicious, snapshot(), 90_000, { notifyOnStall: false });
|
||||||
|
const disabledRecovery = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000, { notifyOnRecovery: false });
|
||||||
|
|
||||||
|
expect(disabledIncident.events).toEqual([]);
|
||||||
|
expect(disabledIncident.state.status).toBe("suspect_no_data");
|
||||||
|
expect(disabledRecovery.events).toEqual([]);
|
||||||
|
expect(disabledRecovery.state.status).toBe("healthy");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("download health restart persistence", () => {
|
||||||
|
it("preserves a persisted incident while startup is still idle", () => {
|
||||||
|
const persisted = createDownloadHealthState({
|
||||||
|
status: "alerted",
|
||||||
|
runFingerprint: RUN_FINGERPRINT,
|
||||||
|
queueFingerprint: QUEUE_FINGERPRINT,
|
||||||
|
suspiciousDurationMs: 90_000,
|
||||||
|
suspiciousSamples: 3,
|
||||||
|
incidentStartedAt: 10_000,
|
||||||
|
alertedAt: 90_000,
|
||||||
|
restartPending: true
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = evaluate(persisted, snapshot({ runActive: false, openItems: 0 }), 100_000);
|
||||||
|
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.status).toBe("suspended");
|
||||||
|
expect(result.state.runFingerprint).toBe(RUN_FINGERPRINT);
|
||||||
|
expect(result.state.queueFingerprint).toBe(QUEUE_FINGERPRINT);
|
||||||
|
expect(result.state.restartPending).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires two fresh samples before re-alerting the same persisted fingerprint", () => {
|
||||||
|
const persisted = createDownloadHealthState({
|
||||||
|
status: "suspect_no_data",
|
||||||
|
runFingerprint: RUN_FINGERPRINT,
|
||||||
|
queueFingerprint: QUEUE_FINGERPRINT,
|
||||||
|
suspiciousDurationMs: 90_000,
|
||||||
|
suspiciousSamples: 3,
|
||||||
|
incidentStartedAt: 10_000,
|
||||||
|
restartPending: true
|
||||||
|
});
|
||||||
|
const first = evaluate(persisted, snapshot(), 100_000);
|
||||||
|
const second = evaluate(first.state, snapshot(), 115_000);
|
||||||
|
|
||||||
|
expect(first.events).toEqual([]);
|
||||||
|
expect(first.state.restartFreshSamples).toBe(1);
|
||||||
|
expect(second.events).toEqual([
|
||||||
|
expect.objectContaining({ type: "download_stalled" })
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("discards a persisted incident when the queue fingerprint changes", () => {
|
||||||
|
const persisted = createDownloadHealthState({
|
||||||
|
status: "alerted",
|
||||||
|
runFingerprint: RUN_FINGERPRINT,
|
||||||
|
queueFingerprint: QUEUE_FINGERPRINT,
|
||||||
|
suspiciousDurationMs: 90_000,
|
||||||
|
suspiciousSamples: 4,
|
||||||
|
incidentStartedAt: 10_000,
|
||||||
|
alertedAt: 90_000,
|
||||||
|
restartPending: true
|
||||||
|
});
|
||||||
|
const result = evaluate(persisted, snapshot({
|
||||||
|
runFingerprint: OTHER_RUN_FINGERPRINT,
|
||||||
|
queueFingerprint: OTHER_QUEUE_FINGERPRINT
|
||||||
|
}), 100_000);
|
||||||
|
|
||||||
|
expect(result.events).toEqual([]);
|
||||||
|
expect(result.state.status).toBe("suspect_no_data");
|
||||||
|
expect(result.state.suspiciousDurationMs).toBe(0);
|
||||||
|
expect(result.state.alertedAt).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes an allowlisted atomic state and reloads it with a fresh-sample gate", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-state-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const filePath = path.join(root, "health.json");
|
||||||
|
const state = {
|
||||||
|
...alertedState(),
|
||||||
|
privateUrl: "https://private.example.test/file",
|
||||||
|
privatePath: "C:\\private\\download.bin",
|
||||||
|
privateAccount: "private@example.test"
|
||||||
|
} as DownloadHealthState;
|
||||||
|
|
||||||
|
saveDownloadHealthState(filePath, state);
|
||||||
|
const persisted = fs.readFileSync(filePath, "utf8");
|
||||||
|
const loaded = loadDownloadHealthState(filePath);
|
||||||
|
|
||||||
|
expect(persisted).not.toMatch(/private|example\.test|download\.bin/i);
|
||||||
|
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
|
||||||
|
expect(loaded.runFingerprint).toBe(RUN_FINGERPRINT);
|
||||||
|
expect(loaded.queueFingerprint).toBe(QUEUE_FINGERPRINT);
|
||||||
|
expect(loaded.restartPending).toBe(true);
|
||||||
|
expect(loaded.restartFreshSamples).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects malformed queue fingerprints instead of restoring an incident", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-invalid-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const filePath = path.join(root, "health.json");
|
||||||
|
fs.writeFileSync(filePath, JSON.stringify({
|
||||||
|
...alertedState(),
|
||||||
|
queueFingerprint: "https://private.example.test/queue"
|
||||||
|
}), "utf8");
|
||||||
|
|
||||||
|
const loaded = loadDownloadHealthState(filePath);
|
||||||
|
|
||||||
|
expect(loaded).toEqual(createDownloadHealthState());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not commit an alert until the outbox accepts the event", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-failure-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const filePath = path.join(root, "health.json");
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||||
|
const monitor = new DownloadHealthMonitor(filePath, suspicious);
|
||||||
|
const eventIds: string[] = [];
|
||||||
|
|
||||||
|
await expect(monitor.sample(snapshot(), 90_000, {
|
||||||
|
stallAfterMs: 90_000,
|
||||||
|
cooldownMs: 600_000,
|
||||||
|
notifyOnStall: true,
|
||||||
|
notifyOnRecovery: true
|
||||||
|
}, async (event) => {
|
||||||
|
eventIds.push(event.id);
|
||||||
|
throw new Error("outbox unavailable");
|
||||||
|
})).rejects.toThrow("outbox unavailable");
|
||||||
|
await expect(monitor.sample(snapshot(), 105_000, {
|
||||||
|
stallAfterMs: 90_000,
|
||||||
|
cooldownMs: 600_000,
|
||||||
|
notifyOnStall: true,
|
||||||
|
notifyOnRecovery: true
|
||||||
|
}, async (event) => {
|
||||||
|
eventIds.push(event.id);
|
||||||
|
throw new Error("outbox unavailable");
|
||||||
|
})).rejects.toThrow("outbox unavailable");
|
||||||
|
|
||||||
|
expect(eventIds[0]).toBe(eventIds[1]);
|
||||||
|
expect(monitor.getState().status).toBe("suspect_no_data");
|
||||||
|
expect(fs.existsSync(filePath)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("persists the alerted state after the outbox accepts the event", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-success-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const filePath = path.join(root, "health.json");
|
||||||
|
const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state;
|
||||||
|
const monitor = new DownloadHealthMonitor(filePath, suspicious);
|
||||||
|
|
||||||
|
const result = await monitor.sample(snapshot(), 90_000, {
|
||||||
|
stallAfterMs: 90_000,
|
||||||
|
cooldownMs: 600_000,
|
||||||
|
notifyOnStall: true,
|
||||||
|
notifyOnRecovery: true
|
||||||
|
}, async () => undefined);
|
||||||
|
|
||||||
|
expect(result.events).toHaveLength(1);
|
||||||
|
expect(monitor.getState().status).toBe("alerted");
|
||||||
|
expect(loadDownloadHealthState(filePath)).toEqual(expect.objectContaining({
|
||||||
|
status: "alerted",
|
||||||
|
restartPending: true
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -14696,3 +14696,245 @@ describe("package lifecycle telemetry boundaries", () => {
|
|||||||
expect(pkg.terminalAt).toBe(pkg.postProcessCompletedAt);
|
expect(pkg.terminalAt).toBe(pkg.postProcessCompletedAt);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("download health snapshot", () => {
|
||||||
|
function createHealthManager(root: string) {
|
||||||
|
const session = emptySession();
|
||||||
|
const packageId = "private-package-id";
|
||||||
|
const itemId = "private-item-id";
|
||||||
|
const now = 100_000;
|
||||||
|
session.running = true;
|
||||||
|
session.runStartedAt = 10_000;
|
||||||
|
session.packageOrder = [packageId];
|
||||||
|
session.packages[packageId] = {
|
||||||
|
id: packageId,
|
||||||
|
name: "Private package name",
|
||||||
|
outputDir: path.join(root, "private-output"),
|
||||||
|
extractDir: path.join(root, "private-extract"),
|
||||||
|
status: "downloading",
|
||||||
|
itemIds: [itemId],
|
||||||
|
cancelled: false,
|
||||||
|
enabled: true,
|
||||||
|
createdAt: 1_000,
|
||||||
|
updatedAt: now
|
||||||
|
};
|
||||||
|
session.items[itemId] = {
|
||||||
|
id: itemId,
|
||||||
|
packageId,
|
||||||
|
url: "https://private.example.test/file",
|
||||||
|
provider: "realdebrid",
|
||||||
|
providerLabel: "Private provider label",
|
||||||
|
providerAccountId: "private-account-id",
|
||||||
|
providerAccountLabel: "private@example.test",
|
||||||
|
status: "downloading",
|
||||||
|
retries: 0,
|
||||||
|
speedBps: 8192,
|
||||||
|
downloadedBytes: 4096,
|
||||||
|
totalBytes: 16384,
|
||||||
|
progressPercent: 25,
|
||||||
|
fileName: "private-file.bin",
|
||||||
|
targetPath: path.join(root, "private-output", "private-file.bin"),
|
||||||
|
resumable: true,
|
||||||
|
attempts: 1,
|
||||||
|
lastError: "",
|
||||||
|
fullStatus: "Download läuft",
|
||||||
|
createdAt: 2_000,
|
||||||
|
updatedAt: now
|
||||||
|
};
|
||||||
|
const manager = new DownloadManager(defaultSettings(), session, createStoragePaths(path.join(root, "state")));
|
||||||
|
const state = manager as any;
|
||||||
|
session.running = true;
|
||||||
|
session.items[itemId].status = "downloading";
|
||||||
|
session.items[itemId].speedBps = 8192;
|
||||||
|
session.packages[packageId].status = "downloading";
|
||||||
|
state.runItemIds.add(itemId);
|
||||||
|
state.runPackageIds.add(packageId);
|
||||||
|
return { manager, session, state, packageId, itemId };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("exposes only aggregate run-scope health data and opaque fingerprints", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-snapshot-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const { manager, state, packageId, itemId } = createHealthManager(root);
|
||||||
|
state.lastSchedulerTickAt = 99_000;
|
||||||
|
state.activeTasks.set(itemId, {
|
||||||
|
itemId,
|
||||||
|
packageId,
|
||||||
|
abortController: new AbortController(),
|
||||||
|
abortReason: "none",
|
||||||
|
resumable: true,
|
||||||
|
nonResumableCounted: false,
|
||||||
|
phase: "downloading",
|
||||||
|
phaseStartedAt: 90_000,
|
||||||
|
phaseDeadlineAt: 0,
|
||||||
|
blockedOnDiskWrite: false,
|
||||||
|
blockedOnDiskSince: 0,
|
||||||
|
blockedOnThrottleUntil: 0
|
||||||
|
});
|
||||||
|
|
||||||
|
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||||
|
const serialized = JSON.stringify(health);
|
||||||
|
|
||||||
|
expect(health).toMatchObject({
|
||||||
|
runActive: true,
|
||||||
|
openItems: 1,
|
||||||
|
openPackages: 1,
|
||||||
|
knownDownloadedBytes: 4096,
|
||||||
|
activeTasks: 1,
|
||||||
|
startableItems: 0,
|
||||||
|
lastSchedulerTickAt: 99_000,
|
||||||
|
currentSpeedBps: 8192
|
||||||
|
});
|
||||||
|
expect(health.runFingerprint).toMatch(/^[a-f0-9]{64}$/);
|
||||||
|
expect(health.queueFingerprint).toMatch(/^[a-f0-9]{64}$/);
|
||||||
|
expect(serialized).not.toMatch(/private|example\.test|realdebrid|file\.bin/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the run fingerprint stable across restart time changes for the same generation scope", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-fingerprint-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const { manager, session } = createHealthManager(root);
|
||||||
|
session.packages[session.packageOrder[0]].resultGeneration = 4;
|
||||||
|
|
||||||
|
const before = manager.getDownloadHealthSnapshot(100_000);
|
||||||
|
session.runStartedAt = 200_000;
|
||||||
|
const after = manager.getDownloadHealthSnapshot(210_000);
|
||||||
|
|
||||||
|
expect(after.runFingerprint).toBe(before.runFingerprint);
|
||||||
|
expect(after.queueFingerprint).toBe(before.queueFingerprint);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("advances progress sequences only for positive byte events and successful item completions", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-sequences-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||||
|
|
||||||
|
state.recordSpeed(1024, packageId);
|
||||||
|
state.recordSpeed(0, packageId);
|
||||||
|
state.recordSpeed(-512, packageId);
|
||||||
|
session.totalDownloadedBytes = 10_000_000;
|
||||||
|
session.totalDownloadedBytes = 0;
|
||||||
|
state.recordRunOutcome(itemId, "completed");
|
||||||
|
state.recordRunOutcome(itemId, "completed");
|
||||||
|
|
||||||
|
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||||
|
|
||||||
|
expect(health.downloadProgressSequence).toBe(1);
|
||||||
|
expect(health.lastPositiveByteAt).toBeGreaterThan(0);
|
||||||
|
expect(health.itemCompletionSequence).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("projects retry, cooldown, disk, throttle and valid phase waits without stale disk events", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-waits-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||||
|
const active = {
|
||||||
|
itemId,
|
||||||
|
packageId,
|
||||||
|
abortController: new AbortController(),
|
||||||
|
abortReason: "none",
|
||||||
|
resumable: true,
|
||||||
|
nonResumableCounted: false,
|
||||||
|
phase: "validating",
|
||||||
|
phaseStartedAt: 90_000,
|
||||||
|
phaseDeadlineAt: 130_000,
|
||||||
|
blockedOnDiskWrite: true,
|
||||||
|
blockedOnDiskSince: 95_000,
|
||||||
|
blockedOnThrottleUntil: 120_000
|
||||||
|
};
|
||||||
|
state.activeTasks.set(itemId, active);
|
||||||
|
state.diskWaitEvents = [{
|
||||||
|
phase: "download",
|
||||||
|
targetPath: "C:\\private\\disk",
|
||||||
|
requiredBytes: 100,
|
||||||
|
availableBytes: 0,
|
||||||
|
reserveBytes: 0,
|
||||||
|
retryAt: 80_000,
|
||||||
|
itemId,
|
||||||
|
packageId
|
||||||
|
}];
|
||||||
|
|
||||||
|
const activeWait = manager.getDownloadHealthSnapshot(100_000);
|
||||||
|
|
||||||
|
expect(activeWait.blockedOnDisk).toBe(true);
|
||||||
|
expect(activeWait.blockedOnThrottleUntil).toBe(120_000);
|
||||||
|
expect(activeWait.activePhaseDeadlineAt).toBe(130_000);
|
||||||
|
|
||||||
|
state.activeTasks.clear();
|
||||||
|
session.items[itemId].status = "queued";
|
||||||
|
state.retryAfterByItem.set(itemId, 140_000);
|
||||||
|
state.providerFailures.set("realdebrid", { count: 20, lastFailAt: 99_000, cooldownUntil: 150_000 });
|
||||||
|
|
||||||
|
const queuedWait = manager.getDownloadHealthSnapshot(100_000);
|
||||||
|
|
||||||
|
expect(queuedWait.startableItems).toBe(0);
|
||||||
|
expect(queuedWait.nextRetryAt).toBe(140_000);
|
||||||
|
expect(queuedWait.providerCooldownUntil).toBe(150_000);
|
||||||
|
expect(queuedWait.blockedOnDisk).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not project a partial retry as a global wait while another download is active", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-partial-wait-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const { manager, session, state, packageId, itemId } = createHealthManager(root);
|
||||||
|
const queuedId = "private-queued-id";
|
||||||
|
session.items[queuedId] = {
|
||||||
|
...session.items[itemId],
|
||||||
|
id: queuedId,
|
||||||
|
status: "queued",
|
||||||
|
downloadedBytes: 0,
|
||||||
|
speedBps: 0
|
||||||
|
};
|
||||||
|
session.packages[packageId].itemIds.push(queuedId);
|
||||||
|
state.runItemIds.add(queuedId);
|
||||||
|
state.activeTasks.set(itemId, {
|
||||||
|
itemId,
|
||||||
|
packageId,
|
||||||
|
abortController: new AbortController(),
|
||||||
|
abortReason: "none",
|
||||||
|
resumable: true,
|
||||||
|
nonResumableCounted: false,
|
||||||
|
phase: "downloading",
|
||||||
|
phaseStartedAt: 90_000,
|
||||||
|
phaseDeadlineAt: 0,
|
||||||
|
blockedOnDiskWrite: false,
|
||||||
|
blockedOnDiskSince: 0,
|
||||||
|
blockedOnThrottleUntil: 0
|
||||||
|
});
|
||||||
|
state.retryAfterByItem.set(queuedId, 140_000);
|
||||||
|
|
||||||
|
const health = manager.getDownloadHealthSnapshot(100_000);
|
||||||
|
|
||||||
|
expect(health.activeTasks).toBe(1);
|
||||||
|
expect(health.nextRetryAt).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("counts one technical recovery when the existing global watchdog restarts stalled tasks", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-watchdog-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const { manager, state, packageId, itemId } = createHealthManager(root);
|
||||||
|
const active = {
|
||||||
|
itemId,
|
||||||
|
packageId,
|
||||||
|
abortController: new AbortController(),
|
||||||
|
abortReason: "none",
|
||||||
|
resumable: true,
|
||||||
|
nonResumableCounted: false,
|
||||||
|
phase: "downloading",
|
||||||
|
phaseStartedAt: 1,
|
||||||
|
phaseDeadlineAt: 0,
|
||||||
|
blockedOnDiskWrite: false,
|
||||||
|
blockedOnDiskSince: 0,
|
||||||
|
blockedOnThrottleUntil: 0
|
||||||
|
};
|
||||||
|
state.activeTasks.set(itemId, active);
|
||||||
|
state.lastGlobalProgressBytes = 0;
|
||||||
|
state.lastGlobalProgressAt = 1;
|
||||||
|
|
||||||
|
state.runGlobalStallWatchdog(90_000);
|
||||||
|
|
||||||
|
expect(active.abortController.signal.aborted).toBe(true);
|
||||||
|
expect(active.abortReason).toBe("stall");
|
||||||
|
expect(manager.getDownloadHealthSnapshot(90_000).technicalRecoveryCount).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,6 +1,12 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import AdmZip from "adm-zip";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { defaultSettings } from "../src/main/constants";
|
import { defaultSettings } from "../src/main/constants";
|
||||||
import { buildAccountSummary, buildStatsPayload } from "../src/main/support-data";
|
import { buildAccountSummary, buildStatsPayload } from "../src/main/support-data";
|
||||||
|
import { buildSupportBundle } from "../src/main/support-bundle";
|
||||||
|
import { createStoragePaths } from "../src/main/storage";
|
||||||
import { serializeRealDebridApiAccounts } from "../src/shared/real-debrid-accounts";
|
import { serializeRealDebridApiAccounts } from "../src/shared/real-debrid-accounts";
|
||||||
import { createVisualFixture } from "./visual/fixtures";
|
import { createVisualFixture } from "./visual/fixtures";
|
||||||
|
|
||||||
@@ -53,4 +59,28 @@ describe("Real-Debrid support summary", () => {
|
|||||||
expect(serialized).toContain("realdebrid");
|
expect(serialized).toContain("realdebrid");
|
||||||
expect(serialized).toContain("4096");
|
expect(serialized).toContain("4096");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps the persisted notification health incident outside support bundles", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-support-health-"));
|
||||||
|
try {
|
||||||
|
const paths = createStoragePaths(root);
|
||||||
|
fs.mkdirSync(root, { recursive: true });
|
||||||
|
fs.writeFileSync(paths.notificationHealthFile, "PRIVATE_HEALTH_INCIDENT_PAYLOAD", "utf8");
|
||||||
|
const snapshot = structuredClone(createVisualFixture("empty").snapshot);
|
||||||
|
const manager = {
|
||||||
|
getSnapshot: () => snapshot,
|
||||||
|
getPackageLogPath: () => null,
|
||||||
|
getItemLogPath: () => null
|
||||||
|
};
|
||||||
|
|
||||||
|
const buffer = await buildSupportBundle(manager as any, root, { hostDiagnosticsMode: "none" });
|
||||||
|
const zip = new AdmZip(buffer);
|
||||||
|
const entries = zip.getEntries().map((entry) => entry.entryName);
|
||||||
|
|
||||||
|
expect(entries).not.toContain(path.basename(paths.notificationHealthFile));
|
||||||
|
expect(buffer.toString("utf8")).not.toContain("PRIVATE_HEALTH_INCIDENT_PAYLOAD");
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user