diff --git a/src/main/app-controller.ts b/src/main/app-controller.ts index 93bfe1a..62f1f40 100644 --- a/src/main/app-controller.ts +++ b/src/main/app-controller.ts @@ -80,6 +80,7 @@ import { getLegacyDesktopLogDirectory, migrateLogDirectories, prepareLogDirector import { normalizeStatisticsLedger, saveStatisticsLedger } from "./statistics-ledger"; import { NotificationOutbox } from "./notification-outbox"; import { sendNotification } from "./notify"; +import { DownloadHealthMonitor } from "./download-health-monitor"; function sanitizeSettingsPatch(partial: Partial): Partial { const entries = Object.entries(partial || {}).filter(([, value]) => value !== undefined); @@ -122,6 +123,12 @@ export class AppController { private notificationOutbox: NotificationOutbox; + private downloadHealthMonitor: DownloadHealthMonitor; + + private downloadHealthTimer: NodeJS.Timeout | null = null; + + private downloadHealthEvaluation: Promise | null = null; + private logDirectory = this.storagePaths.baseDir; private onStateHandler: ((snapshot: UiSnapshot) => void) | null = null; @@ -168,6 +175,7 @@ export class AppController { timestamp: event.createdAt }) }); + this.downloadHealthMonitor = new DownloadHealthMonitor(this.storagePaths.notificationHealthFile); void this.notificationOutbox.drain().catch((error) => { logger.warn(`Notification-Outbox konnte nicht gestartet werden: ${String(error)}`); }); @@ -189,9 +197,14 @@ export class AppController { this.recordHistoryEntry(entry); } }); - this.manager.on("state", (snapshot: UiSnapshot) => { - this.onStateHandler?.(snapshot); - }); + this.manager.on("state", (snapshot: UiSnapshot) => { + this.onStateHandler?.(snapshot); + }); + void this.evaluateDownloadHealth(); + this.downloadHealthTimer = setInterval(() => { + void this.evaluateDownloadHealth(); + }, 15_000); + this.downloadHealthTimer.unref?.(); logger.info(`App gestartet v${APP_VERSION}`); logger.info(`Log-Datei: ${getLogFilePath()}`); logAuditEvent("INFO", "App gestartet", { @@ -1284,12 +1297,46 @@ export class AppController { return this.manager.getPackageLogPath(packageId) || getPackageLogPath(packageId); } - public getItemLogPath(itemId: string): string | null { - return this.manager.getItemLogPath(itemId) || getItemLogPath(itemId); - } - + public getItemLogPath(itemId: string): string | null { + return this.manager.getItemLogPath(itemId) || getItemLogPath(itemId); + } + + private evaluateDownloadHealth(): Promise { + if (this.downloadHealthEvaluation) { + return this.downloadHealthEvaluation; + } + const now = Date.now(); + const task = this.downloadHealthMonitor.sample( + this.manager.getDownloadHealthSnapshot(now), + now, + { + stallAfterMs: this.settings.notifyStallAfterSeconds * 1000, + cooldownMs: this.settings.notifyStallCooldownMinutes * 60_000, + notifyOnStall: this.settings.notifyOnDownloadStall && Boolean(String(this.settings.notifyUrl || "").trim()), + notifyOnRecovery: this.settings.notifyOnDownloadRecovery && Boolean(String(this.settings.notifyUrl || "").trim()) + }, + (event) => this.notificationOutbox.enqueue(event) + ).then(() => undefined).catch((error) => { + logger.warn(`Download-Health-Monitor konnte nicht ausgewertet werden: ${String(error)}`); + }).finally(() => { + if (this.downloadHealthEvaluation === task) { + this.downloadHealthEvaluation = null; + } + }); + this.downloadHealthEvaluation = task; + return task; + } + public async shutdown(): Promise { - if (this.runtimeStatsTimer) { + if (this.downloadHealthTimer) { + clearInterval(this.downloadHealthTimer); + this.downloadHealthTimer = null; + } + this.manager.suspendDownloadHealthMonitoring?.(); + if (this.downloadHealthEvaluation) { + await this.downloadHealthEvaluation; + } + if (this.runtimeStatsTimer) { clearInterval(this.runtimeStatsTimer); this.runtimeStatsTimer = null; } diff --git a/src/main/download-health-monitor.ts b/src/main/download-health-monitor.ts new file mode 100644 index 0000000..6eb4bcd --- /dev/null +++ b/src/main/download-health-monitor.ts @@ -0,0 +1,497 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { NotificationEvent } from "./notification-outbox"; + +export type DownloadHealthStatus = + | "idle" + | "suspended" + | "expected_wait" + | "healthy" + | "suspect_scheduler" + | "suspect_no_data" + | "alerted" + | "recovering"; + +export type DownloadHealthIncidentType = "scheduler" | "no_data"; + +export interface DownloadHealthSnapshot { + runActive: boolean; + runFingerprint: string; + queueFingerprint: string; + openItems: number; + openPackages: number; + knownDownloadedBytes: number; + activeTasks: number; + startableItems: number; + lastSchedulerTickAt: number; + downloadProgressSequence: number; + itemCompletionSequence: number; + lastPositiveByteAt: number; + technicalRecoveryCount: number; + paused: boolean; + reconnectUntil: number; + nextRetryAt: number; + providerCooldownUntil: number; + blockedOnDisk: boolean; + blockedOnThrottleUntil: number; + activePhaseDeadlineAt: number; + terminalFailure: boolean; + manualStop: boolean; + shuttingDown: boolean; + currentSpeedBps: number; +} + +export interface DownloadHealthState { + version: 1; + status: DownloadHealthStatus; + runFingerprint: string | null; + queueFingerprint: string | null; + suspiciousDurationMs: number; + suspiciousSamples: number; + incidentStartedAt: number; + incidentType: DownloadHealthIncidentType | null; + alertedAt: number; + lastAlertAt: number; + cooldownUntil: number; + recoverySamples: number; + lastSampleAt: number | null; + downloadProgressSequence: number; + itemCompletionSequence: number; + lastPositiveByteAt: number; + technicalRecoveryCount: number; + restartPending: boolean; + restartFreshSamples: number; +} + +export interface DownloadHealthOptions { + stallAfterMs: number; + cooldownMs: number; + notifyOnStall: boolean; + notifyOnRecovery: boolean; +} + +export interface DownloadHealthEvaluation { + state: DownloadHealthState; + events: NotificationEvent[]; +} + +const HEALTH_STATUSES = new Set([ + "idle", + "suspended", + "expected_wait", + "healthy", + "suspect_scheduler", + "suspect_no_data", + "alerted", + "recovering" +]); +const INCIDENT_TYPES = new Set(["scheduler", "no_data"]); +const FINGERPRINT_PATTERN = /^[a-f0-9]{64}$/; +const MIN_SUSPICIOUS_SAMPLES = 3; +const SCHEDULER_STALE_AFTER_MS = 30_000; +const ERROR_EVENT_TTL_MS = 24 * 60 * 60 * 1000; +const SUCCESS_EVENT_TTL_MS = 6 * 60 * 60 * 1000; + +function finiteInteger(value: unknown, fallback = 0): number { + const numeric = Number(value); + return Number.isFinite(numeric) ? Math.max(0, Math.floor(numeric)) : fallback; +} + +function validFingerprint(value: unknown): string | null { + return typeof value === "string" && FINGERPRINT_PATTERN.test(value) ? value : null; +} + +function durationText(durationMs: number): string { + const totalSeconds = Math.max(0, Math.floor(durationMs / 1000)); + if (totalSeconds < 60) { + return `${totalSeconds} s`; + } + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return seconds > 0 ? `${minutes} min ${seconds} s` : `${minutes} min`; +} + +function byteText(bytes: number): string { + const value = Math.max(0, finiteInteger(bytes)); + if (value < 1024) { + return `${value} B`; + } + const units = ["KB", "MB", "GB", "TB"]; + let amount = value / 1024; + let unit = units[0]; + for (let index = 1; index < units.length && amount >= 1024; index += 1) { + amount /= 1024; + unit = units[index]; + } + return `${amount >= 10 ? amount.toFixed(0) : amount.toFixed(1)} ${unit}`; +} + +function incidentEvent( + state: DownloadHealthState, + snapshot: DownloadHealthSnapshot, + now: number +): NotificationEvent { + const durationMs = Math.max(state.suspiciousDurationMs, now - state.incidentStartedAt); + return { + id: `health:stall:${snapshot.runFingerprint.slice(0, 16)}:${state.incidentStartedAt}`, + type: "download_stalled", + priority: "error", + createdAt: now, + expiresAt: now + ERROR_EVENT_TTL_MS, + attempts: 0, + nextAttemptAt: now, + payload: { + title: "Downloadstillstand bestätigt", + description: `Seit ${durationText(durationMs)} wurde kein bestätigter Downloadfortschritt erkannt.`, + color: 0xe67e22, + fields: [ + { name: "Dauer", value: durationText(durationMs), inline: true }, + { name: "Offene Pakete", value: String(snapshot.openPackages), inline: true }, + { name: "Offene Dateien", value: String(snapshot.openItems), inline: true }, + { name: "Bekannte Mindestmenge", value: byteText(snapshot.knownDownloadedBytes), inline: true }, + { name: "Aktiv / startfähig", value: `${snapshot.activeTasks} / ${snapshot.startableItems}`, inline: true }, + { name: "Technische Wiederherstellungen", value: String(snapshot.technicalRecoveryCount), inline: true } + ] + } + }; +} + +function recoveryEvent( + state: DownloadHealthState, + snapshot: DownloadHealthSnapshot, + now: number +): NotificationEvent { + return { + id: `health:recovery:${snapshot.runFingerprint.slice(0, 16)}:${state.alertedAt}`, + type: "download_recovered", + priority: "success", + createdAt: now, + expiresAt: now + SUCCESS_EVENT_TTL_MS, + attempts: 0, + nextAttemptAt: now, + payload: { + title: "Download läuft wieder", + description: "Nach dem bestätigten Stillstand wurde neuer Fortschritt erkannt.", + color: 0x2ecc71, + fields: [ + { name: "Incident-Dauer", value: durationText(Math.max(0, now - state.incidentStartedAt)), inline: true }, + { name: "Aktive Downloads", value: String(snapshot.activeTasks), inline: true }, + { name: "Aktuelle Geschwindigkeit", value: `${byteText(snapshot.currentSpeedBps)}/s`, inline: true } + ] + } + }; +} + +export function createDownloadHealthState(overrides: Partial = {}): DownloadHealthState { + return { + version: 1, + status: "idle", + runFingerprint: null, + queueFingerprint: null, + suspiciousDurationMs: 0, + suspiciousSamples: 0, + incidentStartedAt: 0, + incidentType: null, + alertedAt: 0, + lastAlertAt: 0, + cooldownUntil: 0, + recoverySamples: 0, + lastSampleAt: null, + downloadProgressSequence: 0, + itemCompletionSequence: 0, + lastPositiveByteAt: 0, + technicalRecoveryCount: 0, + restartPending: false, + restartFreshSamples: 0, + ...overrides + }; +} + +function resetForSnapshot( + state: DownloadHealthState, + snapshot: DownloadHealthSnapshot, + now: number +): DownloadHealthState { + return createDownloadHealthState({ + runFingerprint: snapshot.runFingerprint, + queueFingerprint: snapshot.queueFingerprint, + lastAlertAt: state.lastAlertAt, + cooldownUntil: state.cooldownUntil, + lastSampleAt: now, + downloadProgressSequence: finiteInteger(snapshot.downloadProgressSequence), + itemCompletionSequence: finiteInteger(snapshot.itemCompletionSequence), + lastPositiveByteAt: finiteInteger(snapshot.lastPositiveByteAt), + technicalRecoveryCount: finiteInteger(snapshot.technicalRecoveryCount) + }); +} + +function endIncident(state: DownloadHealthState): DownloadHealthState { + return createDownloadHealthState({ + lastAlertAt: state.lastAlertAt, + cooldownUntil: state.cooldownUntil, + downloadProgressSequence: state.downloadProgressSequence, + itemCompletionSequence: state.itemCompletionSequence, + lastPositiveByteAt: state.lastPositiveByteAt, + technicalRecoveryCount: state.technicalRecoveryCount + }); +} + +function expectedWait(snapshot: DownloadHealthSnapshot, now: number): boolean { + return snapshot.paused + || snapshot.reconnectUntil > now + || snapshot.nextRetryAt > now + || snapshot.providerCooldownUntil > now + || snapshot.blockedOnDisk + || snapshot.blockedOnThrottleUntil > now + || snapshot.activePhaseDeadlineAt > now; +} + +function suspicionType(snapshot: DownloadHealthSnapshot, now: number): DownloadHealthIncidentType | null { + if (snapshot.activeTasks === 0 && snapshot.startableItems > 0) { + if (snapshot.lastSchedulerTickAt <= 0 || now - snapshot.lastSchedulerTickAt >= SCHEDULER_STALE_AFTER_MS) { + return "scheduler"; + } + return null; + } + return "no_data"; +} + +export function evaluateDownloadHealth( + previous: DownloadHealthState, + snapshot: DownloadHealthSnapshot, + nowValue: number, + options: DownloadHealthOptions +): DownloadHealthEvaluation { + const now = finiteInteger(nowValue); + let state = createDownloadHealthState(previous); + const events: NotificationEvent[] = []; + const runFingerprint = validFingerprint(snapshot.runFingerprint); + const queueFingerprint = validFingerprint(snapshot.queueFingerprint); + + if (state.restartPending && !snapshot.runActive && !snapshot.terminalFailure && !snapshot.manualStop && !snapshot.shuttingDown) { + state.status = "suspended"; + state.lastSampleAt = null; + return { state, events }; + } + + if (!snapshot.runActive || snapshot.openItems <= 0 || snapshot.terminalFailure || snapshot.manualStop || snapshot.shuttingDown || !runFingerprint || !queueFingerprint) { + state.downloadProgressSequence = Math.max(state.downloadProgressSequence, finiteInteger(snapshot.downloadProgressSequence)); + state.itemCompletionSequence = Math.max(state.itemCompletionSequence, finiteInteger(snapshot.itemCompletionSequence)); + state.lastPositiveByteAt = Math.max(state.lastPositiveByteAt, finiteInteger(snapshot.lastPositiveByteAt)); + state.technicalRecoveryCount = Math.max(state.technicalRecoveryCount, finiteInteger(snapshot.technicalRecoveryCount)); + return { state: endIncident(state), events }; + } + + if (state.runFingerprint !== runFingerprint || state.queueFingerprint !== queueFingerprint) { + state = resetForSnapshot(state, snapshot, now); + } + + state.runFingerprint = runFingerprint; + state.queueFingerprint = queueFingerprint; + + if (state.restartPending) { + state.restartFreshSamples += 1; + state.lastSampleAt = now; + if (state.restartFreshSamples < 2) { + state.downloadProgressSequence = finiteInteger(snapshot.downloadProgressSequence); + state.itemCompletionSequence = finiteInteger(snapshot.itemCompletionSequence); + state.lastPositiveByteAt = finiteInteger(snapshot.lastPositiveByteAt); + state.technicalRecoveryCount = finiteInteger(snapshot.technicalRecoveryCount); + return { state, events }; + } + state.restartPending = false; + } + + const progressSequence = finiteInteger(snapshot.downloadProgressSequence); + const completionSequence = finiteInteger(snapshot.itemCompletionSequence); + const positiveByteProgress = progressSequence > state.downloadProgressSequence; + const completionProgress = completionSequence > state.itemCompletionSequence; + const progress = positiveByteProgress || completionProgress; + state.downloadProgressSequence = Math.max(state.downloadProgressSequence, progressSequence); + state.itemCompletionSequence = Math.max(state.itemCompletionSequence, completionSequence); + state.lastPositiveByteAt = Math.max(state.lastPositiveByteAt, finiteInteger(snapshot.lastPositiveByteAt)); + state.technicalRecoveryCount = Math.max(state.technicalRecoveryCount, finiteInteger(snapshot.technicalRecoveryCount)); + + if (state.alertedAt > 0 && progress) { + state.recoverySamples = completionProgress ? 2 : state.recoverySamples + 1; + if (state.recoverySamples >= 2) { + if (options.notifyOnRecovery) { + events.push(recoveryEvent(state, snapshot, now)); + } + const recovered = resetForSnapshot(state, snapshot, now); + recovered.status = "healthy"; + recovered.lastAlertAt = state.lastAlertAt; + recovered.cooldownUntil = state.cooldownUntil; + return { state: recovered, events }; + } + state.status = "recovering"; + state.lastSampleAt = now; + return { state, events }; + } + + if (state.alertedAt > 0) { + if (expectedWait(snapshot, now)) { + state.status = "expected_wait"; + } else { + state.status = "alerted"; + } + state.lastSampleAt = now; + return { state, events }; + } + + if (expectedWait(snapshot, now)) { + state.status = "expected_wait"; + state.lastSampleAt = now; + return { state, events }; + } + + if (progress) { + state.status = "healthy"; + state.suspiciousDurationMs = 0; + state.suspiciousSamples = 0; + state.incidentStartedAt = 0; + state.incidentType = null; + state.recoverySamples = 0; + state.lastSampleAt = now; + return { state, events }; + } + + const incidentType = suspicionType(snapshot, now); + if (!incidentType) { + state.status = "healthy"; + state.suspiciousDurationMs = 0; + state.suspiciousSamples = 0; + state.incidentStartedAt = 0; + state.incidentType = null; + state.lastSampleAt = now; + return { state, events }; + } + + const elapsed = state.lastSampleAt === null ? 0 : Math.max(0, now - state.lastSampleAt); + if (state.incidentStartedAt <= 0) { + state.incidentStartedAt = now; + } + state.suspiciousDurationMs += elapsed; + state.suspiciousSamples += 1; + state.incidentType = incidentType; + state.status = incidentType === "scheduler" ? "suspect_scheduler" : "suspect_no_data"; + state.lastSampleAt = now; + + const stallAfterMs = Math.max(0, finiteInteger(options.stallAfterMs, 90_000)); + const cooldownMs = Math.max(0, finiteInteger(options.cooldownMs, 600_000)); + const confirmed = state.suspiciousDurationMs >= stallAfterMs + && state.suspiciousSamples >= MIN_SUSPICIOUS_SAMPLES; + if (confirmed && options.notifyOnStall && now >= state.cooldownUntil) { + state.status = "alerted"; + state.alertedAt = now; + state.lastAlertAt = now; + state.cooldownUntil = now + cooldownMs; + state.recoverySamples = 0; + events.push(incidentEvent(state, snapshot, now)); + } + + return { state, events }; +} + +function persistedState(state: DownloadHealthState): Omit { + return { + version: 1, + status: state.status, + runFingerprint: state.runFingerprint, + queueFingerprint: state.queueFingerprint, + suspiciousDurationMs: finiteInteger(state.suspiciousDurationMs), + suspiciousSamples: finiteInteger(state.suspiciousSamples), + incidentStartedAt: finiteInteger(state.incidentStartedAt), + incidentType: state.incidentType, + alertedAt: finiteInteger(state.alertedAt), + lastAlertAt: finiteInteger(state.lastAlertAt), + cooldownUntil: finiteInteger(state.cooldownUntil), + recoverySamples: finiteInteger(state.recoverySamples), + lastSampleAt: state.lastSampleAt === null ? null : finiteInteger(state.lastSampleAt), + downloadProgressSequence: finiteInteger(state.downloadProgressSequence), + itemCompletionSequence: finiteInteger(state.itemCompletionSequence), + lastPositiveByteAt: finiteInteger(state.lastPositiveByteAt), + technicalRecoveryCount: finiteInteger(state.technicalRecoveryCount) + }; +} + +export function saveDownloadHealthState(filePath: string, state: DownloadHealthState): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.tmp`; + try { + fs.writeFileSync(tempPath, JSON.stringify(persistedState(state)), "utf8"); + fs.renameSync(tempPath, filePath); + } catch (error) { + try { + fs.rmSync(tempPath, { force: true }); + } catch { + } + throw error; + } +} + +export function loadDownloadHealthState(filePath: string): DownloadHealthState { + try { + const raw = JSON.parse(fs.readFileSync(filePath, "utf8")) as Record; + const runFingerprint = raw.runFingerprint === null ? null : validFingerprint(raw.runFingerprint); + const queueFingerprint = raw.queueFingerprint === null ? null : validFingerprint(raw.queueFingerprint); + if (raw.version !== 1 || !HEALTH_STATUSES.has(raw.status as DownloadHealthStatus)) { + return createDownloadHealthState(); + } + if ((raw.runFingerprint !== null && !runFingerprint) || (raw.queueFingerprint !== null && !queueFingerprint)) { + return createDownloadHealthState(); + } + const incidentType = INCIDENT_TYPES.has(raw.incidentType as DownloadHealthIncidentType) + ? raw.incidentType as DownloadHealthIncidentType + : null; + return createDownloadHealthState({ + status: raw.status as DownloadHealthStatus, + runFingerprint, + queueFingerprint, + suspiciousDurationMs: finiteInteger(raw.suspiciousDurationMs), + suspiciousSamples: finiteInteger(raw.suspiciousSamples), + incidentStartedAt: finiteInteger(raw.incidentStartedAt), + incidentType, + alertedAt: finiteInteger(raw.alertedAt), + lastAlertAt: finiteInteger(raw.lastAlertAt), + cooldownUntil: finiteInteger(raw.cooldownUntil), + recoverySamples: finiteInteger(raw.recoverySamples), + lastSampleAt: null, + downloadProgressSequence: finiteInteger(raw.downloadProgressSequence), + itemCompletionSequence: finiteInteger(raw.itemCompletionSequence), + lastPositiveByteAt: finiteInteger(raw.lastPositiveByteAt), + technicalRecoveryCount: finiteInteger(raw.technicalRecoveryCount), + restartPending: Boolean(runFingerprint && queueFingerprint), + restartFreshSamples: 0 + }); + } catch { + return createDownloadHealthState(); + } +} + +export class DownloadHealthMonitor { + private state: DownloadHealthState; + + public constructor(private readonly filePath: string, initialState?: DownloadHealthState) { + this.state = initialState ? createDownloadHealthState(initialState) : loadDownloadHealthState(filePath); + } + + public getState(): DownloadHealthState { + return createDownloadHealthState(this.state); + } + + public async sample( + snapshot: DownloadHealthSnapshot, + now: number, + options: DownloadHealthOptions, + enqueue: (event: NotificationEvent) => Promise + ): Promise { + const evaluation = evaluateDownloadHealth(this.state, snapshot, now, options); + for (const event of evaluation.events) { + await enqueue(event); + } + saveDownloadHealthState(this.filePath, evaluation.state); + this.state = evaluation.state; + return evaluation; + } +} diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 66890ed..cc6a743 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -1,6 +1,7 @@ -import fs from "node:fs"; -import path from "node:path"; -import os from "node:os"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { createHash } from "node:crypto"; import { EventEmitter } from "node:events"; import { v4 as uuidv4 } from "uuid"; import { @@ -93,6 +94,7 @@ import { } from "./statistics-ledger"; import { finalizePackageResult } from "./package-telemetry"; import type { NotificationEvent } from "./notification-outbox"; +import type { DownloadHealthSnapshot } from "./download-health-monitor"; import { buildHistoryEntry, buildPackageDigestEvents, @@ -118,8 +120,12 @@ type ActiveTask = { stallRetries?: number; genericErrorRetries?: number; unrestrictRetries?: number; - blockedOnDiskWrite?: boolean; - blockedOnDiskSince?: number; + blockedOnDiskWrite?: boolean; + blockedOnDiskSince?: number; + blockedOnThrottleUntil?: number; + phase?: "validating" | "downloading" | "integrity_check"; + phaseStartedAt?: number; + phaseDeadlineAt?: number; }; const DOWNLOAD_ACCOUNT_PROVIDERS: readonly DebridProvider[] = [ @@ -1960,7 +1966,23 @@ export class DownloadManager extends EventEmitter { private itemCount = 0; - private lastSchedulerHeartbeatAt = 0; + private lastSchedulerHeartbeatAt = 0; + + private lastSchedulerTickAt = 0; + + private downloadProgressSequence = 0; + + private itemCompletionSequence = 0; + + private lastPositiveByteAt = 0; + + private technicalRecoveryCount = 0; + + private healthManualStop = false; + + private healthShuttingDown = false; + + private healthTerminalFailure = false; private lastReconnectMarkAt = 0; @@ -2478,11 +2500,118 @@ export class DownloadManager extends EventEmitter { return cloneSession(this.session); } - public getSummary(): DownloadSummary | null { - return this.summary; - } - - public isSessionRunning(): boolean { + public getSummary(): DownloadSummary | null { + return this.summary; + } + + public getDownloadHealthSnapshot(now = nowMs()): DownloadHealthSnapshot { + const openPackages = new Set(); + const queueParts: string[] = []; + const activeTasks: ActiveTask[] = []; + let openItems = 0; + let knownDownloadedBytes = 0; + let currentSpeedBps = 0; + let startableItems = 0; + let nextRetryAt = 0; + let providerCooldownUntil = 0; + + for (const itemId of this.runItemIds) { + queueParts.push(itemId); + const item = this.session.items[itemId]; + if (!item || isFinishedStatus(item.status)) { + continue; + } + const pkg = this.session.packages[item.packageId]; + if (!pkg || pkg.cancelled || !pkg.enabled) { + continue; + } + openItems += 1; + openPackages.add(pkg.id); + knownDownloadedBytes += Math.max(0, Math.floor(Number(item.downloadedBytes) || 0)); + currentSpeedBps += Math.max(0, Math.floor(Number(item.speedBps) || 0)); + const active = this.activeTasks.get(itemId); + if (active && !active.abortController.signal.aborted) { + activeTasks.push(active); + continue; + } + if (item.status !== "queued" && item.status !== "reconnect_wait") { + continue; + } + const retryAt = this.retryAfterByItem.get(itemId) || 0; + const failureKey = this.getProviderFailureKeyForItem(item); + const cooldownAt = this.providerFailures.get(failureKey)?.cooldownUntil || 0; + if (retryAt > now) { + nextRetryAt = nextRetryAt === 0 ? retryAt : Math.min(nextRetryAt, retryAt); + } + if (cooldownAt > now) { + providerCooldownUntil = providerCooldownUntil === 0 + ? cooldownAt + : Math.min(providerCooldownUntil, cooldownAt); + } + if (retryAt <= now && cooldownAt <= now) { + startableItems += 1; + } + } + + const blockedOnDisk = activeTasks.length > 0 && activeTasks.every((active) => Boolean(active.blockedOnDiskWrite)); + const throttleDeadlines = activeTasks.map((active) => active.blockedOnThrottleUntil || 0); + const blockedOnThrottleUntil = throttleDeadlines.length > 0 && throttleDeadlines.every((deadline) => deadline > now) + ? Math.min(...throttleDeadlines) + : 0; + const phaseDeadlines = activeTasks.map((active) => active.phaseDeadlineAt || 0); + const activePhaseDeadlineAt = phaseDeadlines.length > 0 && phaseDeadlines.every((deadline) => deadline > now) + ? Math.min(...phaseDeadlines) + : 0; + const queueFingerprint = createHash("sha256") + .update([...queueParts].sort().join("\n")) + .digest("hex"); + const runParts = [...this.runPackageIds].map((packageId) => { + const generation = Math.max(1, Math.floor(Number(this.session.packages[packageId]?.resultGeneration) || 1)); + return `${packageId}:${generation}`; + }); + const runFingerprint = createHash("sha256") + .update(`${runParts.sort().join("\n")}|${queueFingerprint}`) + .digest("hex"); + + return { + runActive: this.session.running && openItems > 0, + runFingerprint, + queueFingerprint, + openItems, + openPackages: openPackages.size, + knownDownloadedBytes, + activeTasks: activeTasks.length, + startableItems, + lastSchedulerTickAt: this.lastSchedulerTickAt, + downloadProgressSequence: this.downloadProgressSequence, + itemCompletionSequence: this.itemCompletionSequence, + lastPositiveByteAt: this.lastPositiveByteAt, + technicalRecoveryCount: this.technicalRecoveryCount, + paused: this.session.paused, + reconnectUntil: this.session.reconnectUntil, + nextRetryAt: activeTasks.length === 0 && startableItems === 0 ? nextRetryAt : 0, + providerCooldownUntil: activeTasks.length === 0 && startableItems === 0 ? providerCooldownUntil : 0, + blockedOnDisk, + blockedOnThrottleUntil, + activePhaseDeadlineAt, + terminalFailure: this.healthTerminalFailure, + manualStop: this.healthManualStop, + shuttingDown: this.healthShuttingDown, + currentSpeedBps: this.session.running && !this.session.paused ? currentSpeedBps : 0 + }; + } + + private beginHealthRun(): void { + this.healthManualStop = false; + this.healthShuttingDown = false; + this.healthTerminalFailure = false; + } + + public suspendDownloadHealthMonitoring(): void { + this.healthShuttingDown = true; + } + + public isSessionRunning(): boolean { return this.session.running; } @@ -5978,6 +6107,7 @@ export class DownloadManager extends EventEmitter { } public async startPackages(packageIds: string[]): Promise { + this.beginHealthRun(); this.ensureUsableDownloadAccount(); const targetSet = new Set(packageIds); for (const packageId of this.packagePostProcessTasks.keys()) { @@ -6080,6 +6210,7 @@ export class DownloadManager extends EventEmitter { } public async startItems(itemIds: string[]): Promise { + this.beginHealthRun(); this.ensureUsableDownloadAccount(); const targetSet = new Set(itemIds); @@ -6196,6 +6327,7 @@ export class DownloadManager extends EventEmitter { if (this.session.running) { return; } + this.beginHealthRun(); this.ensureUsableDownloadAccount(); this.schedulerGeneration += 1; @@ -6347,8 +6479,10 @@ export class DownloadManager extends EventEmitter { }); } - public stop(options?: { parkForRestart?: boolean }): void { - const parkForRestart = options?.parkForRestart === true; + public stop(options?: { parkForRestart?: boolean }): void { + const parkForRestart = options?.parkForRestart === true; + this.healthManualStop = !parkForRestart; + this.healthShuttingDown = parkForRestart; const abortReason: "stop" | "shutdown" = parkForRestart ? "shutdown" : "stop"; const keepExtraction = this.settings.autoExtractWhenStopped; const wasRunning = this.session.running; @@ -6426,6 +6560,7 @@ export class DownloadManager extends EventEmitter { } public prepareForShutdown(): void { + this.healthShuttingDown = true; logger.info(`Shutdown-Vorbereitung gestartet: active=${this.activeTasks.size}, running=${this.session.running}, paused=${this.session.paused}`); this.updateStatisticsActivity(nowMs()); this.rotationListenerActive = false; @@ -6922,9 +7057,14 @@ export class DownloadManager extends EventEmitter { private lastSpeedPruneAt = 0; - private recordSpeed(bytes: number, packageId: string = ""): void { - const now = nowMs(); - if (bytes > 0 && this.consecutiveReconnects > 0) { + private recordSpeed(bytes: number, packageId: string = ""): void { + const now = nowMs(); + if (!Number.isFinite(bytes) || bytes <= 0) { + return; + } + this.downloadProgressSequence += 1; + this.lastPositiveByteAt = now; + if (bytes > 0 && this.consecutiveReconnects > 0) { this.consecutiveReconnects = 0; } const bucket = now - (now % 120); @@ -6955,8 +7095,9 @@ export class DownloadManager extends EventEmitter { this.statisticsDirty = true; this.statisticsUrgent = true; } - if (status === "completed" && previous !== "completed") { - this.sessionCompletedFiles += 1; + if (status === "completed" && previous !== "completed") { + this.itemCompletionSequence += 1; + this.sessionCompletedFiles += 1; this.settings.totalCompletedFilesAllTime = Math.max(0, Number(this.settings.totalCompletedFilesAllTime || 0)) + 1; this.invalidateStatsCache(); } @@ -9108,6 +9249,7 @@ export class DownloadManager extends EventEmitter { try { while (this.session.running && this.schedulerGeneration === myGeneration) { const now = nowMs(); + this.lastSchedulerTickAt = now; this.updateStatisticsActivity(now); if (now - this.lastSchedulerHeartbeatAt >= 60000) { this.lastSchedulerHeartbeatAt = now; @@ -9246,8 +9388,9 @@ export class DownloadManager extends EventEmitter { return; } - logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`); - for (const active of this.activeTasks.values()) { + logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`); + this.technicalRecoveryCount += 1; + for (const active of this.activeTasks.values()) { if (active.abortController.signal.aborted) { continue; } @@ -9576,10 +9719,14 @@ export class DownloadManager extends EventEmitter { abortController: new AbortController(), abortReason: "none", resumable: true, - nonResumableCounted: false, - blockedOnDiskWrite: false, - blockedOnDiskSince: 0 - }; + nonResumableCounted: false, + blockedOnDiskWrite: false, + blockedOnDiskSince: 0, + blockedOnThrottleUntil: 0, + phase: "validating", + phaseStartedAt: item.updatedAt, + phaseDeadlineAt: item.updatedAt + getUnrestrictTimeoutMs() + 15_000 + }; this.activeTasks.set(itemId, active); this.notePacedStartForItem(item, nowMs()); this.emitState(); @@ -9701,6 +9848,9 @@ export class DownloadManager extends EventEmitter { this.settings, item.url ); + active.phase = "validating"; + active.phaseStartedAt = nowMs(); + active.phaseDeadlineAt = active.phaseStartedAt + unrestrictTimeoutMs; const unrestrictTimeoutSignal = AbortSignal.timeout(unrestrictTimeoutMs); const unrestrictedSignal = AbortSignal.any([active.abortController.signal, unrestrictTimeoutSignal]); let unrestricted; @@ -9805,6 +9955,9 @@ export class DownloadManager extends EventEmitter { throw error; } item.status = "downloading"; + active.phase = "downloading"; + active.phaseStartedAt = nowMs(); + active.phaseDeadlineAt = 0; const pLabel = unrestricted.providerLabel; item.fullStatus = "Starte..."; item.updatedAt = nowMs(); @@ -9855,10 +10008,18 @@ export class DownloadManager extends EventEmitter { item.status = "integrity_check"; item.fullStatus = "CRC-Check läuft"; item.updatedAt = nowMs(); - this.emitState(); - - const integrityStartedAt = nowMs(); - const validation = await validateFileAgainstManifest(item.targetPath, pkg.outputDir); + this.emitState(); + + const integrityStartedAt = nowMs(); + const integrityBytes = Math.max(0, Number(item.downloadedBytes || item.totalBytes || 0)); + const integrityBudgetMs = Math.max( + 120_000, + Math.min(30 * 60 * 1000, 60_000 + Math.ceil(integrityBytes / (25 * 1024 * 1024)) * 1000) + ); + active.phase = "integrity_check"; + active.phaseStartedAt = integrityStartedAt; + active.phaseDeadlineAt = integrityStartedAt + integrityBudgetMs; + const validation = await validateFileAgainstManifest(item.targetPath, pkg.outputDir); if (active.abortController.signal.aborted) { throw new Error(`aborted:${active.abortReason}`); } @@ -9881,9 +10042,12 @@ export class DownloadManager extends EventEmitter { item.progressPercent = 0; item.downloadedBytes = 0; item.totalBytes = mergeKnownTotalBytes(item.totalBytes, unrestricted.fileSize); - this.emitState(); - await sleep(300); - continue; + this.emitState(); + await sleep(300); + active.phase = "downloading"; + active.phaseStartedAt = nowMs(); + active.phaseDeadlineAt = 0; + continue; } throw new Error(`Integritätsprüfung fehlgeschlagen (${validation.message})`); } @@ -11178,7 +11342,7 @@ export class DownloadManager extends EventEmitter { } const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); - await this.applySpeedLimit(buffer.length, windowBytes, windowStarted, active.abortController.signal); + await this.applySpeedLimit(buffer.length, windowBytes, windowStarted, active); if (active.abortController.signal.aborted) { throw new Error(`aborted:${active.abortReason}`); } @@ -12241,45 +12405,53 @@ export class DownloadManager extends EventEmitter { return 0; } - private async applyGlobalSpeedLimit(chunkBytes: number, bytesPerSecond: number, signal?: AbortSignal): Promise { - const task = this.globalSpeedLimitQueue + private async applyGlobalSpeedLimit(chunkBytes: number, bytesPerSecond: number, active?: ActiveTask): Promise { + const signal = active?.abortController.signal; + const task = this.globalSpeedLimitQueue .catch(() => undefined) .then(async () => { if (signal?.aborted) { throw new Error("aborted:speed_limit"); } - const now = nowMs(); - const waitMs = Math.max(0, this.globalSpeedLimitNextAt - now); - if (waitMs > 0) { - await new Promise((resolve, reject) => { - let timer: NodeJS.Timeout | null = setTimeout(() => { - timer = null; - if (signal) { - signal.removeEventListener("abort", onAbort); - } - resolve(); - }, waitMs); - - const onAbort = (): void => { - if (timer) { - clearTimeout(timer); - timer = null; - } - signal?.removeEventListener("abort", onAbort); - reject(new Error("aborted:speed_limit")); - }; - - if (signal) { - if (signal.aborted) { - onAbort(); - return; - } - signal.addEventListener("abort", onAbort, { once: true }); - } - }); - } - - if (signal?.aborted) { + const now = nowMs(); + const waitMs = Math.max(0, this.globalSpeedLimitNextAt - now); + if (waitMs > 0) { + if (active) { + active.blockedOnThrottleUntil = now + waitMs; + } + try { + await new Promise((resolve, reject) => { + let timer: NodeJS.Timeout | null = setTimeout(() => { + timer = null; + signal?.removeEventListener("abort", onAbort); + resolve(); + }, waitMs); + + const onAbort = (): void => { + if (timer) { + clearTimeout(timer); + timer = null; + } + signal?.removeEventListener("abort", onAbort); + reject(new Error("aborted:speed_limit")); + }; + + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + } finally { + if (active) { + active.blockedOnThrottleUntil = 0; + } + } + } + + if (signal?.aborted) { throw new Error("aborted:speed_limit"); } @@ -12292,7 +12464,8 @@ export class DownloadManager extends EventEmitter { await task; } - private async applySpeedLimit(chunkBytes: number, localWindowBytes: number, localWindowStarted: number, signal?: AbortSignal): Promise { + private async applySpeedLimit(chunkBytes: number, localWindowBytes: number, localWindowStarted: number, active?: ActiveTask): Promise { + const signal = active?.abortController.signal; const limitKbps = this.getEffectiveSpeedLimitKbps(); if (limitKbps <= 0) { return; @@ -12304,40 +12477,48 @@ export class DownloadManager extends EventEmitter { const projected = localWindowBytes + chunkBytes; const allowed = bytesPerSecond * elapsed; if (projected > allowed) { - const sleepMs = Math.ceil(((projected - allowed) / bytesPerSecond) * 1000); - if (sleepMs > 0) { - await new Promise((resolve, reject) => { - let timer: NodeJS.Timeout | null = setTimeout(() => { - timer = null; - if (signal) { - signal.removeEventListener("abort", onAbort); - } - resolve(); - }, Math.min(300, sleepMs)); - - const onAbort = (): void => { - if (timer) { - clearTimeout(timer); - timer = null; - } - signal?.removeEventListener("abort", onAbort); - reject(new Error("aborted:speed_limit")); - }; - - if (signal) { - if (signal.aborted) { - onAbort(); - return; - } - signal.addEventListener("abort", onAbort, { once: true }); - } - }); + const sleepMs = Math.ceil(((projected - allowed) / bytesPerSecond) * 1000); + if (sleepMs > 0) { + const boundedSleepMs = Math.min(300, sleepMs); + if (active) { + active.blockedOnThrottleUntil = nowMs() + boundedSleepMs; + } + try { + await new Promise((resolve, reject) => { + let timer: NodeJS.Timeout | null = setTimeout(() => { + timer = null; + signal?.removeEventListener("abort", onAbort); + resolve(); + }, boundedSleepMs); + + const onAbort = (): void => { + if (timer) { + clearTimeout(timer); + timer = null; + } + signal?.removeEventListener("abort", onAbort); + reject(new Error("aborted:speed_limit")); + }; + + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + } finally { + if (active) { + active.blockedOnThrottleUntil = 0; + } + } } } return; } - await this.applyGlobalSpeedLimit(chunkBytes, bytesPerSecond, signal); + await this.applyGlobalSpeedLimit(chunkBytes, bytesPerSecond, active); } private async findReadyArchiveSets(pkg: PackageEntry): Promise> { @@ -13882,9 +14063,10 @@ export class DownloadManager extends EventEmitter { this.session.runStartedAt = 0; const total = this.runItemIds.size; const outcomes = Array.from(this.runOutcomes.values()); - const success = outcomes.filter((status) => status === "completed").length; - const failed = outcomes.filter((status) => status === "failed").length; - const cancelled = outcomes.filter((status) => status === "cancelled").length; + const success = outcomes.filter((status) => status === "completed").length; + const failed = outcomes.filter((status) => status === "failed").length; + const cancelled = outcomes.filter((status) => status === "cancelled").length; + this.healthTerminalFailure = failed > 0; const extracted = this.runCompletedPackages.size; const duration = runStartedAt > 0 ? Math.max(1, Math.floor((completedAt - runStartedAt) / 1000)) : 1; const avgSpeed = Math.floor(this.session.totalDownloadedBytes / duration); diff --git a/src/main/storage.ts b/src/main/storage.ts index 7cf0638..2dfd471 100644 --- a/src/main/storage.ts +++ b/src/main/storage.ts @@ -702,6 +702,7 @@ export interface StoragePaths { historyFile: string; statisticsFile: string; notificationOutboxFile: string; + notificationHealthFile: string; } export function createStoragePaths(baseDir: string): StoragePaths { @@ -711,7 +712,8 @@ export function createStoragePaths(baseDir: string): StoragePaths { sessionFile: path.join(baseDir, "rd_session_state.json"), historyFile: path.join(baseDir, "rd_history.json"), statisticsFile: path.join(baseDir, "rd_statistics.json"), - notificationOutboxFile: path.join(baseDir, "rd_notification_outbox.json") + notificationOutboxFile: path.join(baseDir, "rd_notification_outbox.json"), + notificationHealthFile: path.join(baseDir, "rd_notification_health.json") }; } diff --git a/tests/download-health-monitor.test.ts b/tests/download-health-monitor.test.ts new file mode 100644 index 0000000..8dfd524 --- /dev/null +++ b/tests/download-health-monitor.test.ts @@ -0,0 +1,452 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + DownloadHealthMonitor, + createDownloadHealthState, + evaluateDownloadHealth, + loadDownloadHealthState, + saveDownloadHealthState, + type DownloadHealthSnapshot, + type DownloadHealthState +} from "../src/main/download-health-monitor"; + +const RUN_FINGERPRINT = "a".repeat(64); +const QUEUE_FINGERPRINT = "b".repeat(64); +const OTHER_RUN_FINGERPRINT = "c".repeat(64); +const OTHER_QUEUE_FINGERPRINT = "d".repeat(64); +const tempDirs: string[] = []; + +function snapshot(overrides: Partial = {}): DownloadHealthSnapshot { + return { + runActive: true, + runFingerprint: RUN_FINGERPRINT, + queueFingerprint: QUEUE_FINGERPRINT, + openItems: 2, + openPackages: 1, + knownDownloadedBytes: 4096, + activeTasks: 1, + startableItems: 0, + lastSchedulerTickAt: 0, + downloadProgressSequence: 0, + itemCompletionSequence: 0, + lastPositiveByteAt: 0, + technicalRecoveryCount: 0, + paused: false, + reconnectUntil: 0, + nextRetryAt: 0, + providerCooldownUntil: 0, + blockedOnDisk: false, + blockedOnThrottleUntil: 0, + activePhaseDeadlineAt: 0, + terminalFailure: false, + manualStop: false, + shuttingDown: false, + currentSpeedBps: 0, + ...overrides + }; +} + +function evaluate( + state: DownloadHealthState, + current: DownloadHealthSnapshot, + now: number, + overrides: Partial[3]> = {} +) { + return evaluateDownloadHealth(state, current, now, { + stallAfterMs: 90_000, + cooldownMs: 600_000, + notifyOnStall: true, + notifyOnRecovery: true, + ...overrides + }); +} + +function sampleTimes( + initial: DownloadHealthState, + times: number[], + current: DownloadHealthSnapshot = snapshot() +) { + let state = initial; + const events = []; + for (const now of times) { + const result = evaluate(state, current, now); + state = result.state; + events.push(...result.events); + } + return { state, events }; +} + +function alertedState(now = 90_000): DownloadHealthState { + return sampleTimes(createDownloadHealthState(), [0, 45_000, now]).state; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("evaluateDownloadHealth", () => { + it("keeps a 20 to 30 second silent interval below the alert boundary", () => { + const result = sampleTimes(createDownloadHealthState(), [0, 15_000, 30_000]); + + expect(result.events).toEqual([]); + expect(result.state.status).toBe("suspect_no_data"); + expect(result.state.suspiciousDurationMs).toBe(30_000); + expect(result.state.suspiciousSamples).toBe(3); + }); + + it("confirms a no-data stall only after 90 seconds and at least three suspicious samples", () => { + const before = sampleTimes(createDownloadHealthState(), [0, 45_000]); + const result = evaluate(before.state, snapshot(), 90_000); + + expect(before.events).toEqual([]); + expect(result.state.status).toBe("alerted"); + expect(result.events).toEqual([ + expect.objectContaining({ type: "download_stalled", priority: "error" }) + ]); + }); + + it("does not alert from elapsed time until the third suspicious sample", () => { + const result = sampleTimes(createDownloadHealthState(), [0, 90_000]); + + expect(result.events).toEqual([]); + expect(result.state.suspiciousDurationMs).toBe(90_000); + expect(result.state.suspiciousSamples).toBe(2); + }); + + it("classifies a startable queue with no scheduler as a scheduler suspicion", () => { + const result = evaluate(createDownloadHealthState(), snapshot({ + activeTasks: 0, + startableItems: 2, + lastSchedulerTickAt: 0 + }), 45_000); + + expect(result.state.status).toBe("suspect_scheduler"); + expect(result.events).toEqual([]); + }); + + it("treats a recent scheduler tick without an active task as healthy startup activity", () => { + const result = evaluate(createDownloadHealthState(), snapshot({ + activeTasks: 0, + startableItems: 2, + lastSchedulerTickAt: 29_000 + }), 30_000); + + expect(result.state.status).toBe("healthy"); + expect(result.state.suspiciousSamples).toBe(0); + }); + + it.each([ + ["pause", { paused: true }], + ["reconnect", { reconnectUntil: 120_000 }], + ["future retry", { activeTasks: 0, startableItems: 0, nextRetryAt: 120_000 }], + ["provider cooldown", { activeTasks: 0, startableItems: 0, providerCooldownUntil: 120_000 }], + ["disk wait", { blockedOnDisk: true }], + ["bandwidth throttle", { blockedOnThrottleUntil: 120_000 }], + ["valid phase deadline", { activePhaseDeadlineAt: 120_000 }] + ])("freezes accumulated suspicion during %s", (_name, waitState) => { + const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]); + const waiting = evaluate(suspicious.state, snapshot(waitState), 60_000); + const resumed = evaluate(waiting.state, snapshot(), 90_000); + + expect(waiting.state.status).toBe("expected_wait"); + expect(waiting.state.suspiciousDurationMs).toBe(30_000); + expect(waiting.state.suspiciousSamples).toBe(2); + expect(resumed.events).toEqual([]); + expect(resumed.state.suspiciousDurationMs).toBe(60_000); + }); + + it("resets suspicion after a positive byte sequence", () => { + const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]); + const result = evaluate(suspicious.state, snapshot({ + downloadProgressSequence: 1, + lastPositiveByteAt: 45_000 + }), 45_000); + + expect(result.events).toEqual([]); + expect(result.state.status).toBe("healthy"); + expect(result.state.suspiciousDurationMs).toBe(0); + expect(result.state.suspiciousSamples).toBe(0); + }); + + it("resets suspicion after a successful item completion sequence", () => { + const suspicious = sampleTimes(createDownloadHealthState(), [0, 30_000]); + const result = evaluate(suspicious.state, snapshot({ itemCompletionSequence: 1 }), 45_000); + + expect(result.events).toEqual([]); + expect(result.state.status).toBe("healthy"); + expect(result.state.suspiciousDurationMs).toBe(0); + }); + + it("ignores speed, progress, item timestamps and global totals as progress evidence", () => { + const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]); + const current = { + ...snapshot({ currentSpeedBps: 900_000_000 }), + progressPercent: 99, + updatedAt: 90_000, + totalDownloadedBytes: 10_000_000_000 + } as DownloadHealthSnapshot; + const result = evaluate(suspicious.state, current, 90_000); + + expect(result.state.status).toBe("alerted"); + expect(result.events).toHaveLength(1); + }); + + it("does not treat sequence decreases or a technical recovery attempt as progress", () => { + const initial = createDownloadHealthState({ + downloadProgressSequence: 8, + itemCompletionSequence: 3 + }); + const result = sampleTimes(initial, [0, 45_000, 90_000], snapshot({ + downloadProgressSequence: 2, + itemCompletionSequence: 1, + technicalRecoveryCount: 1 + })); + + expect(result.state.status).toBe("alerted"); + expect(result.events).toHaveLength(1); + }); + + it("requires two positive-byte samples before recovering an alerted incident", () => { + const alerted = alertedState(); + const first = evaluate(alerted, snapshot({ + downloadProgressSequence: 1, + lastPositiveByteAt: 105_000 + }), 105_000); + const second = evaluate(first.state, snapshot({ + downloadProgressSequence: 2, + lastPositiveByteAt: 120_000 + }), 120_000); + + expect(first.state.status).toBe("recovering"); + expect(first.events).toEqual([]); + expect(second.state.status).toBe("healthy"); + expect(second.events).toEqual([ + expect.objectContaining({ type: "download_recovered", priority: "success" }) + ]); + }); + + it("recovers immediately after a successful item completion", () => { + const result = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000); + + expect(result.state.status).toBe("healthy"); + expect(result.events).toEqual([ + expect.objectContaining({ type: "download_recovered" }) + ]); + }); + + it.each([ + ["terminal failure", { terminalFailure: true }], + ["manual stop", { runActive: false, manualStop: true }], + ["shutdown", { runActive: false, shuttingDown: true }] + ])("closes an alerted incident without recovery after %s", (_name, endState) => { + const result = evaluate(alertedState(), snapshot(endState), 105_000); + + expect(result.state.status).toBe("idle"); + expect(result.events).toEqual([]); + expect(result.state.alertedAt).toBe(0); + }); + + it("applies a ten-minute cooldown before another incident event", () => { + const firstAlert = alertedState(); + const recovered = evaluate(firstAlert, snapshot({ itemCompletionSequence: 1 }), 105_000).state; + const duringCooldown = sampleTimes(recovered, [120_000, 165_000, 210_000]); + const afterCooldown = evaluate(duringCooldown.state, snapshot(), 690_000); + + expect(duringCooldown.events).toEqual([]); + expect(duringCooldown.state.status).toBe("suspect_no_data"); + expect(afterCooldown.events).toEqual([ + expect.objectContaining({ type: "download_stalled" }) + ]); + }); + + it("keeps the incident event id stable when outbox persistence rejects the state transition", () => { + const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state; + const firstAttempt = evaluate(suspicious, snapshot(), 90_000); + const retryAttempt = evaluate(suspicious, snapshot(), 105_000); + + expect(firstAttempt.events[0].id).toBe(retryAttempt.events[0].id); + }); + + it("omits identifiers, paths, URLs, providers and accounts from incident and recovery payloads", () => { + const incident = evaluate(sampleTimes(createDownloadHealthState(), [0, 45_000]).state, snapshot(), 90_000).events[0]; + const recovered = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000).events[0]; + const serialized = JSON.stringify([incident, recovered]); + + expect(serialized).not.toMatch(/https?:|\\|\/downloads\/|provider|account|item-|package-/i); + expect(incident.payload.fields).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: "Offene Dateien", value: "2" }), + expect.objectContaining({ name: "Technische Wiederherstellungen", value: "0" }) + ])); + }); + + it("honors disabled incident and recovery settings independently", () => { + const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state; + const disabledIncident = evaluate(suspicious, snapshot(), 90_000, { notifyOnStall: false }); + const disabledRecovery = evaluate(alertedState(), snapshot({ itemCompletionSequence: 1 }), 105_000, { notifyOnRecovery: false }); + + expect(disabledIncident.events).toEqual([]); + expect(disabledIncident.state.status).toBe("suspect_no_data"); + expect(disabledRecovery.events).toEqual([]); + expect(disabledRecovery.state.status).toBe("healthy"); + }); +}); + +describe("download health restart persistence", () => { + it("preserves a persisted incident while startup is still idle", () => { + const persisted = createDownloadHealthState({ + status: "alerted", + runFingerprint: RUN_FINGERPRINT, + queueFingerprint: QUEUE_FINGERPRINT, + suspiciousDurationMs: 90_000, + suspiciousSamples: 3, + incidentStartedAt: 10_000, + alertedAt: 90_000, + restartPending: true + }); + + const result = evaluate(persisted, snapshot({ runActive: false, openItems: 0 }), 100_000); + + expect(result.events).toEqual([]); + expect(result.state.status).toBe("suspended"); + expect(result.state.runFingerprint).toBe(RUN_FINGERPRINT); + expect(result.state.queueFingerprint).toBe(QUEUE_FINGERPRINT); + expect(result.state.restartPending).toBe(true); + }); + + it("requires two fresh samples before re-alerting the same persisted fingerprint", () => { + const persisted = createDownloadHealthState({ + status: "suspect_no_data", + runFingerprint: RUN_FINGERPRINT, + queueFingerprint: QUEUE_FINGERPRINT, + suspiciousDurationMs: 90_000, + suspiciousSamples: 3, + incidentStartedAt: 10_000, + restartPending: true + }); + const first = evaluate(persisted, snapshot(), 100_000); + const second = evaluate(first.state, snapshot(), 115_000); + + expect(first.events).toEqual([]); + expect(first.state.restartFreshSamples).toBe(1); + expect(second.events).toEqual([ + expect.objectContaining({ type: "download_stalled" }) + ]); + }); + + it("discards a persisted incident when the queue fingerprint changes", () => { + const persisted = createDownloadHealthState({ + status: "alerted", + runFingerprint: RUN_FINGERPRINT, + queueFingerprint: QUEUE_FINGERPRINT, + suspiciousDurationMs: 90_000, + suspiciousSamples: 4, + incidentStartedAt: 10_000, + alertedAt: 90_000, + restartPending: true + }); + const result = evaluate(persisted, snapshot({ + runFingerprint: OTHER_RUN_FINGERPRINT, + queueFingerprint: OTHER_QUEUE_FINGERPRINT + }), 100_000); + + expect(result.events).toEqual([]); + expect(result.state.status).toBe("suspect_no_data"); + expect(result.state.suspiciousDurationMs).toBe(0); + expect(result.state.alertedAt).toBe(0); + }); + + it("writes an allowlisted atomic state and reloads it with a fresh-sample gate", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-state-")); + tempDirs.push(root); + const filePath = path.join(root, "health.json"); + const state = { + ...alertedState(), + privateUrl: "https://private.example.test/file", + privatePath: "C:\\private\\download.bin", + privateAccount: "private@example.test" + } as DownloadHealthState; + + saveDownloadHealthState(filePath, state); + const persisted = fs.readFileSync(filePath, "utf8"); + const loaded = loadDownloadHealthState(filePath); + + expect(persisted).not.toMatch(/private|example\.test|download\.bin/i); + expect(fs.existsSync(`${filePath}.tmp`)).toBe(false); + expect(loaded.runFingerprint).toBe(RUN_FINGERPRINT); + expect(loaded.queueFingerprint).toBe(QUEUE_FINGERPRINT); + expect(loaded.restartPending).toBe(true); + expect(loaded.restartFreshSamples).toBe(0); + }); + + it("rejects malformed queue fingerprints instead of restoring an incident", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-invalid-")); + tempDirs.push(root); + const filePath = path.join(root, "health.json"); + fs.writeFileSync(filePath, JSON.stringify({ + ...alertedState(), + queueFingerprint: "https://private.example.test/queue" + }), "utf8"); + + const loaded = loadDownloadHealthState(filePath); + + expect(loaded).toEqual(createDownloadHealthState()); + }); + + it("does not commit an alert until the outbox accepts the event", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-failure-")); + tempDirs.push(root); + const filePath = path.join(root, "health.json"); + const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state; + const monitor = new DownloadHealthMonitor(filePath, suspicious); + const eventIds: string[] = []; + + await expect(monitor.sample(snapshot(), 90_000, { + stallAfterMs: 90_000, + cooldownMs: 600_000, + notifyOnStall: true, + notifyOnRecovery: true + }, async (event) => { + eventIds.push(event.id); + throw new Error("outbox unavailable"); + })).rejects.toThrow("outbox unavailable"); + await expect(monitor.sample(snapshot(), 105_000, { + stallAfterMs: 90_000, + cooldownMs: 600_000, + notifyOnStall: true, + notifyOnRecovery: true + }, async (event) => { + eventIds.push(event.id); + throw new Error("outbox unavailable"); + })).rejects.toThrow("outbox unavailable"); + + expect(eventIds[0]).toBe(eventIds[1]); + expect(monitor.getState().status).toBe("suspect_no_data"); + expect(fs.existsSync(filePath)).toBe(false); + }); + + it("persists the alerted state after the outbox accepts the event", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-outbox-success-")); + tempDirs.push(root); + const filePath = path.join(root, "health.json"); + const suspicious = sampleTimes(createDownloadHealthState(), [0, 45_000]).state; + const monitor = new DownloadHealthMonitor(filePath, suspicious); + + const result = await monitor.sample(snapshot(), 90_000, { + stallAfterMs: 90_000, + cooldownMs: 600_000, + notifyOnStall: true, + notifyOnRecovery: true + }, async () => undefined); + + expect(result.events).toHaveLength(1); + expect(monitor.getState().status).toBe("alerted"); + expect(loadDownloadHealthState(filePath)).toEqual(expect.objectContaining({ + status: "alerted", + restartPending: true + })); + }); +}); diff --git a/tests/download-manager.test.ts b/tests/download-manager.test.ts index 5175b0d..84c703f 100644 --- a/tests/download-manager.test.ts +++ b/tests/download-manager.test.ts @@ -14696,3 +14696,245 @@ describe("package lifecycle telemetry boundaries", () => { expect(pkg.terminalAt).toBe(pkg.postProcessCompletedAt); }); }); + +describe("download health snapshot", () => { + function createHealthManager(root: string) { + const session = emptySession(); + const packageId = "private-package-id"; + const itemId = "private-item-id"; + const now = 100_000; + session.running = true; + session.runStartedAt = 10_000; + session.packageOrder = [packageId]; + session.packages[packageId] = { + id: packageId, + name: "Private package name", + outputDir: path.join(root, "private-output"), + extractDir: path.join(root, "private-extract"), + status: "downloading", + itemIds: [itemId], + cancelled: false, + enabled: true, + createdAt: 1_000, + updatedAt: now + }; + session.items[itemId] = { + id: itemId, + packageId, + url: "https://private.example.test/file", + provider: "realdebrid", + providerLabel: "Private provider label", + providerAccountId: "private-account-id", + providerAccountLabel: "private@example.test", + status: "downloading", + retries: 0, + speedBps: 8192, + downloadedBytes: 4096, + totalBytes: 16384, + progressPercent: 25, + fileName: "private-file.bin", + targetPath: path.join(root, "private-output", "private-file.bin"), + resumable: true, + attempts: 1, + lastError: "", + fullStatus: "Download läuft", + createdAt: 2_000, + updatedAt: now + }; + const manager = new DownloadManager(defaultSettings(), session, createStoragePaths(path.join(root, "state"))); + const state = manager as any; + session.running = true; + session.items[itemId].status = "downloading"; + session.items[itemId].speedBps = 8192; + session.packages[packageId].status = "downloading"; + state.runItemIds.add(itemId); + state.runPackageIds.add(packageId); + return { manager, session, state, packageId, itemId }; + } + + it("exposes only aggregate run-scope health data and opaque fingerprints", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-snapshot-")); + tempDirs.push(root); + const { manager, state, packageId, itemId } = createHealthManager(root); + state.lastSchedulerTickAt = 99_000; + state.activeTasks.set(itemId, { + itemId, + packageId, + abortController: new AbortController(), + abortReason: "none", + resumable: true, + nonResumableCounted: false, + phase: "downloading", + phaseStartedAt: 90_000, + phaseDeadlineAt: 0, + blockedOnDiskWrite: false, + blockedOnDiskSince: 0, + blockedOnThrottleUntil: 0 + }); + + const health = manager.getDownloadHealthSnapshot(100_000); + const serialized = JSON.stringify(health); + + expect(health).toMatchObject({ + runActive: true, + openItems: 1, + openPackages: 1, + knownDownloadedBytes: 4096, + activeTasks: 1, + startableItems: 0, + lastSchedulerTickAt: 99_000, + currentSpeedBps: 8192 + }); + expect(health.runFingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(health.queueFingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(serialized).not.toMatch(/private|example\.test|realdebrid|file\.bin/i); + }); + + it("keeps the run fingerprint stable across restart time changes for the same generation scope", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-fingerprint-")); + tempDirs.push(root); + const { manager, session } = createHealthManager(root); + session.packages[session.packageOrder[0]].resultGeneration = 4; + + const before = manager.getDownloadHealthSnapshot(100_000); + session.runStartedAt = 200_000; + const after = manager.getDownloadHealthSnapshot(210_000); + + expect(after.runFingerprint).toBe(before.runFingerprint); + expect(after.queueFingerprint).toBe(before.queueFingerprint); + }); + + it("advances progress sequences only for positive byte events and successful item completions", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-sequences-")); + tempDirs.push(root); + const { manager, session, state, packageId, itemId } = createHealthManager(root); + + state.recordSpeed(1024, packageId); + state.recordSpeed(0, packageId); + state.recordSpeed(-512, packageId); + session.totalDownloadedBytes = 10_000_000; + session.totalDownloadedBytes = 0; + state.recordRunOutcome(itemId, "completed"); + state.recordRunOutcome(itemId, "completed"); + + const health = manager.getDownloadHealthSnapshot(100_000); + + expect(health.downloadProgressSequence).toBe(1); + expect(health.lastPositiveByteAt).toBeGreaterThan(0); + expect(health.itemCompletionSequence).toBe(1); + }); + + it("projects retry, cooldown, disk, throttle and valid phase waits without stale disk events", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-waits-")); + tempDirs.push(root); + const { manager, session, state, packageId, itemId } = createHealthManager(root); + const active = { + itemId, + packageId, + abortController: new AbortController(), + abortReason: "none", + resumable: true, + nonResumableCounted: false, + phase: "validating", + phaseStartedAt: 90_000, + phaseDeadlineAt: 130_000, + blockedOnDiskWrite: true, + blockedOnDiskSince: 95_000, + blockedOnThrottleUntil: 120_000 + }; + state.activeTasks.set(itemId, active); + state.diskWaitEvents = [{ + phase: "download", + targetPath: "C:\\private\\disk", + requiredBytes: 100, + availableBytes: 0, + reserveBytes: 0, + retryAt: 80_000, + itemId, + packageId + }]; + + const activeWait = manager.getDownloadHealthSnapshot(100_000); + + expect(activeWait.blockedOnDisk).toBe(true); + expect(activeWait.blockedOnThrottleUntil).toBe(120_000); + expect(activeWait.activePhaseDeadlineAt).toBe(130_000); + + state.activeTasks.clear(); + session.items[itemId].status = "queued"; + state.retryAfterByItem.set(itemId, 140_000); + state.providerFailures.set("realdebrid", { count: 20, lastFailAt: 99_000, cooldownUntil: 150_000 }); + + const queuedWait = manager.getDownloadHealthSnapshot(100_000); + + expect(queuedWait.startableItems).toBe(0); + expect(queuedWait.nextRetryAt).toBe(140_000); + expect(queuedWait.providerCooldownUntil).toBe(150_000); + expect(queuedWait.blockedOnDisk).toBe(false); + }); + + it("does not project a partial retry as a global wait while another download is active", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-partial-wait-")); + tempDirs.push(root); + const { manager, session, state, packageId, itemId } = createHealthManager(root); + const queuedId = "private-queued-id"; + session.items[queuedId] = { + ...session.items[itemId], + id: queuedId, + status: "queued", + downloadedBytes: 0, + speedBps: 0 + }; + session.packages[packageId].itemIds.push(queuedId); + state.runItemIds.add(queuedId); + state.activeTasks.set(itemId, { + itemId, + packageId, + abortController: new AbortController(), + abortReason: "none", + resumable: true, + nonResumableCounted: false, + phase: "downloading", + phaseStartedAt: 90_000, + phaseDeadlineAt: 0, + blockedOnDiskWrite: false, + blockedOnDiskSince: 0, + blockedOnThrottleUntil: 0 + }); + state.retryAfterByItem.set(queuedId, 140_000); + + const health = manager.getDownloadHealthSnapshot(100_000); + + expect(health.activeTasks).toBe(1); + expect(health.nextRetryAt).toBe(0); + }); + + it("counts one technical recovery when the existing global watchdog restarts stalled tasks", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-health-watchdog-")); + tempDirs.push(root); + const { manager, state, packageId, itemId } = createHealthManager(root); + const active = { + itemId, + packageId, + abortController: new AbortController(), + abortReason: "none", + resumable: true, + nonResumableCounted: false, + phase: "downloading", + phaseStartedAt: 1, + phaseDeadlineAt: 0, + blockedOnDiskWrite: false, + blockedOnDiskSince: 0, + blockedOnThrottleUntil: 0 + }; + state.activeTasks.set(itemId, active); + state.lastGlobalProgressBytes = 0; + state.lastGlobalProgressAt = 1; + + state.runGlobalStallWatchdog(90_000); + + expect(active.abortController.signal.aborted).toBe(true); + expect(active.abortReason).toBe("stall"); + expect(manager.getDownloadHealthSnapshot(90_000).technicalRecoveryCount).toBe(1); + }); +}); diff --git a/tests/support-data.test.ts b/tests/support-data.test.ts index bd580ae..8db13dd 100644 --- a/tests/support-data.test.ts +++ b/tests/support-data.test.ts @@ -1,6 +1,12 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import AdmZip from "adm-zip"; import { describe, expect, it } from "vitest"; import { defaultSettings } from "../src/main/constants"; import { buildAccountSummary, buildStatsPayload } from "../src/main/support-data"; +import { buildSupportBundle } from "../src/main/support-bundle"; +import { createStoragePaths } from "../src/main/storage"; import { serializeRealDebridApiAccounts } from "../src/shared/real-debrid-accounts"; import { createVisualFixture } from "./visual/fixtures"; @@ -53,4 +59,28 @@ describe("Real-Debrid support summary", () => { expect(serialized).toContain("realdebrid"); expect(serialized).toContain("4096"); }); + + it("keeps the persisted notification health incident outside support bundles", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-support-health-")); + try { + const paths = createStoragePaths(root); + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync(paths.notificationHealthFile, "PRIVATE_HEALTH_INCIDENT_PAYLOAD", "utf8"); + const snapshot = structuredClone(createVisualFixture("empty").snapshot); + const manager = { + getSnapshot: () => snapshot, + getPackageLogPath: () => null, + getItemLogPath: () => null + }; + + const buffer = await buildSupportBundle(manager as any, root, { hostDiagnosticsMode: "none" }); + const zip = new AdmZip(buffer); + const entries = zip.getEntries().map((entry) => entry.entryName); + + expect(entries).not.toContain(path.basename(paths.notificationHealthFile)); + expect(buffer.toString("utf8")).not.toContain("PRIVATE_HEALTH_INCIDENT_PAYLOAD"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); });