Administrator/real-debrid-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 510 Wiki Activity

Fix: Komplett leere Liste nach unsauberem Neustart (Datenverlust bei Stromausfall/Crash waehrend Download)

Browse Source 
Ursache: Session-Writes (writeFile + atomic rename) liefen ohne fsync. Waehrend eines
Downloads feuert persistSoon alle 700ms-3s, die Session-Datei bleibt damit dauerhaft
dirty im OS-Cache. Bei hartem Stromausfall auf NTFS ist die rename-Metadatentransaktion
journaled (durable), aber die Datenbloecke der temp-Datei sind nicht geflusht -> primary
zeigt nach Reboot auf Null/Garbage. Die .bak-Kopie stammt per copyFileSync aus einer
ebenfalls ungeflushten primary -> ebenfalls korrupt. loadSession faellt durch
primary -> bak -> temp auf emptySession() durch, und der naechste persistSoon speichert
diese leere Session ueber die Platte -> dauerhaft leer. Tritt nur bei UNSAUBEREM Neustart
auf (sauberes Beenden flusht ohnehin).

Fix in drei Schichten:
- Durable atomic write: temp wird vor dem rename gefsynct. Reihenfolge zwingend
  write -> fsync -> close -> rename (NTFS kann eine Datei mit offenem Handle nicht renamen).
  Sync-Pfad via openSync/writeSync/fsyncSync/closeSync, Async-Pfad via FileHandle.sync()
  (laeuft auf dem libuv-Threadpool, blockiert den Hot-Path nicht). Kein Throttle: ein
  throttle-skip wuerde eine ungeflushte temp ueber die durable primary renamen und das
  Korruptionsfenster wieder oeffnen.
- Read-Retry: readSessionFile wiederholt bei transienten Sperren (EBUSY/EPERM/EAGAIN, z.B.
  Virenscanner/Disk-not-ready beim Boot) 5x mit Backoff. EACCES und JSON-Parse-Fehler
  werden nicht wiederholt.
- Empty-Clobber-Guard: loadSessionWithStatus meldet, ob alle Tiers unlesbar waren (Status
  empty-unreadable). In dem Fall blockiert der DownloadManager das Speichern einer leeren
  Session ueber vorhandene Daten, bis wieder echte Daten vorliegen; die erste nicht-leere
  Speicherung hebt den Schutz auf.

Tests: tests/session-restart-loss.test.ts um Status-Klassifizierung, fsync-Nachweis,
async-Roundtrip (close-before-rename), EBUSY-Retry und Guard-Clear-Pfad erweitert.
Suite 929 gruen, tsc unveraendert bei 6 Baseline-Fehlern.
This commit is contained in:
Sucukdeluxe 2026-06-19 22:42:05 +02:00
parent 8f3681b160
commit 6cde08dac3
4 changed files with 129 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/app-controller.ts
View File

@ -39,7 +39,7 @@ import { getItemLogPath, initItemLogs, shutdownItemLogs } from "./item-log";
import { getPackageLogPath, initPackageLogs, shutdownPackageLogs } from "./package-log";
import { initSessionLog, getSessionLogPath, shutdownSessionLog } from "./session-log";
import { MegaWebFallback } from "./mega-web-fallback";
import { addHistoryEntry, addHistoryEntryForRetention, cancelPendingAsyncSaves, clearHistory, createStoragePaths, loadHistory, loadHistoryForRetention, loadSession, loadSettings, normalizeHistoryEntry, normalizeLoadedSession, normalizeLoadedSessionTransientFields, normalizeSettings, removeHistoryEntry, resetHistoryForRetention, saveHistory, saveSession, saveSettings } from "./storage";
import { addHistoryEntry, addHistoryEntryForRetention, cancelPendingAsyncSaves, clearHistory, createStoragePaths, loadHistory, loadHistoryForRetention, loadSessionWithStatus, loadSettings, normalizeHistoryEntry, normalizeLoadedSession, normalizeLoadedSessionTransientFields, normalizeSettings, removeHistoryEntry, resetHistoryForRetention, saveHistory, saveSession, saveSettings } from "./storage";
import { abortActiveUpdateDownload, checkGitHubUpdate, installLatestUpdate } from "./update";
import { runInstallWithResume } from "./update-install-flow";
import { rotateDebugToken, startDebugServer, stopDebugServer, restartDebugServer, getDebugServerRuntimeStatus, getActiveDebugToken, getDebugAllowlist, writeDebugServerConfig, clearDebugToken } from "./debug-server";
@ -112,7 +112,8 @@ export class AppController {
initTraceLog(this.storagePaths.baseDir);
this.settings = loadSettings(this.storagePaths);
resetHistoryForRetention(this.storagePaths, this.settings.historyRetentionMode);
const session = loadSession(this.storagePaths);
const loadResult = loadSessionWithStatus(this.storagePaths);
const session = loadResult.session;
this.megaWebFallback = new MegaWebFallback(() => ({
login: this.settings.megaLogin,
password: this.settings.megaPassword
@ -126,6 +127,7 @@ export class AppController {
realDebridWebUnrestrict: (link: string, signal?: AbortSignal) => this.realDebridWebFallback.unrestrict(link, signal),
bestDebridWebUnrestrict: (link: string, signal?: AbortSignal) => this.bestDebridWebFallback.unrestrict(link, signal),
invalidateMegaSession: () => this.megaWebFallback.invalidateSession(),
protectEmptyClobber: loadResult.status === "empty-unreadable",
onHistoryEntry: (entry: HistoryEntry) => {
addHistoryEntryForRetention(this.storagePaths, this.settings.historyRetentionMode, entry, this.historyLimits());
}

32
src/main/download-manager.ts
View File

@ -363,6 +363,7 @@ type DownloadManagerOptions = {
bestDebridWebUnrestrict?: BestDebridWebUnrestrictor;
invalidateMegaSession?: () => void;
onHistoryEntry?: HistoryEntryCallback;
protectEmptyClobber?: boolean;
};
function generateHistoryId(): string {
@ -1688,6 +1689,10 @@ export class DownloadManager extends EventEmitter {
public blockAllPersistence = false;
private protectAgainstEmptyClobber = false;
private emptyClobberProtectionLogged = false;
private debridService: DebridService;
private invalidateMegaSessionFn?: () => void;
@ -1831,6 +1836,10 @@ export class DownloadManager extends EventEmitter {
this.session = session;
this.itemCount = Object.keys(this.session.items).length;
this.storagePaths = storagePaths;
this.protectAgainstEmptyClobber = Boolean(options.protectEmptyClobber);
if (this.protectAgainstEmptyClobber) {
logger.warn("Session-Schutz aktiv: Start mit unlesbarer Session — leere Speicherungen blockiert, bis echte Daten vorliegen");
}
this.debridService = new DebridService(settings, {
megaWebUnrestrict: options.megaWebUnrestrict,
allDebridWebUnrestrict: options.allDebridWebUnrestrict,
@ -5821,7 +5830,9 @@ export class DownloadManager extends EventEmitter {
const itemCount = Object.keys(this.session.items).length;
logger.info(`Shutdown-Save: ${pkgCount} Pakete, ${itemCount} Items`);
this.foldRuntimeIntoSettings(nowMs());
if (!this.guardBlocksSessionSave()) {
saveSession(this.storagePaths, this.session);
}
saveSettings(this.storagePaths, this.settings);
} else {
logger.info(`Shutdown-Save übersprungen: skipShutdownPersist=${this.skipShutdownPersist}, blockAllPersistence=${this.blockAllPersistence}`);
@ -6124,10 +6135,29 @@ export class DownloadManager extends EventEmitter {
}, delay);
}
private guardBlocksSessionSave(): boolean {
if (!this.protectAgainstEmptyClobber) {
return false;
}
const isEmpty = Object.keys(this.session.packages).length === 0 && Object.keys(this.session.items).length === 0;
if (isEmpty) {
if (!this.emptyClobberProtectionLogged) {
logger.warn("Leere Session-Speicherung uebersprungen (Schutz nach unlesbarem Start) — vorhandene Datei bleibt unangetastet");
this.emptyClobberProtectionLogged = true;
}
return true;
}
this.protectAgainstEmptyClobber = false;
logger.info("Session-Schutz aufgehoben: nicht-leere Session wird wieder normal gespeichert");
return false;
}
private persistNow(): void {
const now = nowMs();
this.lastPersistAt = now;
if (!this.guardBlocksSessionSave()) {
void saveSessionAsync(this.storagePaths, this.session).catch((err) => logger.warn(`saveSessionAsync Fehler: ${compactErrorText(err)}`));
}
if (now - this.lastSettingsPersistAt >= 30000) {
this.foldRuntimeIntoSettings(now);
this.lastSettingsPersistAt = now;
@ -6141,7 +6171,9 @@ export class DownloadManager extends EventEmitter {
const itemCount = Object.keys(this.session.items).length;
logger.info(`Pre-Update Sync-Save: ${pkgCount} Pakete, ${itemCount} Items`);
this.foldRuntimeIntoSettings(nowMs());
if (!this.guardBlocksSessionSave()) {
saveSession(this.storagePaths, this.session);
}
saveSettings(this.storagePaths, this.settings);
}

119
src/main/storage.ts
View File

@ -890,21 +890,50 @@ export function normalizeLoadedSessionTransientFields(session: SessionState): Se
return session;
}
const TRANSIENT_READ_CODES = new Set(["EBUSY", "EPERM", "EAGAIN"]);
function sleepSyncMs(ms: number): void {
if (ms <= 0) {
return;
}
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}
function readSessionFile(filePath: string): SessionState | null {
let raw: string | null = null;
const maxAttempts = 5;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
try {
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown;
raw = fs.readFileSync(filePath, "utf8");
break;
} catch (error) {
const code = (error as NodeJS.ErrnoException)?.code || "";
if (TRANSIENT_READ_CODES.has(code) && attempt < maxAttempts) {
const backoffMs = 100 * 2 ** (attempt - 1);
logger.warn(`Session-Datei vorübergehend gesperrt (${code}), Versuch ${attempt}/${maxAttempts}, warte ${backoffMs}ms: ${filePath}`);
sleepSyncMs(backoffMs);
continue;
}
if (code === "EACCES" || code === "EPERM") {
logger.error(`Session-Datei nicht zugreifbar (${code}): ${filePath} - pruefe Datei-/Ordner-Berechtigungen fuer Benutzer ${process.env.USERNAME || process.env.USER || "?"}`);
} else {
logger.error(`Session-Datei nicht lesbar (${code || "?"}): ${filePath}: ${String(error)}`);
}
return null;
}
}
if (raw === null) {
return null;
}
try {
const parsed = JSON.parse(raw) as unknown;
const session = normalizeLoadedSessionTransientFields(normalizeLoadedSession(parsed));
const pkgCount = Object.keys(session.packages).length;
const itemCount = Object.keys(session.items).length;
logger.info(`Session geladen: ${filePath} (${pkgCount} Pakete, ${itemCount} Items)`);
return session;
} catch (error) {
const code = (error as NodeJS.ErrnoException)?.code || "";
if (code === "EACCES" || code === "EPERM") {
logger.error(`Session-Datei nicht zugreifbar (${code}): ${filePath} - pruefe Datei-/Ordner-Berechtigungen fuer Benutzer ${process.env.USERNAME || process.env.USER || "?"}`);
} else {
logger.error(`Session-Datei nicht lesbar: ${filePath}: ${String(error)}`);
}
logger.error(`Session-Datei beschädigt (JSON ungültig): ${filePath}: ${String(error)}`);
return null;
}
}
@ -1004,17 +1033,31 @@ export function emptySession(): SessionState {
};
}
export function loadSession(paths: StoragePaths): SessionState {
export type SessionLoadStatus =
| "ok"
| "recovered-backup"
| "recovered-temp"
| "empty-fresh"
| "empty-unreadable";
export interface SessionLoadResult {
session: SessionState;
status: SessionLoadStatus;
}
export function loadSessionWithStatus(paths: StoragePaths): SessionLoadResult {
ensureBaseDir(paths.baseDir);
const backupFile = sessionBackupPath(paths.sessionFile);
const syncTempFile = sessionTempPath(paths.sessionFile, "sync");
const asyncTempFile = sessionTempPath(paths.sessionFile, "async");
const primaryExists = fs.existsSync(paths.sessionFile);
const backupExists = fs.existsSync(backupFile);
const anyTempExists = fs.existsSync(syncTempFile) || fs.existsSync(asyncTempFile);
if (!primaryExists) {
const hasRecoverable = fs.existsSync(backupFile)
|| fs.existsSync(sessionTempPath(paths.sessionFile, "sync"))
|| fs.existsSync(sessionTempPath(paths.sessionFile, "async"));
if (!hasRecoverable) {
if (!backupExists && !anyTempExists) {
logger.info("Keine Session-Datei vorhanden, starte mit leerer Session");
return emptySession();
return { session: emptySession(), status: "empty-fresh" };
}
logger.warn("Session-Primaerdatei fehlt, aber Backup/Temp vorhanden — Wiederherstellung wird versucht");
}
@ -1023,7 +1066,7 @@ export function loadSession(paths: StoragePaths): SessionState {
if (primary) {
const primaryPkgCount = Object.keys(primary.packages).length;
if (primaryPkgCount === 0 && fs.existsSync(backupFile)) {
if (primaryPkgCount === 0 && backupExists) {
const backup = readSessionFile(backupFile);
if (backup) {
const backupPkgCount = Object.keys(backup.packages).length;
@ -1031,29 +1074,27 @@ export function loadSession(paths: StoragePaths): SessionState {
logger.warn(`Session-Datei ist leer (0 Pakete), aber Backup hat ${backupPkgCount} Pakete — verwende Backup`);
try {
const payload = JSON.stringify({ ...backup, updatedAt: Date.now() }, safeJsonReplacer);
const tempPath = sessionTempPath(paths.sessionFile, "sync");
fs.writeFileSync(tempPath, payload, "utf8");
syncRenameWithExdevFallback(tempPath, paths.sessionFile);
fs.writeFileSync(syncTempFile, payload, "utf8");
syncRenameWithExdevFallback(syncTempFile, paths.sessionFile);
} catch {
}
return backup;
return { session: backup, status: "recovered-backup" };
}
}
}
return primary;
return { session: primary, status: "ok" };
}
const backup = fs.existsSync(backupFile) ? readSessionFile(backupFile) : null;
const backup = backupExists ? readSessionFile(backupFile) : null;
if (backup) {
logger.warn("Session defekt, Backup-Datei wird verwendet");
try {
const payload = JSON.stringify({ ...backup, updatedAt: Date.now() }, safeJsonReplacer);
const tempPath = sessionTempPath(paths.sessionFile, "sync");
fs.writeFileSync(tempPath, payload, "utf8");
syncRenameWithExdevFallback(tempPath, paths.sessionFile);
fs.writeFileSync(syncTempFile, payload, "utf8");
syncRenameWithExdevFallback(syncTempFile, paths.sessionFile);
} catch {
}
return backup;
return { session: backup, status: "recovered-backup" };
}
for (const kind of ["sync", "async"] as const) {
@ -1067,13 +1108,21 @@ export function loadSession(paths: StoragePaths): SessionState {
fs.writeFileSync(paths.sessionFile, payload, "utf8");
} catch {
}
return tmpSession;
return { session: tmpSession, status: "recovered-temp" };
}
}
}
logger.error("Session konnte nicht geladen werden (Primary, Backup und Temp-Dateien fehlgeschlagen)");
return emptySession();
if (primaryExists || backupExists || anyTempExists) {
logger.error("Session konnte nicht geladen werden (Primary, Backup und Temp-Dateien fehlgeschlagen) — Schutz gegen leeres Ueberschreiben aktiv");
return { session: emptySession(), status: "empty-unreadable" };
}
return { session: emptySession(), status: "empty-fresh" };
}
export function loadSession(paths: StoragePaths): SessionState {
return loadSessionWithStatus(paths).session;
}
export function saveSession(paths: StoragePaths, session: SessionState): void {
@ -1088,7 +1137,13 @@ export function saveSession(paths: StoragePaths, session: SessionState): void {
const payload = JSON.stringify({ ...session, updatedAt: Date.now() }, safeJsonReplacer);
const tempPath = sessionTempPath(paths.sessionFile, "sync");
try {
fs.writeFileSync(tempPath, payload, "utf8");
const fd = fs.openSync(tempPath, "w");
try {
fs.writeSync(fd, payload);
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
syncRenameWithExdevFallback(tempPath, paths.sessionFile);
} catch (error) {
try { fs.rmSync(tempPath, { force: true }); } catch { }
@ -1104,7 +1159,13 @@ async function writeSessionPayload(paths: StoragePaths, payload: string, generat
await fs.promises.mkdir(paths.baseDir, { recursive: true });
await fsp.copyFile(paths.sessionFile, sessionBackupPath(paths.sessionFile)).catch(() => {});
const tempPath = sessionTempPath(paths.sessionFile, "async");
await fsp.writeFile(tempPath, payload, "utf8");
const handle = await fsp.open(tempPath, "w");
try {
await handle.writeFile(payload, "utf8");
await handle.sync();
} finally {
await handle.close();
}
if (generation < syncSaveGeneration) {
await fsp.rm(tempPath, { force: true }).catch(() => {});
return;

BIN
tests/session-restart-loss.test.ts
View File

Binary file not shown.