Administrator/Twitch-VOD-Manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 212 Wiki Activity

harden: atomic fsync writes + per-item filename claims

Browse Source 
Two server-side correctness fixes for parallel downloads and crash recovery.

1. Atomic file writes survive power loss / crash mid-write.
   saveConfig and writeQueueToDisk used writeFileSync + renameSync. Node's
   writeFileSync does NOT fsync — a power loss between write and rename can
   leave the renamed file empty or truncated, and the next launch silently
   falls back to defaults / empty queue.
   New writeFileAtomicSync helper: openSync + writeSync + fsyncSync +
   closeSync + renameSync (with the existing Windows copy fallback). fsync
   failure is non-fatal (some FS reject it) but file ordering is preserved.

2. Per-item claimed filenames fix the parallel-download race.
   With max 2 parallel downloads, processOneQueueItem.finally was calling
   claimedFilenames.clear() — wiping every parallel item's claims when any
   one finished. In the window between an active item claiming a filename
   and streamlink actually writing the first bytes, a third item could
   compute the same filename and both downloads would race the same path.
   New Map<itemId, Set<filename>> tracks claims per active download.
   ensureUniqueFilename(path, itemId) registers per-item;
   releaseClaimedFilenamesForItem(itemId) removes only that item's claims.
   splitMergedFile gained an itemId parameter for the same reason. The
   dead releaseClaimedFilename(path) function was removed.

Build: tsc clean. Tests: smoke + smoke-template-guide + smoke-full + merge-split
+ update-version-logic all pass. No new ESLint warnings.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
xRangerDE 2026-05-03 15:10:15 +02:00
parent 54197af863
commit 8d0cb4cefd
1 changed files with 53 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
src/main.ts
View File

@ -268,17 +268,33 @@ function loadConfig(): Config {
return normalizeConfigTemplates(defaultConfig);
}
function saveConfig(config: Config): void {
const tmpPath = CONFIG_FILE + '.tmp';
function writeFileAtomicSync(targetPath: string, payload: string | Buffer): void {
const buffer = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, 'utf-8');
const tmpPath = targetPath + '.tmp';
let fd: number | null = null;
try {
fs.writeFileSync(tmpPath, JSON.stringify(config, null, 2));
try {
fs.renameSync(tmpPath, CONFIG_FILE);
} catch {
// On Windows, rename can fail if target exists in some edge cases
fs.copyFileSync(tmpPath, CONFIG_FILE);
try { fs.unlinkSync(tmpPath); } catch { }
fd = fs.openSync(tmpPath, 'w');
fs.writeSync(fd, buffer, 0, buffer.length, 0);
try { fs.fsyncSync(fd); } catch { /* fsync may fail on some FS; rename is still safer than nothing */ }
} finally {
if (fd !== null) {
try { fs.closeSync(fd); } catch { }
}
}
try {
fs.renameSync(tmpPath, targetPath);
} catch {
// On Windows, rename can fail if target exists or is locked. Fall back to copy.
fs.copyFileSync(tmpPath, targetPath);
try { fs.unlinkSync(tmpPath); } catch { }
}
}
function saveConfig(config: Config): void {
try {
writeFileAtomicSync(CONFIG_FILE, JSON.stringify(config, null, 2));
} catch (e) {
console.error('Error saving config:', e);
}
@ -329,16 +345,8 @@ function writeQueueToDisk(queue: QueueItem[]): void {
return;
}
const tmpPath = QUEUE_FILE + '.tmp';
try {
fs.writeFileSync(tmpPath, JSON.stringify(queue, null, 2));
try {
fs.renameSync(tmpPath, QUEUE_FILE);
} catch {
// On Windows, rename can fail if target exists in some edge cases
fs.copyFileSync(tmpPath, QUEUE_FILE);
try { fs.unlinkSync(tmpPath); } catch { }
}
writeFileAtomicSync(QUEUE_FILE, JSON.stringify(queue, null, 2));
} catch (e) {
console.error('Error saving queue:', e);
}
@ -704,8 +712,9 @@ function formatDurationDashed(seconds: number): string {
}
const claimedFilenames = new Set<string>();
const itemClaimedFilenames = new Map<string, Set<string>>();
function ensureUniqueFilename(filePath: string): string {
function ensureUniqueFilename(filePath: string, itemId: string | null = null): string {
const dir = path.dirname(filePath);
const ext = path.extname(filePath);
const base = path.basename(filePath, ext);
@ -716,11 +725,22 @@ function ensureUniqueFilename(filePath: string): string {
candidate = path.join(dir, `${base}_${counter}${ext}`);
}
claimedFilenames.add(candidate);
if (itemId) {
let perItem = itemClaimedFilenames.get(itemId);
if (!perItem) {
perItem = new Set();
itemClaimedFilenames.set(itemId, perItem);
}
perItem.add(candidate);
}
return candidate;
}
function releaseClaimedFilename(filePath: string): void {
claimedFilenames.delete(filePath);
function releaseClaimedFilenamesForItem(itemId: string): void {
const perItem = itemClaimedFilenames.get(itemId);
if (!perItem) return;
for (const f of perItem) claimedFilenames.delete(f);
itemClaimedFilenames.delete(itemId);
}
function sanitizeFilenamePart(input: string, fallback = 'unnamed'): string {
@ -2124,7 +2144,8 @@ async function splitMergedFile(
partDurationSec: number,
totalDurationSec: number,
filenameGenerator: (partNum: number) => string,
onProgress: (currentPart: number, totalParts: number) => void
onProgress: (currentPart: number, totalParts: number) => void,
itemId: string | null = null
): Promise<{ success: boolean; files: string[] }> {
const ffmpegReady = await ensureFfmpegInstalled();
if (!ffmpegReady) {
@ -2143,7 +2164,7 @@ async function splitMergedFile(
const startSec = i * partDurationSec;
const thisDuration = Math.min(partDurationSec, totalDurationSec - startSec);
const outputFile = ensureUniqueFilename(path.join(outputFolder, filenameGenerator(i + 1)));
const outputFile = ensureUniqueFilename(path.join(outputFolder, filenameGenerator(i + 1)), itemId);
onProgress(i + 1, numParts);
@ -2490,7 +2511,7 @@ async function downloadVOD(
const remainingDuration = clip.durationSec - (i * partDuration);
const thisDuration = Math.min(partDuration, remainingDuration);
const partFilename = ensureUniqueFilename(makeClipFilename(partNum, startOffset, thisDuration));
const partFilename = ensureUniqueFilename(makeClipFilename(partNum, startOffset, thisDuration), item.id);
const result = await downloadVODPart(
item.url,
@ -2513,7 +2534,7 @@ async function downloadVOD(
};
} else {
// Single clip file
const filename = ensureUniqueFilename(makeClipFilename(clip.startPart, clip.startSec, clip.durationSec));
const filename = ensureUniqueFilename(makeClipFilename(clip.startPart, clip.startSec, clip.durationSec), item.id);
return await downloadVODPart(
item.url,
filename,
@ -2536,7 +2557,7 @@ async function downloadVOD(
1,
0,
totalDuration
));
), item.id);
return await downloadVODPart(item.url, filename, null, null, onProgress, item.id, 1, 1);
} else {
// Part-based download
@ -2557,7 +2578,7 @@ async function downloadVOD(
i + 1,
startSec,
duration
));
), item.id);
const result = await downloadVODPart(
item.url,
@ -2642,7 +2663,7 @@ async function processDownloadMergeGroup(
saveQueue(downloadQueue);
const vodItem = mg.items[i];
const tmpFilename = ensureUniqueFilename(path.join(folder, `merge_tmp_${i}_${Date.now()}.mp4`));
const tmpFilename = ensureUniqueFilename(path.join(folder, `merge_tmp_${i}_${Date.now()}.mp4`), item.id);
// Calculate progress weighting per VOD
const vodDuration = parseDuration(vodItem.duration_str);
@ -2781,7 +2802,8 @@ async function processDownloadMergeGroup(
currentPart,
totalParts
});
}
},
item.id
);
if (!splitResult.success) {
@ -2932,8 +2954,8 @@ async function processOneQueueItem(item: QueueItem): Promise<void> {
} finally {
activeDownloads.delete(item.id);
cancelledItemIds.delete(item.id);
// Release any filenames claimed during this download (prevents stale claims blocking re-downloads)
claimedFilenames.clear();
// Release only THIS item's claimed filenames (other parallel downloads keep their claims)
releaseClaimedFilenamesForItem(item.id);
}
}