harden: atomic fsync writes + per-item filename claims
Two server-side correctness fixes for parallel downloads and crash recovery. 1. Atomic file writes survive power loss / crash mid-write. saveConfig and writeQueueToDisk used writeFileSync + renameSync. Node's writeFileSync does NOT fsync — a power loss between write and rename can leave the renamed file empty or truncated, and the next launch silently falls back to defaults / empty queue. New writeFileAtomicSync helper: openSync + writeSync + fsyncSync + closeSync + renameSync (with the existing Windows copy fallback). fsync failure is non-fatal (some FS reject it) but file ordering is preserved. 2. Per-item claimed filenames fix the parallel-download race. With max 2 parallel downloads, processOneQueueItem.finally was calling claimedFilenames.clear() — wiping every parallel item's claims when any one finished. In the window between an active item claiming a filename and streamlink actually writing the first bytes, a third item could compute the same filename and both downloads would race the same path. New Map<itemId, Set<filename>> tracks claims per active download. ensureUniqueFilename(path, itemId) registers per-item; releaseClaimedFilenamesForItem(itemId) removes only that item's claims. splitMergedFile gained an itemId parameter for the same reason. The dead releaseClaimedFilename(path) function was removed. Build: tsc clean. Tests: smoke + smoke-template-guide + smoke-full + merge-split + update-version-logic all pass. No new ESLint warnings. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
54197af863
commit
8d0cb4cefd
84
src/main.ts
84
src/main.ts
@ -268,17 +268,33 @@ function loadConfig(): Config {
|
|||||||
return normalizeConfigTemplates(defaultConfig);
|
return normalizeConfigTemplates(defaultConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
function saveConfig(config: Config): void {
|
function writeFileAtomicSync(targetPath: string, payload: string | Buffer): void {
|
||||||
const tmpPath = CONFIG_FILE + '.tmp';
|
const buffer = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, 'utf-8');
|
||||||
|
const tmpPath = targetPath + '.tmp';
|
||||||
|
|
||||||
|
let fd: number | null = null;
|
||||||
try {
|
try {
|
||||||
fs.writeFileSync(tmpPath, JSON.stringify(config, null, 2));
|
fd = fs.openSync(tmpPath, 'w');
|
||||||
try {
|
fs.writeSync(fd, buffer, 0, buffer.length, 0);
|
||||||
fs.renameSync(tmpPath, CONFIG_FILE);
|
try { fs.fsyncSync(fd); } catch { /* fsync may fail on some FS; rename is still safer than nothing */ }
|
||||||
} catch {
|
} finally {
|
||||||
// On Windows, rename can fail if target exists in some edge cases
|
if (fd !== null) {
|
||||||
fs.copyFileSync(tmpPath, CONFIG_FILE);
|
try { fs.closeSync(fd); } catch { }
|
||||||
try { fs.unlinkSync(tmpPath); } catch { }
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.renameSync(tmpPath, targetPath);
|
||||||
|
} catch {
|
||||||
|
// On Windows, rename can fail if target exists or is locked. Fall back to copy.
|
||||||
|
fs.copyFileSync(tmpPath, targetPath);
|
||||||
|
try { fs.unlinkSync(tmpPath); } catch { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveConfig(config: Config): void {
|
||||||
|
try {
|
||||||
|
writeFileAtomicSync(CONFIG_FILE, JSON.stringify(config, null, 2));
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error('Error saving config:', e);
|
console.error('Error saving config:', e);
|
||||||
}
|
}
|
||||||
@ -329,16 +345,8 @@ function writeQueueToDisk(queue: QueueItem[]): void {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const tmpPath = QUEUE_FILE + '.tmp';
|
|
||||||
try {
|
try {
|
||||||
fs.writeFileSync(tmpPath, JSON.stringify(queue, null, 2));
|
writeFileAtomicSync(QUEUE_FILE, JSON.stringify(queue, null, 2));
|
||||||
try {
|
|
||||||
fs.renameSync(tmpPath, QUEUE_FILE);
|
|
||||||
} catch {
|
|
||||||
// On Windows, rename can fail if target exists in some edge cases
|
|
||||||
fs.copyFileSync(tmpPath, QUEUE_FILE);
|
|
||||||
try { fs.unlinkSync(tmpPath); } catch { }
|
|
||||||
}
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error('Error saving queue:', e);
|
console.error('Error saving queue:', e);
|
||||||
}
|
}
|
||||||
@ -704,8 +712,9 @@ function formatDurationDashed(seconds: number): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const claimedFilenames = new Set<string>();
|
const claimedFilenames = new Set<string>();
|
||||||
|
const itemClaimedFilenames = new Map<string, Set<string>>();
|
||||||
|
|
||||||
function ensureUniqueFilename(filePath: string): string {
|
function ensureUniqueFilename(filePath: string, itemId: string | null = null): string {
|
||||||
const dir = path.dirname(filePath);
|
const dir = path.dirname(filePath);
|
||||||
const ext = path.extname(filePath);
|
const ext = path.extname(filePath);
|
||||||
const base = path.basename(filePath, ext);
|
const base = path.basename(filePath, ext);
|
||||||
@ -716,11 +725,22 @@ function ensureUniqueFilename(filePath: string): string {
|
|||||||
candidate = path.join(dir, `${base}_${counter}${ext}`);
|
candidate = path.join(dir, `${base}_${counter}${ext}`);
|
||||||
}
|
}
|
||||||
claimedFilenames.add(candidate);
|
claimedFilenames.add(candidate);
|
||||||
|
if (itemId) {
|
||||||
|
let perItem = itemClaimedFilenames.get(itemId);
|
||||||
|
if (!perItem) {
|
||||||
|
perItem = new Set();
|
||||||
|
itemClaimedFilenames.set(itemId, perItem);
|
||||||
|
}
|
||||||
|
perItem.add(candidate);
|
||||||
|
}
|
||||||
return candidate;
|
return candidate;
|
||||||
}
|
}
|
||||||
|
|
||||||
function releaseClaimedFilename(filePath: string): void {
|
function releaseClaimedFilenamesForItem(itemId: string): void {
|
||||||
claimedFilenames.delete(filePath);
|
const perItem = itemClaimedFilenames.get(itemId);
|
||||||
|
if (!perItem) return;
|
||||||
|
for (const f of perItem) claimedFilenames.delete(f);
|
||||||
|
itemClaimedFilenames.delete(itemId);
|
||||||
}
|
}
|
||||||
|
|
||||||
function sanitizeFilenamePart(input: string, fallback = 'unnamed'): string {
|
function sanitizeFilenamePart(input: string, fallback = 'unnamed'): string {
|
||||||
@ -2124,7 +2144,8 @@ async function splitMergedFile(
|
|||||||
partDurationSec: number,
|
partDurationSec: number,
|
||||||
totalDurationSec: number,
|
totalDurationSec: number,
|
||||||
filenameGenerator: (partNum: number) => string,
|
filenameGenerator: (partNum: number) => string,
|
||||||
onProgress: (currentPart: number, totalParts: number) => void
|
onProgress: (currentPart: number, totalParts: number) => void,
|
||||||
|
itemId: string | null = null
|
||||||
): Promise<{ success: boolean; files: string[] }> {
|
): Promise<{ success: boolean; files: string[] }> {
|
||||||
const ffmpegReady = await ensureFfmpegInstalled();
|
const ffmpegReady = await ensureFfmpegInstalled();
|
||||||
if (!ffmpegReady) {
|
if (!ffmpegReady) {
|
||||||
@ -2143,7 +2164,7 @@ async function splitMergedFile(
|
|||||||
|
|
||||||
const startSec = i * partDurationSec;
|
const startSec = i * partDurationSec;
|
||||||
const thisDuration = Math.min(partDurationSec, totalDurationSec - startSec);
|
const thisDuration = Math.min(partDurationSec, totalDurationSec - startSec);
|
||||||
const outputFile = ensureUniqueFilename(path.join(outputFolder, filenameGenerator(i + 1)));
|
const outputFile = ensureUniqueFilename(path.join(outputFolder, filenameGenerator(i + 1)), itemId);
|
||||||
|
|
||||||
onProgress(i + 1, numParts);
|
onProgress(i + 1, numParts);
|
||||||
|
|
||||||
@ -2490,7 +2511,7 @@ async function downloadVOD(
|
|||||||
const remainingDuration = clip.durationSec - (i * partDuration);
|
const remainingDuration = clip.durationSec - (i * partDuration);
|
||||||
const thisDuration = Math.min(partDuration, remainingDuration);
|
const thisDuration = Math.min(partDuration, remainingDuration);
|
||||||
|
|
||||||
const partFilename = ensureUniqueFilename(makeClipFilename(partNum, startOffset, thisDuration));
|
const partFilename = ensureUniqueFilename(makeClipFilename(partNum, startOffset, thisDuration), item.id);
|
||||||
|
|
||||||
const result = await downloadVODPart(
|
const result = await downloadVODPart(
|
||||||
item.url,
|
item.url,
|
||||||
@ -2513,7 +2534,7 @@ async function downloadVOD(
|
|||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
// Single clip file
|
// Single clip file
|
||||||
const filename = ensureUniqueFilename(makeClipFilename(clip.startPart, clip.startSec, clip.durationSec));
|
const filename = ensureUniqueFilename(makeClipFilename(clip.startPart, clip.startSec, clip.durationSec), item.id);
|
||||||
return await downloadVODPart(
|
return await downloadVODPart(
|
||||||
item.url,
|
item.url,
|
||||||
filename,
|
filename,
|
||||||
@ -2536,7 +2557,7 @@ async function downloadVOD(
|
|||||||
1,
|
1,
|
||||||
0,
|
0,
|
||||||
totalDuration
|
totalDuration
|
||||||
));
|
), item.id);
|
||||||
return await downloadVODPart(item.url, filename, null, null, onProgress, item.id, 1, 1);
|
return await downloadVODPart(item.url, filename, null, null, onProgress, item.id, 1, 1);
|
||||||
} else {
|
} else {
|
||||||
// Part-based download
|
// Part-based download
|
||||||
@ -2557,7 +2578,7 @@ async function downloadVOD(
|
|||||||
i + 1,
|
i + 1,
|
||||||
startSec,
|
startSec,
|
||||||
duration
|
duration
|
||||||
));
|
), item.id);
|
||||||
|
|
||||||
const result = await downloadVODPart(
|
const result = await downloadVODPart(
|
||||||
item.url,
|
item.url,
|
||||||
@ -2642,7 +2663,7 @@ async function processDownloadMergeGroup(
|
|||||||
saveQueue(downloadQueue);
|
saveQueue(downloadQueue);
|
||||||
|
|
||||||
const vodItem = mg.items[i];
|
const vodItem = mg.items[i];
|
||||||
const tmpFilename = ensureUniqueFilename(path.join(folder, `merge_tmp_${i}_${Date.now()}.mp4`));
|
const tmpFilename = ensureUniqueFilename(path.join(folder, `merge_tmp_${i}_${Date.now()}.mp4`), item.id);
|
||||||
|
|
||||||
// Calculate progress weighting per VOD
|
// Calculate progress weighting per VOD
|
||||||
const vodDuration = parseDuration(vodItem.duration_str);
|
const vodDuration = parseDuration(vodItem.duration_str);
|
||||||
@ -2781,7 +2802,8 @@ async function processDownloadMergeGroup(
|
|||||||
currentPart,
|
currentPart,
|
||||||
totalParts
|
totalParts
|
||||||
});
|
});
|
||||||
}
|
},
|
||||||
|
item.id
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!splitResult.success) {
|
if (!splitResult.success) {
|
||||||
@ -2932,8 +2954,8 @@ async function processOneQueueItem(item: QueueItem): Promise<void> {
|
|||||||
} finally {
|
} finally {
|
||||||
activeDownloads.delete(item.id);
|
activeDownloads.delete(item.id);
|
||||||
cancelledItemIds.delete(item.id);
|
cancelledItemIds.delete(item.id);
|
||||||
// Release any filenames claimed during this download (prevents stale claims blocking re-downloads)
|
// Release only THIS item's claimed filenames (other parallel downloads keep their claims)
|
||||||
claimedFilenames.clear();
|
releaseClaimedFilenamesForItem(item.id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user