feat(merge-split): add processDownloadMergeGroup() 4-phase pipeline
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
xRangerDE
parent
03f47a7240
commit
8501bd17f7
240
src/main.ts
240
src/main.ts
@ -60,6 +60,17 @@ type RetryErrorClass = 'network' | 'rate_limit' | 'auth' | 'tooling' | 'integrit
|
||||
type UpdateCheckSource = 'startup' | 'interval' | 'manual';
|
||||
type UpdateDownloadSource = 'auto' | 'manual';
|
||||
|
||||
function getMergeGroupPhaseText(phase: string): string {
|
||||
const isEnglish = config.language === 'en';
|
||||
switch (phase) {
|
||||
case 'downloading': return isEnglish ? 'Downloading VOD' : 'VOD wird heruntergeladen';
|
||||
case 'merging': return isEnglish ? 'Merging...' : 'Zusammenfugen...';
|
||||
case 'splitting': return isEnglish ? 'Splitting Part' : 'Part wird erstellt';
|
||||
case 'cleanup': return isEnglish ? 'Cleaning up...' : 'Aufraumen...';
|
||||
default: return phase;
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure directories exist
|
||||
if (!fs.existsSync(APPDATA_DIR)) {
|
||||
fs.mkdirSync(APPDATA_DIR, { recursive: true });
|
||||
@ -2973,6 +2984,235 @@ async function downloadVOD(
|
||||
}
|
||||
}
|
||||
|
||||
// ==========================================
|
||||
// MERGE GROUP DOWNLOAD PIPELINE
|
||||
// ==========================================
|
||||
async function processDownloadMergeGroup(
|
||||
item: QueueItem,
|
||||
onProgress: (progress: DownloadProgress) => void
|
||||
): Promise<DownloadResult> {
|
||||
const mg = item.mergeGroup!;
|
||||
const totalDurationSec = mg.totalDurationSec || mg.items.reduce((sum, i) => sum + parseDuration(i.duration_str), 0);
|
||||
mg.totalDurationSec = totalDurationSec;
|
||||
|
||||
// ---- PHASE 1: DOWNLOADING ----
|
||||
if (mg.mergePhase === 'downloading') {
|
||||
const streamlinkReady = await ensureStreamlinkInstalled();
|
||||
if (!streamlinkReady) {
|
||||
return { success: false, error: 'Streamlink fehlt.' };
|
||||
}
|
||||
|
||||
const streamer = mg.items[0].streamer.replace(/[^a-zA-Z0-9_-]/g, '');
|
||||
const date = new Date(mg.items[0].date);
|
||||
const dateStr = `${date.getDate().toString().padStart(2, '0')}.${(date.getMonth() + 1).toString().padStart(2, '0')}.${date.getFullYear()}`;
|
||||
const folder = path.join(config.download_path, streamer, dateStr);
|
||||
fs.mkdirSync(folder, { recursive: true });
|
||||
|
||||
// Disk space pre-check: 3x total estimated size
|
||||
const estimatedBytes = mg.items.reduce((sum, i) => {
|
||||
const dur = parseDuration(i.duration_str);
|
||||
return sum + Math.ceil(dur * 500_000); // ~500KB/s estimate
|
||||
}, 0);
|
||||
const requiredBytes = Math.max(256 * 1024 * 1024, estimatedBytes * 3);
|
||||
const diskCheck = ensureDiskSpace(folder, requiredBytes, 'Merge-Group-Download');
|
||||
if (!diskCheck.success) {
|
||||
return diskCheck;
|
||||
}
|
||||
|
||||
for (let i = 0; i < mg.items.length; i++) {
|
||||
if (currentDownloadCancelled) {
|
||||
return { success: false, error: 'Download wurde abgebrochen.' };
|
||||
}
|
||||
|
||||
// Skip already downloaded files (retry recovery)
|
||||
if (mg.downloadedFiles[i] && fs.existsSync(mg.downloadedFiles[i])) {
|
||||
appendDebugLog('merge-group-skip-existing', { index: i, file: mg.downloadedFiles[i] });
|
||||
continue;
|
||||
}
|
||||
|
||||
currentDownloadCancelled = false; // Reset stale cancel state
|
||||
mg.currentItemIndex = i;
|
||||
mg.mergePhase = 'downloading';
|
||||
saveQueue(downloadQueue);
|
||||
|
||||
const vodItem = mg.items[i];
|
||||
const tmpFilename = path.join(folder, `merge_tmp_${i}_${Date.now()}.mp4`);
|
||||
|
||||
// Calculate progress weighting per VOD
|
||||
const vodDuration = parseDuration(vodItem.duration_str);
|
||||
const vodWeight = vodDuration / totalDurationSec;
|
||||
const priorWeight = mg.items.slice(0, i).reduce((s, v) => s + parseDuration(v.duration_str), 0) / totalDurationSec;
|
||||
|
||||
const result = await downloadVODPart(
|
||||
vodItem.url,
|
||||
tmpFilename,
|
||||
null, // startTime: null = full VOD
|
||||
null, // endTime: null = full VOD
|
||||
(progress) => {
|
||||
// Weighted progress: download phase = 0-70%
|
||||
const vodProgress = progress.progress > 0 ? progress.progress : 0;
|
||||
const overallProgress = (priorWeight + vodWeight * (vodProgress / 100)) * 70;
|
||||
onProgress({
|
||||
...progress,
|
||||
id: item.id,
|
||||
progress: overallProgress,
|
||||
status: `${getMergeGroupPhaseText('downloading')} ${i + 1}/${mg.items.length} — ${progress.status}`,
|
||||
currentPart: i + 1,
|
||||
totalParts: mg.items.length
|
||||
});
|
||||
},
|
||||
item.id,
|
||||
i + 1,
|
||||
mg.items.length
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
return result;
|
||||
}
|
||||
|
||||
mg.downloadedFiles[i] = tmpFilename;
|
||||
saveQueue(downloadQueue);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- PHASE 2: MERGING ----
|
||||
mg.mergePhase = 'merging';
|
||||
saveQueue(downloadQueue);
|
||||
emitQueueUpdated();
|
||||
|
||||
// Check all downloaded files exist (retry recovery)
|
||||
for (let i = 0; i < mg.items.length; i++) {
|
||||
if (!mg.downloadedFiles[i] || !fs.existsSync(mg.downloadedFiles[i])) {
|
||||
mg.mergePhase = 'downloading';
|
||||
return { success: false, error: `Heruntergeladene Datei ${i + 1} fehlt.` };
|
||||
}
|
||||
}
|
||||
|
||||
if (!mg.mergedFile || !fs.existsSync(mg.mergedFile)) {
|
||||
const streamer = mg.items[0].streamer.replace(/[^a-zA-Z0-9_-]/g, '');
|
||||
const date = new Date(mg.items[0].date);
|
||||
const dateStr = `${date.getDate().toString().padStart(2, '0')}.${(date.getMonth() + 1).toString().padStart(2, '0')}.${date.getFullYear()}`;
|
||||
const folder = path.join(config.download_path, streamer, dateStr);
|
||||
const mergedFilePath = path.join(folder, `merged_${Date.now()}.mp4`);
|
||||
|
||||
// Get files in correct order (explicit sort by index — do NOT rely on Object.values ordering)
|
||||
const sortedFiles = Object.keys(mg.downloadedFiles)
|
||||
.sort((a, b) => Number(a) - Number(b))
|
||||
.map(k => mg.downloadedFiles[Number(k)]);
|
||||
|
||||
const mergeSuccess = await mergeVideos(
|
||||
sortedFiles,
|
||||
mergedFilePath,
|
||||
(percent) => {
|
||||
const overallProgress = 70 + (percent / 100) * 20; // merge = 70-90%
|
||||
onProgress({
|
||||
id: item.id,
|
||||
progress: overallProgress,
|
||||
speed: '',
|
||||
eta: '',
|
||||
status: getMergeGroupPhaseText('merging'),
|
||||
currentPart: 0,
|
||||
totalParts: 0
|
||||
});
|
||||
},
|
||||
totalDurationSec
|
||||
);
|
||||
|
||||
if (!mergeSuccess) {
|
||||
return { success: false, error: 'FFmpeg Merge fehlgeschlagen.' };
|
||||
}
|
||||
|
||||
mg.mergedFile = mergedFilePath;
|
||||
saveQueue(downloadQueue);
|
||||
}
|
||||
|
||||
// ---- PHASE 3: SPLITTING ----
|
||||
mg.mergePhase = 'splitting';
|
||||
saveQueue(downloadQueue);
|
||||
emitQueueUpdated();
|
||||
|
||||
if (currentDownloadCancelled) {
|
||||
return { success: false, error: 'Download wurde abgebrochen.' };
|
||||
}
|
||||
|
||||
const partDuration = config.part_minutes * 60;
|
||||
const streamer = mg.items[0].streamer.replace(/[^a-zA-Z0-9_-]/g, '');
|
||||
const date = new Date(mg.items[0].date);
|
||||
const dateStr = `${date.getDate().toString().padStart(2, '0')}.${(date.getMonth() + 1).toString().padStart(2, '0')}.${date.getFullYear()}`;
|
||||
const folder = path.join(config.download_path, streamer, dateStr);
|
||||
const vodId = parseVodId(mg.items[0].url) || 'merged';
|
||||
|
||||
const splitResult = await splitMergedFile(
|
||||
mg.mergedFile!,
|
||||
folder,
|
||||
partDuration,
|
||||
totalDurationSec,
|
||||
(partNum: number) => {
|
||||
const startSec = (partNum - 1) * partDuration;
|
||||
const thisDuration = Math.min(partDuration, totalDurationSec - startSec);
|
||||
return renderClipFilenameTemplate({
|
||||
template: normalizeFilenameTemplate(config.filename_template_parts, DEFAULT_FILENAME_TEMPLATE_PARTS),
|
||||
title: mg.items[0].title,
|
||||
vodId,
|
||||
channel: mg.items[0].streamer,
|
||||
date,
|
||||
part: partNum,
|
||||
partPadded: partNum.toString().padStart(2, '0'),
|
||||
trimStartSec: startSec,
|
||||
trimEndSec: startSec + thisDuration,
|
||||
trimLengthSec: thisDuration,
|
||||
fullLengthSec: totalDurationSec
|
||||
});
|
||||
},
|
||||
(currentPart, totalParts) => {
|
||||
const overallProgress = 90 + (currentPart / totalParts) * 10; // split = 90-100%
|
||||
onProgress({
|
||||
id: item.id,
|
||||
progress: overallProgress,
|
||||
speed: '',
|
||||
eta: '',
|
||||
status: `${getMergeGroupPhaseText('splitting')} ${currentPart}/${totalParts}...`,
|
||||
currentPart,
|
||||
totalParts
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
if (!splitResult.success) {
|
||||
return { success: false, error: 'FFmpeg Split fehlgeschlagen.' };
|
||||
}
|
||||
|
||||
mg.splitFiles = splitResult.files;
|
||||
|
||||
// ---- PHASE 4: CLEANUP ----
|
||||
mg.mergePhase = 'cleanup';
|
||||
saveQueue(downloadQueue);
|
||||
|
||||
// Delete individual downloads
|
||||
for (const key of Object.keys(mg.downloadedFiles)) {
|
||||
const filePath = mg.downloadedFiles[Number(key)];
|
||||
try {
|
||||
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
||||
} catch { }
|
||||
}
|
||||
|
||||
// Delete merged file
|
||||
if (mg.mergedFile) {
|
||||
try {
|
||||
if (fs.existsSync(mg.mergedFile)) fs.unlinkSync(mg.mergedFile);
|
||||
} catch { }
|
||||
}
|
||||
|
||||
mg.mergePhase = 'done';
|
||||
appendDebugLog('merge-group-complete', {
|
||||
itemId: item.id,
|
||||
parts: splitResult.files.length,
|
||||
totalDurationSec
|
||||
});
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isDownloading || !downloadQueue.some((item) => item.status === 'pending')) return;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user