feat(merge-split): add processDownloadMergeGroup() 4-phase pipeline
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
03f47a7240
commit
8501bd17f7
240
src/main.ts
240
src/main.ts
@ -60,6 +60,17 @@ type RetryErrorClass = 'network' | 'rate_limit' | 'auth' | 'tooling' | 'integrit
|
|||||||
type UpdateCheckSource = 'startup' | 'interval' | 'manual';
|
type UpdateCheckSource = 'startup' | 'interval' | 'manual';
|
||||||
type UpdateDownloadSource = 'auto' | 'manual';
|
type UpdateDownloadSource = 'auto' | 'manual';
|
||||||
|
|
||||||
|
function getMergeGroupPhaseText(phase: string): string {
|
||||||
|
const isEnglish = config.language === 'en';
|
||||||
|
switch (phase) {
|
||||||
|
case 'downloading': return isEnglish ? 'Downloading VOD' : 'VOD wird heruntergeladen';
|
||||||
|
case 'merging': return isEnglish ? 'Merging...' : 'Zusammenfugen...';
|
||||||
|
case 'splitting': return isEnglish ? 'Splitting Part' : 'Part wird erstellt';
|
||||||
|
case 'cleanup': return isEnglish ? 'Cleaning up...' : 'Aufraumen...';
|
||||||
|
default: return phase;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Ensure directories exist
|
// Ensure directories exist
|
||||||
if (!fs.existsSync(APPDATA_DIR)) {
|
if (!fs.existsSync(APPDATA_DIR)) {
|
||||||
fs.mkdirSync(APPDATA_DIR, { recursive: true });
|
fs.mkdirSync(APPDATA_DIR, { recursive: true });
|
||||||
@ -2973,6 +2984,235 @@ async function downloadVOD(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==========================================
|
||||||
|
// MERGE GROUP DOWNLOAD PIPELINE
|
||||||
|
// ==========================================
|
||||||
|
async function processDownloadMergeGroup(
|
||||||
|
item: QueueItem,
|
||||||
|
onProgress: (progress: DownloadProgress) => void
|
||||||
|
): Promise<DownloadResult> {
|
||||||
|
const mg = item.mergeGroup!;
|
||||||
|
const totalDurationSec = mg.totalDurationSec || mg.items.reduce((sum, i) => sum + parseDuration(i.duration_str), 0);
|
||||||
|
mg.totalDurationSec = totalDurationSec;
|
||||||
|
|
||||||
|
// ---- PHASE 1: DOWNLOADING ----
|
||||||
|
if (mg.mergePhase === 'downloading') {
|
||||||
|
const streamlinkReady = await ensureStreamlinkInstalled();
|
||||||
|
if (!streamlinkReady) {
|
||||||
|
return { success: false, error: 'Streamlink fehlt.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const streamer = mg.items[0].streamer.replace(/[^a-zA-Z0-9_-]/g, '');
|
||||||
|
const date = new Date(mg.items[0].date);
|
||||||
|
const dateStr = `${date.getDate().toString().padStart(2, '0')}.${(date.getMonth() + 1).toString().padStart(2, '0')}.${date.getFullYear()}`;
|
||||||
|
const folder = path.join(config.download_path, streamer, dateStr);
|
||||||
|
fs.mkdirSync(folder, { recursive: true });
|
||||||
|
|
||||||
|
// Disk space pre-check: 3x total estimated size
|
||||||
|
const estimatedBytes = mg.items.reduce((sum, i) => {
|
||||||
|
const dur = parseDuration(i.duration_str);
|
||||||
|
return sum + Math.ceil(dur * 500_000); // ~500KB/s estimate
|
||||||
|
}, 0);
|
||||||
|
const requiredBytes = Math.max(256 * 1024 * 1024, estimatedBytes * 3);
|
||||||
|
const diskCheck = ensureDiskSpace(folder, requiredBytes, 'Merge-Group-Download');
|
||||||
|
if (!diskCheck.success) {
|
||||||
|
return diskCheck;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < mg.items.length; i++) {
|
||||||
|
if (currentDownloadCancelled) {
|
||||||
|
return { success: false, error: 'Download wurde abgebrochen.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip already downloaded files (retry recovery)
|
||||||
|
if (mg.downloadedFiles[i] && fs.existsSync(mg.downloadedFiles[i])) {
|
||||||
|
appendDebugLog('merge-group-skip-existing', { index: i, file: mg.downloadedFiles[i] });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
currentDownloadCancelled = false; // Reset stale cancel state
|
||||||
|
mg.currentItemIndex = i;
|
||||||
|
mg.mergePhase = 'downloading';
|
||||||
|
saveQueue(downloadQueue);
|
||||||
|
|
||||||
|
const vodItem = mg.items[i];
|
||||||
|
const tmpFilename = path.join(folder, `merge_tmp_${i}_${Date.now()}.mp4`);
|
||||||
|
|
||||||
|
// Calculate progress weighting per VOD
|
||||||
|
const vodDuration = parseDuration(vodItem.duration_str);
|
||||||
|
const vodWeight = vodDuration / totalDurationSec;
|
||||||
|
const priorWeight = mg.items.slice(0, i).reduce((s, v) => s + parseDuration(v.duration_str), 0) / totalDurationSec;
|
||||||
|
|
||||||
|
const result = await downloadVODPart(
|
||||||
|
vodItem.url,
|
||||||
|
tmpFilename,
|
||||||
|
null, // startTime: null = full VOD
|
||||||
|
null, // endTime: null = full VOD
|
||||||
|
(progress) => {
|
||||||
|
// Weighted progress: download phase = 0-70%
|
||||||
|
const vodProgress = progress.progress > 0 ? progress.progress : 0;
|
||||||
|
const overallProgress = (priorWeight + vodWeight * (vodProgress / 100)) * 70;
|
||||||
|
onProgress({
|
||||||
|
...progress,
|
||||||
|
id: item.id,
|
||||||
|
progress: overallProgress,
|
||||||
|
status: `${getMergeGroupPhaseText('downloading')} ${i + 1}/${mg.items.length} — ${progress.status}`,
|
||||||
|
currentPart: i + 1,
|
||||||
|
totalParts: mg.items.length
|
||||||
|
});
|
||||||
|
},
|
||||||
|
item.id,
|
||||||
|
i + 1,
|
||||||
|
mg.items.length
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!result.success) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
mg.downloadedFiles[i] = tmpFilename;
|
||||||
|
saveQueue(downloadQueue);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- PHASE 2: MERGING ----
|
||||||
|
mg.mergePhase = 'merging';
|
||||||
|
saveQueue(downloadQueue);
|
||||||
|
emitQueueUpdated();
|
||||||
|
|
||||||
|
// Check all downloaded files exist (retry recovery)
|
||||||
|
for (let i = 0; i < mg.items.length; i++) {
|
||||||
|
if (!mg.downloadedFiles[i] || !fs.existsSync(mg.downloadedFiles[i])) {
|
||||||
|
mg.mergePhase = 'downloading';
|
||||||
|
return { success: false, error: `Heruntergeladene Datei ${i + 1} fehlt.` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!mg.mergedFile || !fs.existsSync(mg.mergedFile)) {
|
||||||
|
const streamer = mg.items[0].streamer.replace(/[^a-zA-Z0-9_-]/g, '');
|
||||||
|
const date = new Date(mg.items[0].date);
|
||||||
|
const dateStr = `${date.getDate().toString().padStart(2, '0')}.${(date.getMonth() + 1).toString().padStart(2, '0')}.${date.getFullYear()}`;
|
||||||
|
const folder = path.join(config.download_path, streamer, dateStr);
|
||||||
|
const mergedFilePath = path.join(folder, `merged_${Date.now()}.mp4`);
|
||||||
|
|
||||||
|
// Get files in correct order (explicit sort by index — do NOT rely on Object.values ordering)
|
||||||
|
const sortedFiles = Object.keys(mg.downloadedFiles)
|
||||||
|
.sort((a, b) => Number(a) - Number(b))
|
||||||
|
.map(k => mg.downloadedFiles[Number(k)]);
|
||||||
|
|
||||||
|
const mergeSuccess = await mergeVideos(
|
||||||
|
sortedFiles,
|
||||||
|
mergedFilePath,
|
||||||
|
(percent) => {
|
||||||
|
const overallProgress = 70 + (percent / 100) * 20; // merge = 70-90%
|
||||||
|
onProgress({
|
||||||
|
id: item.id,
|
||||||
|
progress: overallProgress,
|
||||||
|
speed: '',
|
||||||
|
eta: '',
|
||||||
|
status: getMergeGroupPhaseText('merging'),
|
||||||
|
currentPart: 0,
|
||||||
|
totalParts: 0
|
||||||
|
});
|
||||||
|
},
|
||||||
|
totalDurationSec
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!mergeSuccess) {
|
||||||
|
return { success: false, error: 'FFmpeg Merge fehlgeschlagen.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
mg.mergedFile = mergedFilePath;
|
||||||
|
saveQueue(downloadQueue);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- PHASE 3: SPLITTING ----
|
||||||
|
mg.mergePhase = 'splitting';
|
||||||
|
saveQueue(downloadQueue);
|
||||||
|
emitQueueUpdated();
|
||||||
|
|
||||||
|
if (currentDownloadCancelled) {
|
||||||
|
return { success: false, error: 'Download wurde abgebrochen.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const partDuration = config.part_minutes * 60;
|
||||||
|
const streamer = mg.items[0].streamer.replace(/[^a-zA-Z0-9_-]/g, '');
|
||||||
|
const date = new Date(mg.items[0].date);
|
||||||
|
const dateStr = `${date.getDate().toString().padStart(2, '0')}.${(date.getMonth() + 1).toString().padStart(2, '0')}.${date.getFullYear()}`;
|
||||||
|
const folder = path.join(config.download_path, streamer, dateStr);
|
||||||
|
const vodId = parseVodId(mg.items[0].url) || 'merged';
|
||||||
|
|
||||||
|
const splitResult = await splitMergedFile(
|
||||||
|
mg.mergedFile!,
|
||||||
|
folder,
|
||||||
|
partDuration,
|
||||||
|
totalDurationSec,
|
||||||
|
(partNum: number) => {
|
||||||
|
const startSec = (partNum - 1) * partDuration;
|
||||||
|
const thisDuration = Math.min(partDuration, totalDurationSec - startSec);
|
||||||
|
return renderClipFilenameTemplate({
|
||||||
|
template: normalizeFilenameTemplate(config.filename_template_parts, DEFAULT_FILENAME_TEMPLATE_PARTS),
|
||||||
|
title: mg.items[0].title,
|
||||||
|
vodId,
|
||||||
|
channel: mg.items[0].streamer,
|
||||||
|
date,
|
||||||
|
part: partNum,
|
||||||
|
partPadded: partNum.toString().padStart(2, '0'),
|
||||||
|
trimStartSec: startSec,
|
||||||
|
trimEndSec: startSec + thisDuration,
|
||||||
|
trimLengthSec: thisDuration,
|
||||||
|
fullLengthSec: totalDurationSec
|
||||||
|
});
|
||||||
|
},
|
||||||
|
(currentPart, totalParts) => {
|
||||||
|
const overallProgress = 90 + (currentPart / totalParts) * 10; // split = 90-100%
|
||||||
|
onProgress({
|
||||||
|
id: item.id,
|
||||||
|
progress: overallProgress,
|
||||||
|
speed: '',
|
||||||
|
eta: '',
|
||||||
|
status: `${getMergeGroupPhaseText('splitting')} ${currentPart}/${totalParts}...`,
|
||||||
|
currentPart,
|
||||||
|
totalParts
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!splitResult.success) {
|
||||||
|
return { success: false, error: 'FFmpeg Split fehlgeschlagen.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
mg.splitFiles = splitResult.files;
|
||||||
|
|
||||||
|
// ---- PHASE 4: CLEANUP ----
|
||||||
|
mg.mergePhase = 'cleanup';
|
||||||
|
saveQueue(downloadQueue);
|
||||||
|
|
||||||
|
// Delete individual downloads
|
||||||
|
for (const key of Object.keys(mg.downloadedFiles)) {
|
||||||
|
const filePath = mg.downloadedFiles[Number(key)];
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
||||||
|
} catch { }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete merged file
|
||||||
|
if (mg.mergedFile) {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(mg.mergedFile)) fs.unlinkSync(mg.mergedFile);
|
||||||
|
} catch { }
|
||||||
|
}
|
||||||
|
|
||||||
|
mg.mergePhase = 'done';
|
||||||
|
appendDebugLog('merge-group-complete', {
|
||||||
|
itemId: item.id,
|
||||||
|
parts: splitResult.files.length,
|
||||||
|
totalDurationSec
|
||||||
|
});
|
||||||
|
|
||||||
|
return { success: true };
|
||||||
|
}
|
||||||
|
|
||||||
async function processQueue(): Promise<void> {
|
async function processQueue(): Promise<void> {
|
||||||
if (isDownloading || !downloadQueue.some((item) => item.status === 'pending')) return;
|
if (isDownloading || !downloadQueue.some((item) => item.status === 'pending')) return;
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user