Parallel archive extraction within packages

maxParallelExtract now controls how many archives extract simultaneously
within a single package (e.g. 4 episodes at once). Packages still
extract sequentially (one package at a time) to focus I/O. Progress
handler updated to track multiple active archives independently.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Sucukdeluxe 2026-03-03 21:43:34 +01:00
parent d9fe98231f
commit 5dabee332e
3 changed files with 107 additions and 50 deletions

View File

@ -1,6 +1,6 @@
{ {
"name": "real-debrid-downloader", "name": "real-debrid-downloader",
"version": "1.5.70", "version": "1.5.71",
"description": "Real-Debrid Downloader Desktop (Electron + React + TypeScript)", "description": "Real-Debrid Downloader Desktop (Electron + React + TypeScript)",
"main": "build/main/main/main.js", "main": "build/main/main/main.js",
"author": "Sucukdeluxe", "author": "Sucukdeluxe",

View File

@ -5516,32 +5516,8 @@ export class DownloadManager extends EventEmitter {
const resolveArchiveItems = (archiveName: string): DownloadItem[] => const resolveArchiveItems = (archiveName: string): DownloadItem[] =>
resolveArchiveItemsFromList(archiveName, completedItems); resolveArchiveItemsFromList(archiveName, completedItems);
// Only update items of the currently extracting archive, not all items
let currentArchiveItems: DownloadItem[] = [];
const updateExtractingStatus = (text: string): void => {
const normalized = String(text || "");
if (lastExtractStatusText === normalized) {
return;
}
lastExtractStatusText = normalized;
const updatedAt = nowMs();
for (const entry of currentArchiveItems) {
if (isExtractedLabel(entry.fullStatus)) {
continue;
}
if (entry.fullStatus === normalized) {
continue;
}
entry.fullStatus = normalized;
entry.updatedAt = updatedAt;
}
};
let lastExtractStatusText = "";
let lastExtractEmitAt = 0; let lastExtractEmitAt = 0;
let lastExtractArchiveName = ""; const emitExtractStatus = (_text: string, force = false): void => {
const emitExtractStatus = (text: string, force = false): void => {
updateExtractingStatus(text);
const now = nowMs(); const now = nowMs();
if (!force && now - lastExtractEmitAt < EXTRACT_PROGRESS_EMIT_INTERVAL_MS) { if (!force && now - lastExtractEmitAt < EXTRACT_PROGRESS_EMIT_INTERVAL_MS) {
return; return;
@ -5586,6 +5562,9 @@ export class DownloadManager extends EventEmitter {
} }
}, extractTimeoutMs); }, extractTimeoutMs);
try { try {
// Track multiple active archives for parallel extraction
const activeArchiveItemsMap = new Map<string, DownloadItem[]>();
const result = await extractPackageArchives({ const result = await extractPackageArchives({
packageDir: pkg.outputDir, packageDir: pkg.outputDir,
targetDir: pkg.extractDir, targetDir: pkg.extractDir,
@ -5596,33 +5575,69 @@ export class DownloadManager extends EventEmitter {
passwordList: this.settings.archivePasswordList, passwordList: this.settings.archivePasswordList,
signal: extractAbortController.signal, signal: extractAbortController.signal,
packageId, packageId,
maxParallel: this.settings.maxParallelExtract || 2,
onProgress: (progress) => { onProgress: (progress) => {
// When a new archive starts, mark the previous archive's items as done if (progress.phase === "done") {
if (progress.archiveName && progress.archiveName !== lastExtractArchiveName) { // Mark all remaining active archives as done
if (lastExtractArchiveName && currentArchiveItems.length > 0) { for (const [, items] of activeArchiveItemsMap) {
const doneAt = nowMs(); const doneAt = nowMs();
for (const entry of currentArchiveItems) { for (const entry of items) {
if (!isExtractedLabel(entry.fullStatus)) { if (!isExtractedLabel(entry.fullStatus)) {
entry.fullStatus = "Entpackt - Done"; entry.fullStatus = "Entpackt - Done";
entry.updatedAt = doneAt; entry.updatedAt = doneAt;
} }
} }
} }
lastExtractArchiveName = progress.archiveName; activeArchiveItemsMap.clear();
currentArchiveItems = resolveArchiveItems(progress.archiveName); emitExtractStatus("Entpacken 100%", true);
return;
} }
const label = progress.phase === "done"
? "Entpacken 100%" if (progress.archiveName) {
: (() => { // Resolve items for this archive if not yet tracked
if (!activeArchiveItemsMap.has(progress.archiveName)) {
activeArchiveItemsMap.set(progress.archiveName, resolveArchiveItems(progress.archiveName));
}
const archiveItems = activeArchiveItemsMap.get(progress.archiveName)!;
// If archive is at 100%, mark its items as done and remove from active
if (Number(progress.archivePercent ?? 0) >= 100) {
const doneAt = nowMs();
for (const entry of archiveItems) {
if (!isExtractedLabel(entry.fullStatus)) {
entry.fullStatus = "Entpackt - Done";
entry.updatedAt = doneAt;
}
}
activeArchiveItemsMap.delete(progress.archiveName);
} else {
// Update this archive's items with current progress
const archive = progress.archiveName ? ` · ${progress.archiveName}` : ""; const archive = progress.archiveName ? ` · ${progress.archiveName}` : "";
const elapsed = progress.elapsedMs && progress.elapsedMs >= 1000 const elapsed = progress.elapsedMs && progress.elapsedMs >= 1000
? ` · ${Math.floor(progress.elapsedMs / 1000)}s` ? ` · ${Math.floor(progress.elapsedMs / 1000)}s`
: ""; : "";
const activeArchive = Number(progress.archivePercent ?? 0) > 0 ? 1 : 0; const activeArchive = Number(progress.archivePercent ?? 0) > 0 ? 1 : 0;
const currentDisplay = Math.max(0, Math.min(progress.total, progress.current + activeArchive)); const currentDisplay = Math.max(0, Math.min(progress.total, progress.current + activeArchive));
return `Entpacken ${progress.percent}% (${currentDisplay}/${progress.total})${archive}${elapsed}`; const label = `Entpacken ${progress.percent}% (${currentDisplay}/${progress.total})${archive}${elapsed}`;
})(); const updatedAt = nowMs();
emitExtractStatus(label); for (const entry of archiveItems) {
if (!isExtractedLabel(entry.fullStatus) && entry.fullStatus !== label) {
entry.fullStatus = label;
entry.updatedAt = updatedAt;
}
}
}
}
// Emit overall status (throttled)
const archive = progress.archiveName ? ` · ${progress.archiveName}` : "";
const elapsed = progress.elapsedMs && progress.elapsedMs >= 1000
? ` · ${Math.floor(progress.elapsedMs / 1000)}s`
: "";
const activeArchive = Number(progress.archivePercent ?? 0) > 0 ? 1 : 0;
const currentDisplay = Math.max(0, Math.min(progress.total, progress.current + activeArchive));
const overallLabel = `Entpacken ${progress.percent}% (${currentDisplay}/${progress.total})${archive}${elapsed}`;
emitExtractStatus(overallLabel);
} }
}); });
logger.info(`Post-Processing Entpacken Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}, lastError=${result.lastError || ""}`); logger.info(`Post-Processing Entpacken Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}, lastError=${result.lastError || ""}`);

View File

@ -87,6 +87,7 @@ export interface ExtractOptions {
skipPostCleanup?: boolean; skipPostCleanup?: boolean;
packageId?: string; packageId?: string;
hybridMode?: boolean; hybridMode?: boolean;
maxParallel?: number;
} }
export interface ExtractProgressUpdate { export interface ExtractProgressUpdate {
@ -1905,8 +1906,11 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{
emitProgress(extracted, "", "extracting"); emitProgress(extracted, "", "extracting");
for (const archivePath of pendingCandidates) { const maxParallel = Math.max(1, options.maxParallel || 1);
if (options.signal?.aborted) { let noExtractorEncountered = false;
const extractSingleArchive = async (archivePath: string): Promise<void> => {
if (options.signal?.aborted || noExtractorEncountered) {
throw new Error("aborted:extract"); throw new Error("aborted:extract");
} }
const archiveName = path.basename(archivePath); const archiveName = path.basename(archivePath);
@ -1930,7 +1934,8 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{
const sig = await detectArchiveSignature(archivePath); const sig = await detectArchiveSignature(archivePath);
if (!sig) { if (!sig) {
logger.info(`Generische Split-Datei übersprungen (keine Archiv-Signatur): ${archiveName}`); logger.info(`Generische Split-Datei übersprungen (keine Archiv-Signatur): ${archiveName}`);
continue; clearInterval(pulseTimer);
return;
} }
logger.info(`Generische Split-Datei verifiziert (Signatur: ${sig}): ${archiveName}`); logger.info(`Generische Split-Datei verifiziert (Signatur: ${sig}): ${archiveName}`);
} }
@ -1994,23 +1999,60 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{
failed += 1; failed += 1;
const errorText = String(error); const errorText = String(error);
if (isExtractAbortError(errorText)) { if (isExtractAbortError(errorText)) {
throw new Error("aborted:extract"); throw error;
} }
lastError = errorText; lastError = errorText;
const errorCategory = classifyExtractionError(errorText); const errorCategory = classifyExtractionError(errorText);
logger.error(`Entpack-Fehler ${path.basename(archivePath)} [${errorCategory}]: ${errorText}`); logger.error(`Entpack-Fehler ${path.basename(archivePath)} [${errorCategory}]: ${errorText}`);
emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt); emitProgress(extracted + failed, archiveName, "extracting", archivePercent, Date.now() - archiveStartedAt);
if (isNoExtractorError(errorText)) { if (isNoExtractorError(errorText)) {
const remaining = candidates.length - (extracted + failed); noExtractorEncountered = true;
if (remaining > 0) {
failed += remaining;
emitProgress(candidates.length, archiveName, "extracting", 0, Date.now() - archiveStartedAt);
}
break;
} }
} finally { } finally {
clearInterval(pulseTimer); clearInterval(pulseTimer);
} }
};
if (maxParallel <= 1) {
for (const archivePath of pendingCandidates) {
if (options.signal?.aborted || noExtractorEncountered) break;
await extractSingleArchive(archivePath);
}
} else {
// Parallel extraction pool: N workers pull from a shared queue
const queue = [...pendingCandidates];
let nextIdx = 0;
let abortError: Error | null = null;
const worker = async (): Promise<void> => {
while (nextIdx < queue.length && !abortError && !noExtractorEncountered) {
if (options.signal?.aborted) break;
const idx = nextIdx;
nextIdx += 1;
try {
await extractSingleArchive(queue[idx]);
} catch (error) {
if (isExtractAbortError(String(error))) {
abortError = error instanceof Error ? error : new Error(String(error));
break;
}
// Non-abort errors are already handled inside extractSingleArchive
}
}
};
const workerCount = Math.min(maxParallel, pendingCandidates.length);
logger.info(`Parallele Extraktion: ${workerCount} gleichzeitige Worker für ${pendingCandidates.length} Archive`);
await Promise.all(Array.from({ length: workerCount }, () => worker()));
if (abortError) throw new Error("aborted:extract");
if (noExtractorEncountered) {
const remaining = candidates.length - (extracted + failed);
if (remaining > 0) {
failed += remaining;
emitProgress(candidates.length, "", "extracting", 0, 0);
}
}
} }
// ── Nested extraction: extract archives found inside the output (1 level) ── // ── Nested extraction: extract archives found inside the output (1 level) ──