From f70237f13d2e402d7c7f061267a8cca8ac65d52e Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Sat, 28 Feb 2026 07:25:18 +0100 Subject: [PATCH] Release v1.4.22 with incremental hybrid extraction (JDownloader-style) Implements hybrid extraction: when a package has multiple episodes with multi-part archives, completed archive sets are extracted immediately while the rest of the package continues downloading. Uses the existing hybridExtract setting (already in UI/types/storage). Key changes: - Export findArchiveCandidates/pathSetKey from extractor.ts - Add onlyArchives/skipPostCleanup options to ExtractOptions - Add findReadyArchiveSets to identify complete archive sets - Add runHybridExtraction for incremental extraction passes - Requeue logic in runPackagePostProcessing for new completions - Resume state preserved across hybrid passes (no premature clear) - Guard against extracting incomplete multi-part archives - Correct abort/toggle handling during hybrid extraction - Package toggle now also aborts active post-processing Co-Authored-By: Claude Opus 4.6 --- package.json | 2 +- src/main/download-manager.ts | 200 ++++++++++++++++++++++++++++++++++- src/main/extractor.ts | 80 ++++++++------ 3 files changed, 243 insertions(+), 39 deletions(-) diff --git a/package.json b/package.json index 62e5de9..7335f52 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "real-debrid-downloader", - "version": "1.4.21", + "version": "1.4.22", "description": "Real-Debrid Downloader Desktop (Electron + React + TypeScript)", "main": "build/main/main/main.js", "author": "Sucukdeluxe", diff --git a/src/main/download-manager.ts b/src/main/download-manager.ts index 24d1595..d68f408 100644 --- a/src/main/download-manager.ts +++ b/src/main/download-manager.ts @@ -20,7 +20,7 @@ import { import { REQUEST_RETRIES, SAMPLE_VIDEO_EXTENSIONS } from "./constants"; import { cleanupCancelledPackageArtifactsAsync } from "./cleanup"; import { DebridService, MegaWebUnrestrictor } from "./debrid"; -import { collectArchiveCleanupTargets, extractPackageArchives } from "./extractor"; +import { collectArchiveCleanupTargets, extractPackageArchives, findArchiveCandidates } from "./extractor"; import { validateFileAgainstManifest } from "./integrity"; import { logger } from "./logger"; import { StoragePaths, saveSession, saveSessionAsync } from "./storage"; @@ -319,6 +319,8 @@ export class DownloadManager extends EventEmitter { private packagePostProcessAbortControllers = new Map(); + private hybridExtractRequeue = new Set(); + private reservedTargetPaths = new Map(); private claimedTargetPathByItem = new Map(); @@ -526,9 +528,13 @@ export class DownloadManager extends EventEmitter { pkg.enabled = nextEnabled; if (!nextEnabled) { - if (pkg.status === "downloading") { + if (pkg.status === "downloading" || pkg.status === "extracting") { pkg.status = "paused"; } + const postProcessController = this.packagePostProcessAbortControllers.get(packageId); + if (postProcessController && !postProcessController.signal.aborted) { + postProcessController.abort("package_toggle"); + } for (const itemId of pkg.itemIds) { const item = this.session.items[itemId]; if (!item) { @@ -627,6 +633,7 @@ export class DownloadManager extends EventEmitter { this.itemContributedBytes.clear(); this.packagePostProcessTasks.clear(); this.packagePostProcessAbortControllers.clear(); + this.hybridExtractRequeue.clear(); this.packagePostProcessQueue = Promise.resolve(); this.summary = null; this.persistNow(); @@ -1671,6 +1678,7 @@ export class DownloadManager extends EventEmitter { private runPackagePostProcessing(packageId: string): Promise { const existing = this.packagePostProcessTasks.get(packageId); if (existing) { + this.hybridExtractRequeue.add(packageId); return existing; } @@ -1690,6 +1698,11 @@ export class DownloadManager extends EventEmitter { this.packagePostProcessAbortControllers.delete(packageId); this.persistSoon(); this.emitState(); + if (this.hybridExtractRequeue.delete(packageId)) { + void this.runPackagePostProcessing(packageId).catch((err) => + logger.warn(`runPackagePostProcessing Fehler (hybridRequeue): ${compactErrorText(err)}`) + ); + } }); this.packagePostProcessTasks.set(packageId, task); @@ -1808,6 +1821,7 @@ export class DownloadManager extends EventEmitter { this.session.packageOrder = this.session.packageOrder.filter((id) => id !== packageId); this.runPackageIds.delete(packageId); this.runCompletedPackages.delete(packageId); + this.hybridExtractRequeue.delete(packageId); } private async ensureScheduler(): Promise { @@ -2961,6 +2975,169 @@ export class DownloadManager extends EventEmitter { } } + private findReadyArchiveSets(pkg: PackageEntry): Set { + const ready = new Set(); + if (!pkg.outputDir || !fs.existsSync(pkg.outputDir)) { + return ready; + } + + const completedPaths = new Set(); + const pendingPaths = new Set(); + for (const itemId of pkg.itemIds) { + const item = this.session.items[itemId]; + if (!item) { + continue; + } + if (item.status === "completed" && item.targetPath) { + completedPaths.add(pathKey(item.targetPath)); + } else if (item.targetPath) { + pendingPaths.add(pathKey(item.targetPath)); + } + } + if (completedPaths.size === 0) { + return ready; + } + + const candidates = findArchiveCandidates(pkg.outputDir); + if (candidates.length === 0) { + return ready; + } + + let dirFiles: string[] | undefined; + try { + dirFiles = fs.readdirSync(pkg.outputDir, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name); + } catch { + return ready; + } + + for (const candidate of candidates) { + const partsOnDisk = collectArchiveCleanupTargets(candidate, dirFiles); + const allPartsCompleted = partsOnDisk.every((part) => completedPaths.has(pathKey(part))); + if (!allPartsCompleted) { + continue; + } + const hasUnstartedParts = [...pendingPaths].some((pendingPath) => { + const pendingName = path.basename(pendingPath).toLowerCase(); + const candidateStem = path.basename(candidate).toLowerCase(); + return this.looksLikeArchivePart(pendingName, candidateStem); + }); + if (hasUnstartedParts) { + continue; + } + ready.add(pathKey(candidate)); + } + + return ready; + } + + private looksLikeArchivePart(fileName: string, entryPointName: string): boolean { + const multipartMatch = entryPointName.match(/^(.*)\.part0*1\.rar$/i); + if (multipartMatch) { + const prefix = multipartMatch[1].toLowerCase(); + return new RegExp(`^${prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\.part\\d+\\.rar$`, "i").test(fileName); + } + if (/\.rar$/i.test(entryPointName) && !/\.part\d+\.rar$/i.test(entryPointName)) { + const stem = entryPointName.replace(/\.rar$/i, "").toLowerCase(); + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`^${escaped}\\.r(ar|\\d{2})$`, "i").test(fileName); + } + if (/\.zip\.001$/i.test(entryPointName)) { + const stem = entryPointName.replace(/\.zip\.001$/i, "").toLowerCase(); + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`^${escaped}\\.zip(\\.\\d{3})?$`, "i").test(fileName); + } + if (/\.7z\.001$/i.test(entryPointName)) { + const stem = entryPointName.replace(/\.7z\.001$/i, "").toLowerCase(); + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`^${escaped}\\.7z(\\.\\d{3})?$`, "i").test(fileName); + } + return false; + } + + private async runHybridExtraction(packageId: string, pkg: PackageEntry, items: DownloadItem[], signal?: AbortSignal): Promise { + const readyArchives = this.findReadyArchiveSets(pkg); + if (readyArchives.size === 0) { + logger.info(`Hybrid-Extract: pkg=${pkg.name}, keine fertigen Archive-Sets`); + return; + } + + logger.info(`Hybrid-Extract Start: pkg=${pkg.name}, readyArchives=${readyArchives.size}`); + pkg.status = "extracting"; + this.emitState(); + + const completedItems = items.filter((item) => item.status === "completed"); + const updateExtractingStatus = (text: string): void => { + const updatedAt = nowMs(); + for (const entry of completedItems) { + if (isExtractedLabel(entry.fullStatus)) { + continue; + } + if (entry.fullStatus === text) { + continue; + } + entry.fullStatus = text; + entry.updatedAt = updatedAt; + } + }; + + updateExtractingStatus("Entpacken (hybrid) 0%"); + this.emitState(); + + try { + const result = await extractPackageArchives({ + packageDir: pkg.outputDir, + targetDir: pkg.extractDir, + cleanupMode: this.settings.cleanupMode, + conflictMode: this.settings.extractConflictMode, + removeLinks: false, + removeSamples: false, + passwordList: this.settings.archivePasswordList, + signal, + onlyArchives: readyArchives, + skipPostCleanup: true, + onProgress: (progress) => { + if (progress.phase === "done") { + return; + } + const archive = progress.archiveName ? ` · ${progress.archiveName}` : ""; + const elapsed = progress.elapsedMs && progress.elapsedMs >= 1000 + ? ` · ${Math.floor(progress.elapsedMs / 1000)}s` + : ""; + const activeArchive = Number(progress.archivePercent ?? 0) > 0 ? 1 : 0; + const currentDisplay = Math.max(0, Math.min(progress.total, progress.current + activeArchive)); + const label = `Entpacken (hybrid) ${progress.percent}% (${currentDisplay}/${progress.total})${archive}${elapsed}`; + updateExtractingStatus(label); + this.emitState(); + } + }); + + logger.info(`Hybrid-Extract Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}`); + if (result.extracted > 0) { + this.autoRenameExtractedVideoFiles(pkg.extractDir); + } + if (result.failed > 0) { + logger.warn(`Hybrid-Extract: ${result.failed} Archive fehlgeschlagen, wird beim finalen Durchlauf erneut versucht`); + } + + const updatedAt = nowMs(); + for (const entry of completedItems) { + if (/^Entpacken \(hybrid\)/i.test(entry.fullStatus || "")) { + entry.fullStatus = `Fertig (${humanSize(entry.downloadedBytes)})`; + entry.updatedAt = updatedAt; + } + } + } catch (error) { + const errorText = String(error || ""); + if (errorText.includes("aborted:extract")) { + logger.info(`Hybrid-Extract abgebrochen: pkg=${pkg.name}`); + return; + } + logger.warn(`Hybrid-Extract Fehler: pkg=${pkg.name}, reason=${compactErrorText(error)}`); + } + } + private async handlePackagePostProcessing(packageId: string, signal?: AbortSignal): Promise { const pkg = this.session.packages[packageId]; if (!pkg || pkg.cancelled) { @@ -2975,8 +3152,23 @@ export class DownloadManager extends EventEmitter { const cancelled = items.filter((item) => item.status === "cancelled").length; logger.info(`Post-Processing Start: pkg=${pkg.name}, success=${success}, failed=${failed}, cancelled=${cancelled}, autoExtract=${this.settings.autoExtract}`); - if (success + failed + cancelled < items.length) { - pkg.status = "downloading"; + const allDone = success + failed + cancelled >= items.length; + + if (!allDone && this.settings.hybridExtract && this.settings.autoExtract && failed === 0 && success > 0) { + await this.runHybridExtraction(packageId, pkg, items, signal); + if (signal?.aborted) { + pkg.status = pkg.enabled ? "queued" : "paused"; + pkg.updatedAt = nowMs(); + return; + } + pkg.status = pkg.enabled ? "downloading" : "paused"; + pkg.updatedAt = nowMs(); + this.emitState(); + return; + } + + if (!allDone) { + pkg.status = pkg.enabled ? "downloading" : "paused"; logger.info(`Post-Processing verschoben: pkg=${pkg.name}, noch offene items`); return; } diff --git a/src/main/extractor.ts b/src/main/extractor.ts index 5fdf8df..5d5a07a 100644 --- a/src/main/extractor.ts +++ b/src/main/extractor.ts @@ -24,6 +24,8 @@ export interface ExtractOptions { passwordList?: string; signal?: AbortSignal; onProgress?: (update: ExtractProgressUpdate) => void; + onlyArchives?: Set; + skipPostCleanup?: boolean; } export interface ExtractProgressUpdate { @@ -43,7 +45,7 @@ const EXTRACT_PER_GIB_TIMEOUT_MS = 4 * 60 * 1000; const EXTRACT_MAX_TIMEOUT_MS = 120 * 60 * 1000; const ARCHIVE_SORT_COLLATOR = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }); -function pathSetKey(filePath: string): string { +export function pathSetKey(filePath: string): string { return process.platform === "win32" ? filePath.toLowerCase() : filePath; } @@ -80,7 +82,7 @@ type ExtractResumeState = { completedArchives: string[]; }; -function findArchiveCandidates(packageDir: string): string[] { +export function findArchiveCandidates(packageDir: string): string[] { if (!packageDir || !fs.existsSync(packageDir)) { return []; } @@ -831,23 +833,31 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ throw new Error("aborted:extract"); } - const candidates = findArchiveCandidates(options.packageDir); - logger.info(`Entpacken gestartet: packageDir=${options.packageDir}, targetDir=${options.targetDir}, archives=${candidates.length}, cleanupMode=${options.cleanupMode}, conflictMode=${options.conflictMode}`); + const allCandidates = findArchiveCandidates(options.packageDir); + const candidates = options.onlyArchives + ? allCandidates.filter((archivePath) => { + const key = process.platform === "win32" ? path.resolve(archivePath).toLowerCase() : path.resolve(archivePath); + return options.onlyArchives!.has(key); + }) + : allCandidates; + logger.info(`Entpacken gestartet: packageDir=${options.packageDir}, targetDir=${options.targetDir}, archives=${candidates.length}${options.onlyArchives ? ` (hybrid, gesamt=${allCandidates.length})` : ""}, cleanupMode=${options.cleanupMode}, conflictMode=${options.conflictMode}`); if (candidates.length === 0) { - const existingResume = readExtractResumeState(options.packageDir); - if (existingResume.size > 0 && hasAnyEntries(options.targetDir)) { + if (!options.onlyArchives) { + const existingResume = readExtractResumeState(options.packageDir); + if (existingResume.size > 0 && hasAnyEntries(options.targetDir)) { + clearExtractResumeState(options.packageDir); + logger.info(`Entpacken übersprungen (Archive bereinigt, Ziel hat Dateien): ${options.packageDir}`); + options.onProgress?.({ + current: existingResume.size, + total: existingResume.size, + percent: 100, + archiveName: "", + phase: "done" + }); + return { extracted: existingResume.size, failed: 0, lastError: "" }; + } clearExtractResumeState(options.packageDir); - logger.info(`Entpacken übersprungen (Archive bereinigt, Ziel hat Dateien): ${options.packageDir}`); - options.onProgress?.({ - current: existingResume.size, - total: existingResume.size, - percent: 100, - archiveName: "", - phase: "done" - }); - return { extracted: existingResume.size, failed: 0, lastError: "" }; } - clearExtractResumeState(options.packageDir); logger.info(`Entpacken übersprungen (keine Archive gefunden): ${options.packageDir}`); return { extracted: 0, failed: 0, lastError: "" }; } @@ -856,9 +866,9 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ let passwordCandidates = archivePasswords(options.passwordList || ""); const resumeCompleted = readExtractResumeState(options.packageDir); const resumeCompletedAtStart = resumeCompleted.size; - const candidateNames = new Set(candidates.map((archivePath) => path.basename(archivePath))); + const allCandidateNames = new Set(allCandidates.map((archivePath) => path.basename(archivePath))); for (const archiveName of Array.from(resumeCompleted.values())) { - if (!candidateNames.has(archiveName)) { + if (!allCandidateNames.has(archiveName)) { resumeCompleted.delete(archiveName); } } @@ -869,7 +879,7 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ } const pendingCandidates = candidates.filter((archivePath) => !resumeCompleted.has(path.basename(archivePath))); - let extracted = resumeCompleted.size; + let extracted = candidates.length - pendingCandidates.length; let failed = 0; let lastError = ""; const extractedArchives = new Set(); @@ -996,32 +1006,34 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<{ extracted = 0; logger.error(`Entpacken ohne neue Ausgabe erkannt: ${options.targetDir}. Cleanup wird NICHT ausgeführt.`); } else { - const cleanupSources = failed === 0 ? candidates : Array.from(extractedArchives.values()); - const removedArchives = cleanupArchives(cleanupSources, options.cleanupMode); - if (options.cleanupMode !== "none") { - logger.info(`Archive-Cleanup abgeschlossen: ${removedArchives} Datei(en) entfernt`); - } - if (options.removeLinks) { - const removedLinks = removeDownloadLinkArtifacts(options.targetDir); - logger.info(`Link-Artefakt-Cleanup: ${removedLinks} Datei(en) entfernt`); - } - if (options.removeSamples) { - const removedSamples = removeSampleArtifacts(options.targetDir); - logger.info(`Sample-Cleanup: ${removedSamples.files} Datei(en), ${removedSamples.dirs} Ordner entfernt`); + if (!options.skipPostCleanup) { + const cleanupSources = failed === 0 ? candidates : Array.from(extractedArchives.values()); + const removedArchives = cleanupArchives(cleanupSources, options.cleanupMode); + if (options.cleanupMode !== "none") { + logger.info(`Archive-Cleanup abgeschlossen: ${removedArchives} Datei(en) entfernt`); + } + if (options.removeLinks) { + const removedLinks = removeDownloadLinkArtifacts(options.targetDir); + logger.info(`Link-Artefakt-Cleanup: ${removedLinks} Datei(en) entfernt`); + } + if (options.removeSamples) { + const removedSamples = removeSampleArtifacts(options.targetDir); + logger.info(`Sample-Cleanup: ${removedSamples.files} Datei(en), ${removedSamples.dirs} Ordner entfernt`); + } } - if (failed === 0 && resumeCompleted.size >= candidates.length) { + if (failed === 0 && resumeCompleted.size >= allCandidates.length) { clearExtractResumeState(options.packageDir); } - if (options.cleanupMode === "delete" && !hasAnyFilesRecursive(options.packageDir)) { + if (!options.skipPostCleanup && options.cleanupMode === "delete" && !hasAnyFilesRecursive(options.packageDir)) { const removedDirs = removeEmptyDirectoryTree(options.packageDir); if (removedDirs > 0) { logger.info(`Leere Download-Ordner entfernt: ${removedDirs} (root=${options.packageDir})`); } } } - } else { + } else if (!options.skipPostCleanup) { try { if (fs.existsSync(options.targetDir) && fs.readdirSync(options.targetDir).length === 0) { fs.rmSync(options.targetDir, { recursive: true, force: true });