Administrator/Multi-Debrid-Downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 596 Wiki Activity

fix(extraction): close final scope integration gaps

Browse Source 
Ignore late JVM output events after abort or timeout until the real child close reconciles opened files. Restrict deferred archive cleanup to package-owned item paths so shared download roots retain foreign archives while package-local empty directory cleanup still works. Update deferred ownership and output-scope fixtures to the coordinator contract.
This commit is contained in:
Sucukdeluxe
2026-08-22 17:29:03 +02:00
parent 3e18ec57ea
commit 522f53d787
4 changed files with 231 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/download-manager.ts
+34 -38
View File
@@ -64,7 +64,7 @@ function releaseTlsSkip(): void {
import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifactsFromScope, removeSampleArtifactsFromScope } from "./cleanup";
import { planDownloadCompletion, reconcileFinalizedSize, validateDownloadedFileCompletion } from "./download-completion";
import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkDdownloadOnline, checkOneFichierLinks, checkRapidgatorOnline, fetchAllDebridHostInfo, filenameFromDdownloadUrlPath, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, isDdownloadLink, isOneFichierLink, isProviderDisabledForSelection, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown, type DdownloadCheckResult, type OneFichierCheckResult } from "./debrid";
import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, detectArchiveSignature, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, resetExtractorCachesForPasswordChange, type ExtractArchiveFailureInfo, type ExtractProgressUpdate } from "./extractor";
import { clearExtractResumeState, collectArchiveCleanupTargets, detectArchiveSignature, extractPackageArchives, findArchiveCandidates, resetExtractorCachesForPasswordChange, type ExtractArchiveFailureInfo, type ExtractProgressUpdate } from "./extractor";
import { validateFileAgainstManifest } from "./integrity";
import { classifyDiskError } from "./fs-error";
import { processVideoFile, resolveVideoTooling, stripDualLangMarker, hasDualLangMarker, isRemuxableVideoFile, type GermanAudioMode, type VideoProcessResult } from "./video-processor";
@@ -5883,39 +5883,47 @@ export class DownloadManager extends EventEmitter {
return removed;
}
private async cleanupRemainingArchiveArtifacts(packageDir: string, shouldAbort?: () => boolean): Promise<number> {
private async cleanupRemainingArchiveArtifacts(pkg: PackageEntry, shouldAbort?: () => boolean): Promise<number> {
if (this.settings.cleanupMode === "none") {
return 0;
}
if (shouldAbort?.()) {
return 0;
}
const candidates = await findArchiveCandidates(packageDir);
if (candidates.length === 0) {
const ownedPaths = new Map<string, string>();
for (const itemId of pkg.itemIds) {
const item = this.session.items[itemId];
const rawPath = String(item?.targetPath || (item?.fileName ? path.join(pkg.outputDir, item.fileName) : "")).trim();
if (!rawPath || !isArchiveLikePath(rawPath) || !isPathInsideDir(rawPath, pkg.outputDir)) {
continue;
}
const resolved = path.resolve(rawPath);
ownedPaths.set(pathKey(resolved), resolved);
}
if (ownedPaths.size === 0) {
return 0;
}
let removed = 0;
const dirFilesCache = new Map<string, string[]>();
const dirFiles = new Map<string, string[]>();
for (const ownedPath of ownedPaths.values()) {
const directory = path.dirname(ownedPath);
const directoryKey = pathKey(directory);
const files = dirFiles.get(directoryKey) || [];
files.push(path.basename(ownedPath));
dirFiles.set(directoryKey, files);
}
const targets = new Set<string>();
for (const sourceFile of candidates) {
for (const sourceFile of ownedPaths.values()) {
if (shouldAbort?.()) {
return removed;
}
const dir = path.dirname(sourceFile);
let filesInDir = dirFilesCache.get(dir);
if (!filesInDir) {
try {
filesInDir = (await fs.promises.readdir(dir, { withFileTypes: true }))
.filter((entry) => entry.isFile())
.map((entry) => entry.name);
} catch {
filesInDir = [];
for (const target of collectArchiveCleanupTargets(sourceFile, dirFiles.get(pathKey(dir)) || [])) {
const resolved = path.resolve(target);
if (ownedPaths.has(pathKey(resolved))) {
targets.add(resolved);
}
dirFilesCache.set(dir, filesInDir);
}
for (const target of collectArchiveCleanupTargets(sourceFile, filesInDir)) {
targets.add(target);
}
}
@@ -14599,24 +14607,13 @@ export class DownloadManager extends EventEmitter {
} else {
const sourceAndTargetEqual = path.resolve(pkg.outputDir).toLowerCase() === path.resolve(pkg.extractDir).toLowerCase();
if (!sourceAndTargetEqual) {
const candidates = await findArchiveCandidates(pkg.outputDir);
if (candidates.length > 0) {
const removed = await cleanupArchives(candidates, this.settings.cleanupMode, { shouldAbort });
const removed = await this.cleanupRemainingArchiveArtifacts(pkg, shouldAbort);
if (removed > 0) {
logger.info(`Deferred Archive-Cleanup: pkg=${pkg.name}, entfernt=${removed}`);
}
}
}
}
}
if (this.settings.autoExtract && alreadyMarkedExtracted && failed === 0 && success > 0 && this.settings.cleanupMode !== "none" && !hasBlockingExtractError) {
throwIfAborted();
const removedArchives = await this.cleanupRemainingArchiveArtifacts(pkg.outputDir, shouldAbort);
if (removedArchives > 0) {
logger.info(`Hybrid-Post-Cleanup entfernte Archive: pkg=${pkg.name}, entfernt=${removedArchives}`);
}
}
if (extractedCount > 0 || alreadyMarkedExtracted) {
throwIfAborted();
@@ -14640,16 +14637,15 @@ export class DownloadManager extends EventEmitter {
throwIfAborted();
await clearExtractResumeState(pkg.outputDir, packageId);
await clearExtractResumeState(pkg.outputDir);
}
if ((extractedCount > 0 || alreadyMarkedExtracted) && failed === 0 && this.settings.cleanupMode === "delete") {
throwIfAborted();
if (!(await hasAnyFilesRecursive(pkg.outputDir))) {
const removedDirs = await removeEmptyDirectoryTree(pkg.outputDir);
if (removedDirs > 0) {
logger.info(`Deferred leere Download-Ordner entfernt: pkg=${pkg.name}, dirs=${removedDirs}`);
const archiveParents = new Set<string>();
for (const itemId of pkg.itemIds) {
const item = this.session.items[itemId];
const itemPath = String(item?.targetPath || (item?.fileName ? path.join(pkg.outputDir, item.fileName) : "")).trim();
if (itemPath && isArchiveLikePath(itemPath) && isPathInsideDir(itemPath, pkg.outputDir)) {
archiveParents.add(path.dirname(path.resolve(itemPath)));
}
}
await this.removeEmptyScopedParentChains(pkg.outputDir, archiveParents);
}
if (success > 0 && (pkg.status === "completed" || pkg.status === "failed")) {
src/main/extractor.ts
+6
View File
@@ -1863,6 +1863,9 @@ function handleDaemonLine(line: string): void {
if (daemonCurrentRequest) {
const req = daemonCurrentRequest;
if (req.terminationStarted) {
return;
}
parseJvmLine(trimmed, req.onArchiveProgress, req.parseState, req.onOutput);
failDaemonOutputCallback(req);
}
@@ -1918,6 +1921,9 @@ function startDaemon(layout: JvmExtractorLayout): boolean {
for (const line of lines) {
if (daemonCurrentRequest) {
const req = daemonCurrentRequest;
if (req.terminationStarted) {
continue;
}
parseJvmLine(line, req.onArchiveProgress, req.parseState, req.onOutput);
failDaemonOutputCallback(req);
}
tests/download-manager.test.ts
+98 -1
View File
@@ -43,6 +43,19 @@ function writePackageOutputOwnerMarker(pkg: PackageEntry): void {
}));
}
function setExtractOutputRecords(pkg: PackageEntry, outputPaths: string[]): void {
pkg.outputProvenanceVersion = 1;
pkg.outputRecords = outputPaths.map((outputPath) => ({
version: 1,
archivePath: path.join(pkg.outputDir, "source.zip"),
entryPath: path.relative(pkg.extractDir, outputPath).replace(/\\/g, "/"),
outputPath,
state: "complete",
disposition: "written"
}));
pkg.outputCount = outputPaths.length;
}
describe("runWithLimitedConcurrency", () => {
it("processes the full batch without exceeding the configured worker count", async () => {
let active = 0;
@@ -12064,6 +12077,10 @@ describe("download manager", () => {
createdAt,
updatedAt: createdAt
};
setExtractOutputRecords(session.packages[packageId], [
path.join(extractDir, "episode.links.txt"),
path.join(extractDir, "sample", "sample.mkv")
]);
const manager = new DownloadManager(
{
@@ -12096,6 +12113,83 @@ describe("download manager", () => {
expect(fs.existsSync(path.join(extractDir, "sample", "sample.mkv"))).toBe(false);
});
it("deletes only package-owned archive members from a shared download root", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-shared-cleanup-"));
tempDirs.push(root);
const outputDir = path.join(root, "downloads");
const extractDir = path.join(root, "extract", "owned-package");
fs.mkdirSync(outputDir, { recursive: true });
fs.mkdirSync(extractDir, { recursive: true });
const ownedPaths = [path.join(outputDir, "owned.part1.rar"), path.join(outputDir, "owned.part2.rar")];
const foreignPaths = [path.join(outputDir, "foreign.part1.rar"), path.join(outputDir, "foreign.part2.rar")];
for (const archivePath of [...ownedPaths, ...foreignPaths]) {
fs.writeFileSync(archivePath, archivePath);
}
const extractedPath = path.join(extractDir, "episode.mkv");
fs.writeFileSync(extractedPath, "video");
const session = emptySession();
const packageId = "owned-package";
const createdAt = Date.now() - 20_000;
session.packageOrder = [packageId];
session.packages[packageId] = {
id: packageId,
name: packageId,
outputDir,
extractDir,
status: "completed",
itemIds: ["owned-part-1", "owned-part-2"],
cancelled: false,
enabled: true,
createdAt,
updatedAt: createdAt
};
for (let index = 0; index < ownedPaths.length; index += 1) {
const itemId = `owned-part-${index + 1}`;
session.items[itemId] = {
id: itemId,
packageId,
url: `https://example.com/${path.basename(ownedPaths[index])}`,
provider: "realdebrid",
status: "completed",
retries: 0,
speedBps: 0,
downloadedBytes: fs.statSync(ownedPaths[index]).size,
totalBytes: fs.statSync(ownedPaths[index]).size,
progressPercent: 100,
fileName: path.basename(ownedPaths[index]),
targetPath: ownedPaths[index],
resumable: true,
attempts: 1,
lastError: "",
fullStatus: "Entpackt - Done (1s)",
createdAt,
updatedAt: createdAt
};
}
setExtractOutputRecords(session.packages[packageId], [extractedPath]);
const manager = new DownloadManager(
{
...defaultSettings(),
outputDir,
extractDir: path.join(root, "extract"),
autoExtract: true,
autoRename4sf4sj: false,
collectMkvToLibrary: false,
removeLinkFilesAfterExtract: false,
removeSamplesAfterExtract: false,
enableIntegrityCheck: false,
cleanupMode: "delete"
},
session,
createStoragePaths(path.join(root, "state"))
);
await (manager as any).runDeferredPostExtraction(packageId, session.packages[packageId], 1, 0, true, 1);
expect(ownedPaths.map((archivePath) => fs.existsSync(archivePath))).toEqual([false, false]);
expect(foreignPaths.map((archivePath) => fs.existsSync(archivePath))).toEqual([true, true]);
});
it("does not delete startup archives when any completed item has an extract error", async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
tempDirs.push(root);
@@ -13089,12 +13183,14 @@ describe("download manager", () => {
createStoragePaths(path.join(root, "state"))
);
(manager as any).fileStabilizeMinAgeMs = 30_000;
setExtractOutputRecords(session.packages[packageId], [scenePath]);
const expectedBase = "Test.Show.S02E05.Title.GERMAN.WS.720p.HDTV.x264-aWake";
const renamedLibPath = path.join(mkvLibraryDir, `${expectedBase}.mkv`);
const sceneLibPath = path.join(mkvLibraryDir, sceneName);
await (manager as any).autoRenameExtractedVideoFiles(extractDir, session.packages[packageId], undefined, true);
const outputScope = (manager as any).getPackageOutputScope(session.packages[packageId]);
await (manager as any).autoRenameExtractedVideoFiles(extractDir, outputScope, session.packages[packageId], undefined, true);
await (manager as any).collectMkvFilesToLibrary(packageId, session.packages[packageId], undefined, false);
expect(fs.existsSync(renamedLibPath)).toBe(true);
@@ -13153,6 +13249,7 @@ describe("download manager", () => {
(manager as any).fileStabilizeMinAgeMs = 30_000;
const expectedBase = "Test.Show.S02E05.Title.GERMAN.WS.720p.HDTV.x264-aWake";
setExtractOutputRecords(session.packages[packageId], [path.join(epFolder, sceneName)]);
await (manager as any).runDeferredPostExtraction(packageId, session.packages[packageId], 1, 0, true, 1);
expect(fs.existsSync(path.join(mkvLibraryDir, `${expectedBase}.mkv`))).toBe(true);
tests/notify-hooks.test.ts
+17 -3
View File
@@ -789,7 +789,21 @@ describe("authoritative run completion", () => {
packageAItem.fullStatus = "Fertig";
packageA.status = "completed";
state.runOutcomes.set(packageAItem.id, "completed");
state.packagePostProcessActive = 1;
const handlePackagePostProcessing = state.handlePackagePostProcessing.bind(state);
let releaseMainPostProcess = (): void => {};
let markMainPostProcessEntered = (): void => {};
const mainPostProcessGate = new Promise<void>((resolve) => {
releaseMainPostProcess = resolve;
});
const mainPostProcessEntered = new Promise<void>((resolve) => {
markMainPostProcessEntered = resolve;
});
vi.spyOn(state, "handlePackagePostProcessing").mockImplementation(async (...args: unknown[]) => {
const [packageId, signal] = args as [string, AbortSignal?];
markMainPostProcessEntered();
await mainPostProcessGate;
await handlePackagePostProcessing(packageId, signal);
});
let releaseCollection = (): void => {};
const collectionGate = new Promise<void>((resolve) => {
@@ -797,14 +811,14 @@ describe("authoritative run completion", () => {
});
const collect = vi.spyOn(state, "collectMkvFilesToLibrary").mockImplementation(async () => collectionGate);
const packageAMainPostProcess = state.runPackagePostProcessing(packageA.id);
await vi.waitFor(() => expect(state.packagePostProcessWaiters).toHaveLength(1));
await mainPostProcessEntered;
state.finishRun();
const packageB = addPackage(session, ["queued"], "active-run-package");
await manager.start();
expect(state.runPackageIds).toEqual(new Set([packageB.id]));
state.releasePostProcessSlot();
releaseMainPostProcess();
await packageAMainPostProcess;
await vi.waitFor(() => expect(collect).toHaveBeenCalled());
const deferredTasks = [...(state.packageDeferredPostProcessTasks.get(packageA.id) || [])];