Fix deferred cleanup after MKV move
This commit is contained in:
parent
a5d53eff74
commit
b41bb0aeb5
@ -3500,6 +3500,11 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return removed;
|
return removed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private hasDeferredPostProcessPending(packageId: string): boolean {
|
||||||
|
const controller = this.packageDeferredPostProcessAbortControllers.get(packageId);
|
||||||
|
return Boolean(controller && !controller.signal.aborted);
|
||||||
|
}
|
||||||
|
|
||||||
private async buildUniqueFlattenTargetPath(targetDir: string, sourcePath: string, reserved: Set<string>): Promise<string> {
|
private async buildUniqueFlattenTargetPath(targetDir: string, sourcePath: string, reserved: Set<string>): Promise<string> {
|
||||||
const parsed = path.parse(path.basename(sourcePath));
|
const parsed = path.parse(path.basename(sourcePath));
|
||||||
const extension = parsed.ext || ".mkv";
|
const extension = parsed.ext || ".mkv";
|
||||||
@ -3599,6 +3604,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
let moved = 0;
|
let moved = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
|
let sourceArtifactsChanged = false;
|
||||||
|
|
||||||
for (const sourcePath of mkvFiles) {
|
for (const sourcePath of mkvFiles) {
|
||||||
if (shouldAbort?.()) {
|
if (shouldAbort?.()) {
|
||||||
@ -3643,7 +3649,12 @@ export class DownloadManager extends EventEmitter {
|
|||||||
sourceSize
|
sourceSize
|
||||||
}, resolved.item, resolved.matchedBy);
|
}, resolved.item, resolved.matchedBy);
|
||||||
// Remove the duplicate source file to avoid future re-processing
|
// Remove the duplicate source file to avoid future re-processing
|
||||||
try { await fs.promises.unlink(sourcePath); } catch { /* ignore */ }
|
try {
|
||||||
|
await fs.promises.unlink(sourcePath);
|
||||||
|
sourceArtifactsChanged = true;
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
skipped += 1;
|
skipped += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -3660,6 +3671,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
try {
|
try {
|
||||||
await this.moveFileWithExdevFallback(sourcePath, targetPath);
|
await this.moveFileWithExdevFallback(sourcePath, targetPath);
|
||||||
moved += 1;
|
moved += 1;
|
||||||
|
sourceArtifactsChanged = true;
|
||||||
this.logPackageForPackage(pkg, "INFO", "MKV verschoben", {
|
this.logPackageForPackage(pkg, "INFO", "MKV verschoben", {
|
||||||
sourcePath,
|
sourcePath,
|
||||||
targetPath,
|
targetPath,
|
||||||
@ -3689,7 +3701,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (moved > 0 && await this.existsAsync(sourceDir)) {
|
if (sourceArtifactsChanged && await this.existsAsync(sourceDir)) {
|
||||||
const removedResidual = await this.cleanupNonMkvResidualFiles(sourceDir, targetDir);
|
const removedResidual = await this.cleanupNonMkvResidualFiles(sourceDir, targetDir);
|
||||||
if (removedResidual > 0) {
|
if (removedResidual > 0) {
|
||||||
logger.info(`MKV-Sammelordner entfernte Restdateien: pkg=${pkg.name}, entfernt=${removedResidual}`);
|
logger.info(`MKV-Sammelordner entfernte Restdateien: pkg=${pkg.name}, entfernt=${removedResidual}`);
|
||||||
@ -10444,7 +10456,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|
|
||||||
if (policy === "immediate") {
|
if (policy === "immediate") {
|
||||||
for (const itemId of [...pkg.itemIds]) {
|
for (const itemId of [...pkg.itemIds]) {
|
||||||
this.applyCompletedCleanupPolicy(packageId, itemId);
|
this.applyCompletedCleanupPolicy(packageId, itemId, { ignoreDeferred: true });
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -10473,7 +10485,11 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.removePackageFromSession(packageId, [...pkg.itemIds], "completed");
|
this.removePackageFromSession(packageId, [...pkg.itemIds], "completed");
|
||||||
}
|
}
|
||||||
|
|
||||||
private applyCompletedCleanupPolicy(packageId: string, itemId: string): void {
|
private applyCompletedCleanupPolicy(
|
||||||
|
packageId: string,
|
||||||
|
itemId: string,
|
||||||
|
options?: { ignoreDeferred?: boolean }
|
||||||
|
): void {
|
||||||
const policy = this.settings.completedCleanupPolicy;
|
const policy = this.settings.completedCleanupPolicy;
|
||||||
if (policy === "never" || policy === "on_start") {
|
if (policy === "never" || policy === "on_start") {
|
||||||
return;
|
return;
|
||||||
@ -10484,6 +10500,10 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!options?.ignoreDeferred && this.hasDeferredPostProcessPending(packageId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (policy === "immediate") {
|
if (policy === "immediate") {
|
||||||
const item = this.session.items[itemId];
|
const item = this.session.items[itemId];
|
||||||
if (!item || item.status !== "completed") {
|
if (!item || item.status !== "completed") {
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import fs from "node:fs";
|
|||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import http from "node:http";
|
import http from "node:http";
|
||||||
|
import crypto from "node:crypto";
|
||||||
import { EventEmitter, once } from "node:events";
|
import { EventEmitter, once } from "node:events";
|
||||||
import AdmZip from "adm-zip";
|
import AdmZip from "adm-zip";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
@ -6143,6 +6144,7 @@ describe("download manager", () => {
|
|||||||
|
|
||||||
const zip = new AdmZip();
|
const zip = new AdmZip();
|
||||||
zip.addFile("episode.txt", Buffer.from("ok"));
|
zip.addFile("episode.txt", Buffer.from("ok"));
|
||||||
|
zip.addFile("padding.bin", crypto.randomBytes(8 * 1024));
|
||||||
const archiveBinary = zip.toBuffer();
|
const archiveBinary = zip.toBuffer();
|
||||||
|
|
||||||
const server = http.createServer((req, res) => {
|
const server = http.createServer((req, res) => {
|
||||||
@ -6203,6 +6205,7 @@ describe("download manager", () => {
|
|||||||
manager.addPackages([{ name: "cleanup-package", links: ["https://dummy/cleanup-package"] }]);
|
manager.addPackages([{ name: "cleanup-package", links: ["https://dummy/cleanup-package"] }]);
|
||||||
await manager.start();
|
await manager.start();
|
||||||
await waitFor(() => !manager.getSnapshot().session.running, 30000);
|
await waitFor(() => !manager.getSnapshot().session.running, 30000);
|
||||||
|
await waitFor(() => manager.getSnapshot().session.packageOrder.length === 0, 12000);
|
||||||
|
|
||||||
const snapshot = manager.getSnapshot();
|
const snapshot = manager.getSnapshot();
|
||||||
const summary = manager.getSummary();
|
const summary = manager.getSummary();
|
||||||
@ -6216,6 +6219,92 @@ describe("download manager", () => {
|
|||||||
}
|
}
|
||||||
}, 35000);
|
}, 35000);
|
||||||
|
|
||||||
|
it("waits for deferred MKV collection before package_done cleanup removes the package", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
|
||||||
|
const zip = new AdmZip();
|
||||||
|
zip.addFile("Season 1/Episode01.mkv", Buffer.from("video"));
|
||||||
|
zip.addFile("Season 1/sample.txt", Buffer.from("sample"));
|
||||||
|
zip.addFile("padding.bin", crypto.randomBytes(8 * 1024));
|
||||||
|
const archiveBinary = zip.toBuffer();
|
||||||
|
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
if ((req.url || "") !== "/cleanup-package-mkv") {
|
||||||
|
res.statusCode = 404;
|
||||||
|
res.end("not-found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.statusCode = 200;
|
||||||
|
res.setHeader("Accept-Ranges", "bytes");
|
||||||
|
res.setHeader("Content-Length", String(archiveBinary.length));
|
||||||
|
res.end(archiveBinary);
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(0, "127.0.0.1");
|
||||||
|
await once(server, "listening");
|
||||||
|
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") {
|
||||||
|
throw new Error("server address unavailable");
|
||||||
|
}
|
||||||
|
const directUrl = `http://127.0.0.1:${address.port}/cleanup-package-mkv`;
|
||||||
|
|
||||||
|
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||||
|
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||||
|
if (url.includes("/unrestrict/link")) {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
download: directUrl,
|
||||||
|
filename: "cleanup-package-mkv.zip",
|
||||||
|
filesize: archiveBinary.length
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
status: 200,
|
||||||
|
headers: { "Content-Type": "application/json" }
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return originalFetch(input, init);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const extractRoot = path.join(root, "extract");
|
||||||
|
const mkvLibraryDir = path.join(root, "mkv-library");
|
||||||
|
const manager = new DownloadManager(
|
||||||
|
{
|
||||||
|
...defaultSettings(),
|
||||||
|
token: "rd-token",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: extractRoot,
|
||||||
|
autoExtract: true,
|
||||||
|
autoRename4sf4sj: false,
|
||||||
|
collectMkvToLibrary: true,
|
||||||
|
mkvLibraryDir,
|
||||||
|
enableIntegrityCheck: false,
|
||||||
|
cleanupMode: "delete",
|
||||||
|
completedCleanupPolicy: "package_done"
|
||||||
|
},
|
||||||
|
emptySession(),
|
||||||
|
createStoragePaths(path.join(root, "state"))
|
||||||
|
);
|
||||||
|
|
||||||
|
manager.addPackages([{ name: "cleanup-package-mkv", links: ["https://dummy/cleanup-package-mkv"] }]);
|
||||||
|
await manager.start();
|
||||||
|
await waitFor(() => !manager.getSnapshot().session.running, 30000);
|
||||||
|
await waitFor(() => manager.getSnapshot().session.packageOrder.length === 0, 12000);
|
||||||
|
|
||||||
|
const flattenedPath = path.join(mkvLibraryDir, "Episode01.mkv");
|
||||||
|
const extractDir = path.join(extractRoot, "cleanup-package-mkv");
|
||||||
|
expect(fs.existsSync(flattenedPath)).toBe(true);
|
||||||
|
expect(fs.existsSync(extractDir)).toBe(false);
|
||||||
|
expect(Object.keys(manager.getSnapshot().session.items)).toHaveLength(0);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
await once(server, "close");
|
||||||
|
}
|
||||||
|
}, 35000);
|
||||||
|
|
||||||
it("counts queued package cancellations in run summary", async () => {
|
it("counts queued package cancellations in run summary", async () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
@ -7650,10 +7739,10 @@ describe("download manager", () => {
|
|||||||
await waitFor(() => fs.existsSync(flattenedPath), 12000);
|
await waitFor(() => fs.existsSync(flattenedPath), 12000);
|
||||||
|
|
||||||
expect(manager.getSnapshot().session.packages[packageId]?.status).toBe("completed");
|
expect(manager.getSnapshot().session.packages[packageId]?.status).toBe("completed");
|
||||||
expect(manager.getSnapshot().session.items[itemId]?.fullStatus).toBe("Entpackt - Done");
|
expect(manager.getSnapshot().session.items[itemId]?.fullStatus.startsWith("Entpackt - Done")).toBe(true);
|
||||||
expect(fs.existsSync(flattenedPath)).toBe(true);
|
expect(fs.existsSync(flattenedPath)).toBe(true);
|
||||||
expect(fs.existsSync(originalExtractedPath)).toBe(false);
|
expect(fs.existsSync(originalExtractedPath)).toBe(false);
|
||||||
});
|
}, 20000);
|
||||||
|
|
||||||
it("keeps existing MKV names and appends a suffix while flattening", async () => {
|
it("keeps existing MKV names and appends a suffix while flattening", async () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
@ -7773,7 +7862,88 @@ describe("download manager", () => {
|
|||||||
|
|
||||||
expect(fs.existsSync(flattenedPath)).toBe(true);
|
expect(fs.existsSync(flattenedPath)).toBe(true);
|
||||||
expect(fs.existsSync(extractDir)).toBe(false);
|
expect(fs.existsSync(extractDir)).toBe(false);
|
||||||
});
|
}, 20000);
|
||||||
|
|
||||||
|
it("cleans duplicate-skipped MKV source trees including leftover sample files", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
|
||||||
|
const packageName = "Flat-Duplicate-Cleanup";
|
||||||
|
const outputDir = path.join(root, "downloads", packageName);
|
||||||
|
const extractDir = path.join(root, "extract", packageName);
|
||||||
|
fs.mkdirSync(outputDir, { recursive: true });
|
||||||
|
|
||||||
|
const zip = new AdmZip();
|
||||||
|
zip.addFile("Season 1/Episode01.mkv", Buffer.from("video"));
|
||||||
|
zip.addFile("Season 1/sample.txt", Buffer.from("sample"));
|
||||||
|
const archivePath = path.join(outputDir, "episode.zip");
|
||||||
|
zip.writeZip(archivePath);
|
||||||
|
const archiveSize = fs.statSync(archivePath).size;
|
||||||
|
|
||||||
|
const session = emptySession();
|
||||||
|
const packageId = `${packageName}-pkg`;
|
||||||
|
const itemId = `${packageName}-item`;
|
||||||
|
const createdAt = Date.now() - 20_000;
|
||||||
|
session.packageOrder = [packageId];
|
||||||
|
session.packages[packageId] = {
|
||||||
|
id: packageId,
|
||||||
|
name: packageName,
|
||||||
|
outputDir,
|
||||||
|
extractDir,
|
||||||
|
status: "downloading",
|
||||||
|
itemIds: [itemId],
|
||||||
|
cancelled: false,
|
||||||
|
enabled: true,
|
||||||
|
createdAt,
|
||||||
|
updatedAt: createdAt
|
||||||
|
};
|
||||||
|
session.items[itemId] = {
|
||||||
|
id: itemId,
|
||||||
|
packageId,
|
||||||
|
url: "https://dummy/flat-duplicate-cleanup",
|
||||||
|
provider: "realdebrid",
|
||||||
|
status: "completed",
|
||||||
|
retries: 0,
|
||||||
|
speedBps: 0,
|
||||||
|
downloadedBytes: archiveSize,
|
||||||
|
totalBytes: archiveSize,
|
||||||
|
progressPercent: 100,
|
||||||
|
fileName: "episode.zip",
|
||||||
|
targetPath: archivePath,
|
||||||
|
resumable: true,
|
||||||
|
attempts: 1,
|
||||||
|
lastError: "",
|
||||||
|
fullStatus: "Fertig",
|
||||||
|
createdAt,
|
||||||
|
updatedAt: createdAt
|
||||||
|
};
|
||||||
|
|
||||||
|
const mkvLibraryDir = path.join(root, "mkv-library");
|
||||||
|
fs.mkdirSync(mkvLibraryDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(mkvLibraryDir, "Episode01.mkv"), Buffer.from("video"));
|
||||||
|
|
||||||
|
new DownloadManager(
|
||||||
|
{
|
||||||
|
...defaultSettings(),
|
||||||
|
token: "rd-token",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: path.join(root, "extract"),
|
||||||
|
autoExtract: true,
|
||||||
|
autoRename4sf4sj: false,
|
||||||
|
collectMkvToLibrary: true,
|
||||||
|
mkvLibraryDir,
|
||||||
|
enableIntegrityCheck: false,
|
||||||
|
cleanupMode: "none"
|
||||||
|
},
|
||||||
|
session,
|
||||||
|
createStoragePaths(path.join(root, "state"))
|
||||||
|
);
|
||||||
|
|
||||||
|
await waitFor(() => !fs.existsSync(extractDir), 12000);
|
||||||
|
|
||||||
|
expect(fs.existsSync(path.join(mkvLibraryDir, "Episode01.mkv"))).toBe(true);
|
||||||
|
expect(fs.existsSync(extractDir)).toBe(false);
|
||||||
|
}, 20000);
|
||||||
|
|
||||||
it("throws a controlled error for invalid queue import JSON", () => {
|
it("throws a controlled error for invalid queue import JSON", () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user