Release v1.4.14 with extraction performance optimization and bug fixes
- Add multi-threaded extraction via WinRAR -mt flag (uses all CPU cores) - Fix -idq flag suppressing progress output, replaced with -idc - Fix extraction timeout for multi-part archives (now calculates total size across all parts) - Raise extraction timeout cap from 40min to 2h for large archives (40GB+) - Add natural episode sorting (E1, E2, E10 instead of E1, E10, E2) - Add split archive support (.zip.001, .7z.001) with proper cleanup - Add write-stream drain timeout to prevent download freezes on backpressure - Fix regex global-state bug in progress percentage parsing - Optimize speed event pruning (every 1.5s instead of every chunk) - Add performance flag fallback for older WinRAR versions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Sucukdeluxe
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
6d8ead8598
commit
cc887eb8a1
@@ -2,7 +2,7 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import http from "node:http";
|
||||
import { once } from "node:events";
|
||||
import { EventEmitter, once } from "node:events";
|
||||
import AdmZip from "adm-zip";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { DownloadManager } from "../src/main/download-manager";
|
||||
@@ -726,6 +726,115 @@ describe("download manager", () => {
|
||||
}
|
||||
}, 35000);
|
||||
|
||||
it("recovers when write stream backpressure never drains", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
const binary = Buffer.alloc(180 * 1024, 19);
|
||||
const previousStallTimeout = process.env.RD_STALL_TIMEOUT_MS;
|
||||
process.env.RD_STALL_TIMEOUT_MS = "2200";
|
||||
let directCalls = 0;
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
if ((req.url || "") !== "/drain-stall") {
|
||||
res.statusCode = 404;
|
||||
res.end("not-found");
|
||||
return;
|
||||
}
|
||||
directCalls += 1;
|
||||
res.statusCode = 200;
|
||||
res.setHeader("Accept-Ranges", "bytes");
|
||||
res.setHeader("Content-Length", String(binary.length));
|
||||
res.end(binary);
|
||||
});
|
||||
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("server address unavailable");
|
||||
}
|
||||
const directUrl = `http://127.0.0.1:${address.port}/drain-stall`;
|
||||
|
||||
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||
if (url.includes("/unrestrict/link")) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
download: directUrl,
|
||||
filename: "drain-stall.bin",
|
||||
filesize: binary.length
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" }
|
||||
}
|
||||
);
|
||||
}
|
||||
return originalFetch(input, init);
|
||||
};
|
||||
|
||||
const originalCreateWriteStream = fs.createWriteStream;
|
||||
let writeStreamCalls = 0;
|
||||
const fsMutable = fs as unknown as { createWriteStream: typeof fs.createWriteStream };
|
||||
fsMutable.createWriteStream = ((...args: Parameters<typeof fs.createWriteStream>) => {
|
||||
writeStreamCalls += 1;
|
||||
if (writeStreamCalls === 1) {
|
||||
class HangingWriteStream extends EventEmitter {
|
||||
public closed = false;
|
||||
|
||||
public destroyed = false;
|
||||
|
||||
public write(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public end(): void {
|
||||
this.closed = true;
|
||||
this.emit("close");
|
||||
}
|
||||
}
|
||||
return new HangingWriteStream() as unknown as ReturnType<typeof fs.createWriteStream>;
|
||||
}
|
||||
return originalCreateWriteStream(...args);
|
||||
}) as typeof fs.createWriteStream;
|
||||
|
||||
try {
|
||||
const manager = new DownloadManager(
|
||||
{
|
||||
...defaultSettings(),
|
||||
token: "rd-token",
|
||||
outputDir: path.join(root, "downloads"),
|
||||
extractDir: path.join(root, "extract"),
|
||||
autoExtract: false,
|
||||
autoReconnect: false
|
||||
},
|
||||
emptySession(),
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
|
||||
manager.addPackages([{ name: "drain-stall", links: ["https://dummy/drain-stall"] }]);
|
||||
manager.start();
|
||||
await waitFor(() => !manager.getSnapshot().session.running, 40000);
|
||||
|
||||
const item = Object.values(manager.getSnapshot().session.items)[0];
|
||||
expect(item?.status).toBe("completed");
|
||||
expect(item?.retries).toBeGreaterThan(0);
|
||||
expect(directCalls).toBeGreaterThan(1);
|
||||
expect(fs.existsSync(item.targetPath)).toBe(true);
|
||||
expect(fs.statSync(item.targetPath).size).toBe(binary.length);
|
||||
} finally {
|
||||
fsMutable.createWriteStream = originalCreateWriteStream;
|
||||
if (previousStallTimeout === undefined) {
|
||||
delete process.env.RD_STALL_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.RD_STALL_TIMEOUT_MS = previousStallTimeout;
|
||||
}
|
||||
server.close();
|
||||
await once(server, "close");
|
||||
}
|
||||
}, 45000);
|
||||
|
||||
it("uses content-disposition filename when provider filename is opaque", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
|
||||
+86
-17
@@ -15,29 +15,30 @@ afterEach(() => {
|
||||
|
||||
describe("extractor", () => {
|
||||
it("maps external extractor args by conflict mode", () => {
|
||||
expect(buildExternalExtractArgs("WinRAR.exe", "archive.rar", "C:\\target", "overwrite")).toEqual([
|
||||
"x",
|
||||
"-o+",
|
||||
"-p-",
|
||||
"-y",
|
||||
"archive.rar",
|
||||
"C:\\target\\"
|
||||
]);
|
||||
expect(buildExternalExtractArgs("WinRAR.exe", "archive.rar", "C:\\target", "ask", "serienfans.org")).toEqual([
|
||||
"x",
|
||||
"-o-",
|
||||
"-pserienfans.org",
|
||||
"-y",
|
||||
"archive.rar",
|
||||
"C:\\target\\"
|
||||
]);
|
||||
const overwriteArgs = buildExternalExtractArgs("WinRAR.exe", "archive.rar", "C:\\target", "overwrite");
|
||||
expect(overwriteArgs.slice(0, 4)).toEqual(["x", "-o+", "-p-", "-y"]);
|
||||
expect(overwriteArgs).toContain("-idc");
|
||||
expect(overwriteArgs.some((value) => /^-mt\d+$/i.test(value))).toBe(true);
|
||||
expect(overwriteArgs[overwriteArgs.length - 2]).toBe("archive.rar");
|
||||
expect(overwriteArgs[overwriteArgs.length - 1]).toBe("C:\\target\\");
|
||||
|
||||
const askArgs = buildExternalExtractArgs("WinRAR.exe", "archive.rar", "C:\\target", "ask", "serienfans.org");
|
||||
expect(askArgs.slice(0, 4)).toEqual(["x", "-o-", "-pserienfans.org", "-y"]);
|
||||
expect(askArgs).toContain("-idc");
|
||||
expect(askArgs.some((value) => /^-mt\d+$/i.test(value))).toBe(true);
|
||||
expect(askArgs[askArgs.length - 2]).toBe("archive.rar");
|
||||
expect(askArgs[askArgs.length - 1]).toBe("C:\\target\\");
|
||||
|
||||
const compatibilityArgs = buildExternalExtractArgs("WinRAR.exe", "archive.rar", "C:\\target", "overwrite", "", false);
|
||||
expect(compatibilityArgs).not.toContain("-idc");
|
||||
expect(compatibilityArgs.some((value) => /^-mt\d+$/i.test(value))).toBe(false);
|
||||
|
||||
const unrarRename = buildExternalExtractArgs("unrar", "archive.rar", "C:\\target", "rename");
|
||||
expect(unrarRename[0]).toBe("x");
|
||||
expect(unrarRename[1]).toBe("-or");
|
||||
expect(unrarRename[2]).toBe("-p-");
|
||||
expect(unrarRename[3]).toBe("-y");
|
||||
expect(unrarRename[4]).toBe("archive.rar");
|
||||
expect(unrarRename[unrarRename.length - 2]).toBe("archive.rar");
|
||||
});
|
||||
|
||||
it("deletes only successfully extracted archives", async () => {
|
||||
@@ -94,6 +95,74 @@ describe("extractor", () => {
|
||||
expect(targets.has(other)).toBe(false);
|
||||
});
|
||||
|
||||
it("collects split 7z companion parts for cleanup", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-extract-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
|
||||
const part1 = path.join(packageDir, "release.7z.001");
|
||||
const part2 = path.join(packageDir, "release.7z.002");
|
||||
const part3 = path.join(packageDir, "release.7z.003");
|
||||
const other = path.join(packageDir, "other.7z.001");
|
||||
|
||||
fs.writeFileSync(part1, "a", "utf8");
|
||||
fs.writeFileSync(part2, "b", "utf8");
|
||||
fs.writeFileSync(part3, "c", "utf8");
|
||||
fs.writeFileSync(other, "x", "utf8");
|
||||
|
||||
const targets = new Set(collectArchiveCleanupTargets(part1));
|
||||
expect(targets.has(part1)).toBe(true);
|
||||
expect(targets.has(part2)).toBe(true);
|
||||
expect(targets.has(part3)).toBe(true);
|
||||
expect(targets.has(other)).toBe(false);
|
||||
});
|
||||
|
||||
it("extracts archives in natural episode order", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-extract-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
const targetDir = path.join(root, "out");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
|
||||
const zip10 = new AdmZip();
|
||||
zip10.addFile("e10.txt", Buffer.from("10"));
|
||||
zip10.writeZip(path.join(packageDir, "Show.S01E10.zip"));
|
||||
|
||||
const zip2 = new AdmZip();
|
||||
zip2.addFile("e02.txt", Buffer.from("02"));
|
||||
zip2.writeZip(path.join(packageDir, "Show.S01E02.zip"));
|
||||
|
||||
const zip1 = new AdmZip();
|
||||
zip1.addFile("e01.txt", Buffer.from("01"));
|
||||
zip1.writeZip(path.join(packageDir, "Show.S01E01.zip"));
|
||||
|
||||
const seenOrder: string[] = [];
|
||||
await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false,
|
||||
onProgress: (update) => {
|
||||
if (update.phase !== "extracting" || !update.archiveName) {
|
||||
return;
|
||||
}
|
||||
if (seenOrder[seenOrder.length - 1] === update.archiveName) {
|
||||
return;
|
||||
}
|
||||
seenOrder.push(update.archiveName);
|
||||
}
|
||||
});
|
||||
|
||||
expect(seenOrder.slice(0, 3)).toEqual([
|
||||
"Show.S01E01.zip",
|
||||
"Show.S01E02.zip",
|
||||
"Show.S01E10.zip"
|
||||
]);
|
||||
});
|
||||
|
||||
it("deletes split zip companion parts when cleanup is enabled", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-extract-"));
|
||||
tempDirs.push(root);
|
||||
|
||||
Reference in New Issue
Block a user