feat: track direct extraction outputs
Replace package staging assumptions with validated per-generation output records for internal ZIP and both JVM backends. Persist resume v2 using relative archive identities, complete member fingerprints, and concrete verified outputs so basename collisions, changed members, missing outputs, and unknown versions fail closed.
This commit is contained in:
Sucukdeluxe
+63
-10
@@ -47,23 +47,76 @@ describe.skipIf(!hasJavaRuntime() || !hasJvmExtractorRuntime())("extractor jvm b
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
|
||||
const zipPath = path.join(packageDir, "release.zip");
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("episode.txt", Buffer.from("ok"));
|
||||
zip.writeZip(zipPath);
|
||||
|
||||
const result = await extractPackageArchives({
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("episode.txt", Buffer.from("ok"));
|
||||
zip.writeZip(zipPath);
|
||||
const events: import("../src/main/extractor").ExtractOutputEvent[] = [];
|
||||
|
||||
const result = await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false
|
||||
});
|
||||
removeSamples: false,
|
||||
onOutput: (event) => events.push(event)
|
||||
});
|
||||
|
||||
expect(result.extracted).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(fs.existsSync(path.join(targetDir, "episode.txt"))).toBe(true);
|
||||
});
|
||||
expect(result.failed).toBe(0);
|
||||
expect(fs.existsSync(path.join(targetDir, "episode.txt"))).toBe(true);
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
version: 1,
|
||||
archivePath: path.resolve(zipPath),
|
||||
entryPath: "episode.txt",
|
||||
outputPath: path.join(targetDir, "episode.txt"),
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
})
|
||||
]);
|
||||
expect(result.outputFiles).toEqual([path.join(targetDir, "episode.txt")]);
|
||||
});
|
||||
|
||||
it.each(["7zjbinding", "zip4j"])("emits versioned Base64 output lines from %s", (backend) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), `rd-jvm-output-${backend}-`));
|
||||
tempDirs.push(root);
|
||||
const targetDir = path.join(root, "out");
|
||||
const zipPath = path.join(root, "release.zip");
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("folder/episode.txt", Buffer.from("ok"));
|
||||
zip.writeZip(zipPath);
|
||||
const runtimeRoot = path.join(process.cwd(), "resources", "extractor-jvm");
|
||||
const classPath = [
|
||||
path.join(runtimeRoot, "classes"),
|
||||
path.join(runtimeRoot, "lib", "sevenzipjbinding.jar"),
|
||||
path.join(runtimeRoot, "lib", "sevenzipjbinding-all-platforms.jar"),
|
||||
path.join(runtimeRoot, "lib", "zip4j.jar")
|
||||
].join(path.delimiter);
|
||||
|
||||
const run = spawnSync("java", [
|
||||
"-cp",
|
||||
classPath,
|
||||
"com.sucukdeluxe.extractor.JBindExtractorMain",
|
||||
"--archive",
|
||||
zipPath,
|
||||
"--target",
|
||||
targetDir,
|
||||
"--conflict",
|
||||
"overwrite",
|
||||
"--backend",
|
||||
backend
|
||||
], { encoding: "utf8" });
|
||||
|
||||
expect(run.status).toBe(0);
|
||||
const outputLine = String(run.stdout).split(/\r?\n/).find((line) => line.startsWith("RD_OUTPUT "));
|
||||
expect(outputLine).toBeTruthy();
|
||||
const fields = String(outputLine).split(" ");
|
||||
expect(fields.slice(0, 5)).toEqual(["RD_OUTPUT", "1", "complete", "written", fields[4]]);
|
||||
expect(Buffer.from(fields[4], "base64").toString("utf8")).toBe(path.resolve(zipPath));
|
||||
expect(Buffer.from(fields[5], "base64").toString("utf8")).toBe("folder/episode.txt");
|
||||
expect(Buffer.from(fields[6], "base64").toString("utf8")).toBe(path.join(targetDir, "folder", "episode.txt"));
|
||||
});
|
||||
|
||||
it("emits progress callbacks with archiveName and percent", async () => {
|
||||
process.env.RD_EXTRACT_BACKEND = "jvm";
|
||||
|
||||
+191
-7
@@ -16,7 +16,7 @@ import {
|
||||
selectZipFallbackError,
|
||||
shouldSerialRetryParallelFailures,
|
||||
findArchiveCandidates,
|
||||
orderExtractorCandidatesForArchive,
|
||||
orderExtractorCandidatesForArchive,
|
||||
resolveExtractorBackendModeForArchive,
|
||||
resolveExtractorBackendMode,
|
||||
shouldFallbackLegacyRarToJvm,
|
||||
@@ -610,7 +610,7 @@ describe("extractor", () => {
|
||||
expect(selectZipFallbackError(internalError, externalError)).toBe(externalError);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform !== "win32")("matches resume-state archive names case-insensitively on Windows", async () => {
|
||||
it.skipIf(process.platform !== "win32")("invalidates legacy basename-only resume state on Windows", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-extract-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
@@ -630,8 +630,8 @@ describe("extractor", () => {
|
||||
removeSamples: false
|
||||
});
|
||||
|
||||
expect(result.extracted).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.extracted).toBe(0);
|
||||
expect(result.failed).toBe(1);
|
||||
});
|
||||
|
||||
describe("disk space check", () => {
|
||||
@@ -1256,7 +1256,7 @@ describe("extractor", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("orderExtractorCandidatesForArchive", () => {
|
||||
describe("orderExtractorCandidatesForArchive", () => {
|
||||
it("prefers RAR-native CLIs over 7-Zip for rar archives", () => {
|
||||
const ordered = orderExtractorCandidatesForArchive(
|
||||
["7z.exe", "Rar.exe", "UnRAR.exe", "WinRAR.exe"],
|
||||
@@ -1283,5 +1283,189 @@ describe("extractor", () => {
|
||||
expect(ordered[0]).toBe("WinRAR.exe");
|
||||
expect(ordered[1]).toBe("UnRAR.exe");
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("direct output scope", () => {
|
||||
it.each([
|
||||
["overwrite", "new", "overwritten", ["episode.mkv"]],
|
||||
["ask", "old", "skipped", []],
|
||||
["rename", "old", "renamed", ["episode (1).mkv"]]
|
||||
] as const)("emits exact internal ZIP outputs for %s conflicts", async (conflictMode, originalContent, disposition, outputNames) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), `rd-direct-${conflictMode}-`));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
const targetDir = path.join(root, "out");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(targetDir, "episode.mkv"), "old");
|
||||
const archivePath = path.join(packageDir, "release.zip");
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("episode.mkv", Buffer.from("new"));
|
||||
zip.writeZip(archivePath);
|
||||
const events: import("../src/main/extractor").ExtractOutputEvent[] = [];
|
||||
|
||||
const result = await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode,
|
||||
removeLinks: false,
|
||||
removeSamples: false,
|
||||
onOutput: (event) => events.push(event)
|
||||
});
|
||||
|
||||
expect(fs.readFileSync(path.join(targetDir, "episode.mkv"), "utf8")).toBe(originalContent);
|
||||
expect(events).toEqual([expect.objectContaining({
|
||||
version: 1,
|
||||
archivePath: path.resolve(archivePath),
|
||||
entryPath: "episode.mkv",
|
||||
outputPath: path.join(targetDir, outputNames[0] || "episode.mkv"),
|
||||
state: "complete",
|
||||
disposition
|
||||
})]);
|
||||
expect(result.outputFiles.map((filePath) => path.basename(filePath))).toEqual([...outputNames]);
|
||||
});
|
||||
|
||||
it("extracts only nested archives produced by the package in a shared root", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-direct-nested-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
const targetDir = path.join(root, "shared");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
const foreign = new AdmZip();
|
||||
foreign.addFile("foreign.txt", Buffer.from("foreign"));
|
||||
foreign.writeZip(path.join(targetDir, "foreign.zip"));
|
||||
const nested = new AdmZip();
|
||||
nested.addFile("owned.txt", Buffer.from("owned"));
|
||||
const outer = new AdmZip();
|
||||
outer.addFile("owned.zip", nested.toBuffer());
|
||||
outer.writeZip(path.join(packageDir, "outer.zip"));
|
||||
|
||||
const result = await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false
|
||||
});
|
||||
|
||||
expect(result.extracted).toBe(2);
|
||||
expect(fs.existsSync(path.join(targetDir, "owned.txt"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(targetDir, "foreign.txt"))).toBe(false);
|
||||
});
|
||||
|
||||
it("resumes same-basename archives by relative path and invalidates changed multipart fingerprints", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-resume-v2-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
const targetDir = path.join(root, "out");
|
||||
const firstDir = path.join(packageDir, "first");
|
||||
const secondDir = path.join(packageDir, "second");
|
||||
fs.mkdirSync(firstDir, { recursive: true });
|
||||
fs.mkdirSync(secondDir, { recursive: true });
|
||||
const firstArchive = path.join(firstDir, "release.zip");
|
||||
const secondArchive = path.join(secondDir, "release.zip");
|
||||
const companionPath = path.join(firstDir, "release.sfv");
|
||||
const firstZip = new AdmZip();
|
||||
firstZip.addFile("first.txt", Buffer.from("first"));
|
||||
firstZip.writeZip(firstArchive);
|
||||
fs.writeFileSync(companionPath, "part-a");
|
||||
const secondZip = new AdmZip();
|
||||
secondZip.addFile("second.txt", Buffer.from("second"));
|
||||
secondZip.writeZip(secondArchive);
|
||||
|
||||
const firstResult = await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false,
|
||||
onlyArchives: new Set([path.resolve(firstArchive).toLowerCase()])
|
||||
});
|
||||
expect(firstResult.extracted).toBe(1);
|
||||
expect(firstResult.failed).toBe(0);
|
||||
expect(fs.existsSync(path.join(packageDir, ".rd_extract_progress.json"))).toBe(true);
|
||||
fs.writeFileSync(companionPath, "part-b-changed");
|
||||
const emittedArchives: string[] = [];
|
||||
const result = await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false,
|
||||
onOutput: (event) => emittedArchives.push(event.archivePath)
|
||||
});
|
||||
|
||||
expect(result.extracted).toBe(2);
|
||||
expect(emittedArchives).toContain(path.resolve(firstArchive));
|
||||
expect(emittedArchives).toContain(path.resolve(secondArchive));
|
||||
expect(fs.existsSync(path.join(targetDir, "second.txt"))).toBe(true);
|
||||
});
|
||||
|
||||
it("fails closed for an unknown resume-state version", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-resume-unknown-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
const targetDir = path.join(root, "out");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(packageDir, "broken.zip"), "not-a-zip");
|
||||
fs.writeFileSync(path.join(targetDir, "foreign.txt"), "foreign");
|
||||
fs.writeFileSync(path.join(packageDir, ".rd_extract_progress.json"), JSON.stringify({
|
||||
version: 99,
|
||||
completedArchives: ["broken.zip"]
|
||||
}));
|
||||
|
||||
const result = await extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false
|
||||
});
|
||||
|
||||
expect(result.extracted).toBe(0);
|
||||
expect(result.failed).toBe(1);
|
||||
});
|
||||
|
||||
it("retains concrete completed outputs when a later entry aborts", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-abort-"));
|
||||
tempDirs.push(root);
|
||||
const packageDir = path.join(root, "pkg");
|
||||
const targetDir = path.join(root, "out");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
const archivePath = path.join(packageDir, "release.zip");
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("first.txt", Buffer.from("first"));
|
||||
zip.addFile("second.txt", Buffer.from("second"));
|
||||
zip.writeZip(archivePath);
|
||||
const controller = new AbortController();
|
||||
const events: import("../src/main/extractor").ExtractOutputEvent[] = [];
|
||||
|
||||
await expect(extractPackageArchives({
|
||||
packageDir,
|
||||
targetDir,
|
||||
cleanupMode: "none",
|
||||
conflictMode: "overwrite",
|
||||
removeLinks: false,
|
||||
removeSamples: false,
|
||||
signal: controller.signal,
|
||||
onOutput: (event) => {
|
||||
events.push(event);
|
||||
controller.abort();
|
||||
}
|
||||
})).rejects.toThrow("aborted:extract");
|
||||
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toEqual(expect.objectContaining({ state: "complete", outputPath: path.join(targetDir, "first.txt") }));
|
||||
expect(fs.existsSync(path.join(targetDir, "first.txt"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(targetDir, "second.txt"))).toBe(false);
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { PackageOutputScope } from "../src/main/package-output-scope";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function createRoot(): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-scope-"));
|
||||
tempDirs.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const directory of tempDirs.splice(0)) {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("PackageOutputScope", () => {
|
||||
it("normalizes and deduplicates complete Windows paths case-insensitively", () => {
|
||||
const root = createRoot();
|
||||
const outputPath = path.join(root, "Season", "Episode.mkv");
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, "video");
|
||||
const scope = new PackageOutputScope([root]);
|
||||
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive-a.rar"),
|
||||
entryPath: "Season/Episode.mkv",
|
||||
outputPath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive-b.rar"),
|
||||
entryPath: "season/episode.mkv",
|
||||
outputPath: outputPath.toUpperCase(),
|
||||
state: "complete",
|
||||
disposition: "overwritten"
|
||||
});
|
||||
|
||||
expect(scope.completeFiles()).toEqual([path.resolve(outputPath)]);
|
||||
expect(scope.partialFiles()).toEqual([]);
|
||||
expect(scope.records()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"../foreign.mkv",
|
||||
"folder/../../foreign.mkv",
|
||||
"/absolute.mkv",
|
||||
"C:\\absolute.mkv"
|
||||
])("rejects unsafe archive entry path %s", (entryPath) => {
|
||||
const root = createRoot();
|
||||
const outputPath = path.join(root, "safe.mkv");
|
||||
fs.writeFileSync(outputPath, "video");
|
||||
const scope = new PackageOutputScope([root]);
|
||||
|
||||
expect(() => scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive.rar"),
|
||||
entryPath,
|
||||
outputPath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
})).toThrow(/Ausgabepfad|entry/i);
|
||||
});
|
||||
|
||||
it("rejects outputs outside authorized roots and unknown event versions", () => {
|
||||
const root = createRoot();
|
||||
const foreignRoot = createRoot();
|
||||
const foreignPath = path.join(foreignRoot, "foreign.mkv");
|
||||
fs.writeFileSync(foreignPath, "foreign");
|
||||
const scope = new PackageOutputScope([root]);
|
||||
const event = {
|
||||
version: 1 as const,
|
||||
archivePath: path.join(root, "archive.rar"),
|
||||
entryPath: "foreign.mkv",
|
||||
outputPath: foreignPath,
|
||||
state: "complete" as const,
|
||||
disposition: "written" as const
|
||||
};
|
||||
|
||||
expect(() => scope.add(event)).toThrow(/autorisiert/i);
|
||||
expect(() => scope.add({ ...event, outputPath: path.join(root, "safe.mkv"), version: 2 as 1 })).toThrow(/Version/i);
|
||||
});
|
||||
|
||||
it("rejects symlinked parent chains and keeps complete and partial outputs separate", () => {
|
||||
const root = createRoot();
|
||||
const foreignRoot = createRoot();
|
||||
const completePath = path.join(root, "complete.mkv");
|
||||
const partialPath = path.join(root, "partial.mkv");
|
||||
fs.writeFileSync(completePath, "complete");
|
||||
fs.writeFileSync(partialPath, "partial");
|
||||
const scope = new PackageOutputScope([root]);
|
||||
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive.rar"),
|
||||
entryPath: "complete.mkv",
|
||||
outputPath: completePath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive.rar"),
|
||||
entryPath: "partial.mkv",
|
||||
outputPath: partialPath,
|
||||
state: "partial",
|
||||
disposition: "written"
|
||||
});
|
||||
|
||||
expect(scope.completeFiles()).toEqual([path.resolve(completePath)]);
|
||||
expect(scope.partialFiles()).toEqual([path.resolve(partialPath)]);
|
||||
|
||||
const link = path.join(root, "linked");
|
||||
try {
|
||||
fs.symlinkSync(foreignRoot, link, "junction");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const linkedOutput = path.join(link, "linked.mkv");
|
||||
fs.writeFileSync(path.join(foreignRoot, "linked.mkv"), "foreign");
|
||||
expect(() => scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive.rar"),
|
||||
entryPath: "linked/linked.mkv",
|
||||
outputPath: linkedOutput,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
})).toThrow(/symbol|reparse/i);
|
||||
});
|
||||
|
||||
it("updates ownership after rename and removal without enumerating the root", () => {
|
||||
const root = createRoot();
|
||||
const sourcePath = path.join(root, "source.mkv");
|
||||
const targetPath = path.join(root, "renamed.mkv");
|
||||
fs.writeFileSync(sourcePath, "video");
|
||||
const scope = new PackageOutputScope([root]);
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(root, "archive.rar"),
|
||||
entryPath: "source.mkv",
|
||||
outputPath: sourcePath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
fs.renameSync(sourcePath, targetPath);
|
||||
|
||||
scope.replacePath(sourcePath, targetPath);
|
||||
expect(scope.completeFiles()).toEqual([path.resolve(targetPath)]);
|
||||
expect(scope.removePath(targetPath)).toBe(true);
|
||||
expect(scope.completeFiles()).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user