fix: scope extracted output counts to package provenance

Snapshot non-archive outputs around each main, hybrid, and nested extraction and attribute only files created or changed by that package. Persist deduplicated hashed provenance keys so shared extract directories and restarts cannot count unrelated package outputs.

Cover startup extraction in a shared target containing an existing foreign file and validate persisted provenance normalization.
This commit is contained in:
Sucukdeluxe
2026-08-22 12:27:59 +02:00
parent 91b9596d6f
commit a266610c31
5 changed files with 49 additions and 19 deletions
+30 -8
View File
@@ -4363,12 +4363,12 @@ export class DownloadManager extends EventEmitter {
return false; return false;
} }
private async countPackageOutputFiles(rootDir: string): Promise<number> { private async snapshotPackageOutputFiles(rootDir: string): Promise<Map<string, string>> {
const snapshot = new Map<string, string>();
if (!rootDir) { if (!rootDir) {
return 0; return snapshot;
} }
const stack = [rootDir]; const stack = [rootDir];
let count = 0;
while (stack.length > 0) { while (stack.length > 0) {
const current = stack.pop() as string; const current = stack.pop() as string;
let entries: fs.Dirent[] = []; let entries: fs.Dirent[] = [];
@@ -4385,11 +4385,29 @@ export class DownloadManager extends EventEmitter {
if (entry.isDirectory()) { if (entry.isDirectory()) {
stack.push(fullPath); stack.push(fullPath);
} else if (entry.isFile() && !isArchiveLikePath(fullPath) && !isIgnorableEmptyDirFileName(entry.name)) { } else if (entry.isFile() && !isArchiveLikePath(fullPath) && !isIgnorableEmptyDirFileName(entry.name)) {
count += 1; try {
const stat = await fs.promises.stat(fullPath);
const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/");
const key = process.platform === "win32" ? relativePath.toLowerCase() : relativePath;
snapshot.set(key, `${stat.size}:${stat.mtimeMs}`);
} catch {
} }
} }
} }
return count; }
return snapshot;
}
private async recordPackageOutputFiles(pkg: PackageEntry, before: ReadonlyMap<string, string>): Promise<void> {
const after = await this.snapshotPackageOutputFiles(pkg.extractDir);
const provenance = new Set(pkg.outputProvenance || []);
for (const [relativePath, signature] of after) {
if (before.get(relativePath) !== signature) {
provenance.add(createHash("sha256").update(relativePath).digest("hex"));
}
}
pkg.outputProvenance = [...provenance];
pkg.outputCount = Math.max(pkg.outputCount || 0, provenance.size);
} }
private async removeEmptyDirectoryTree(rootDir: string): Promise<number> { private async removeEmptyDirectoryTree(rootDir: string): Promise<number> {
@@ -12244,6 +12262,7 @@ export class DownloadManager extends EventEmitter {
pkg.archiveOperations = []; pkg.archiveOperations = [];
pkg.remuxOperations = []; pkg.remuxOperations = [];
pkg.outputCount = 0; pkg.outputCount = 0;
pkg.outputProvenance = [];
pkg.cleanupErrorCategory = ""; pkg.cleanupErrorCategory = "";
} }
return next; return next;
@@ -13225,6 +13244,7 @@ export class DownloadManager extends EventEmitter {
return 0; return 0;
} }
const packageOutputBefore = await this.snapshotPackageOutputFiles(pkg.extractDir);
const result = await extractPackageArchives({ const result = await extractPackageArchives({
packageDir: pkg.outputDir, packageDir: pkg.outputDir,
targetDir: pkg.extractDir, targetDir: pkg.extractDir,
@@ -13384,7 +13404,7 @@ export class DownloadManager extends EventEmitter {
} }
} }
}); });
pkg.outputCount = Math.max(pkg.outputCount || 0, await this.countPackageOutputFiles(pkg.extractDir)); await this.recordPackageOutputFiles(pkg, packageOutputBefore);
logger.info(`Hybrid-Extract Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}`); logger.info(`Hybrid-Extract Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}`);
this.logPackageForPackage(pkg, "INFO", "Hybrid-Extract abgeschlossen", { this.logPackageForPackage(pkg, "INFO", "Hybrid-Extract abgeschlossen", {
@@ -13801,6 +13821,7 @@ export class DownloadManager extends EventEmitter {
entry.updatedAt = pendingAt; entry.updatedAt = pendingAt;
} }
this.emitState(); this.emitState();
const packageOutputBefore = await this.snapshotPackageOutputFiles(pkg.extractDir);
const result = await extractPackageArchives({ const result = await extractPackageArchives({
packageDir: pkg.outputDir, packageDir: pkg.outputDir,
targetDir: pkg.extractDir, targetDir: pkg.extractDir,
@@ -13943,7 +13964,7 @@ export class DownloadManager extends EventEmitter {
emitExtractStatus(overallLabel); emitExtractStatus(overallLabel);
} }
}); });
pkg.outputCount = Math.max(pkg.outputCount || 0, await this.countPackageOutputFiles(pkg.extractDir)); await this.recordPackageOutputFiles(pkg, packageOutputBefore);
logger.info(`Post-Processing Entpacken Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}, lastError=${result.lastError || ""}`); logger.info(`Post-Processing Entpacken Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}, lastError=${result.lastError || ""}`);
this.logPackageForPackage(pkg, "INFO", "Post-Processing Entpacken Ende", { this.logPackageForPackage(pkg, "INFO", "Post-Processing Entpacken Ende", {
extracted: result.extracted, extracted: result.extracted,
@@ -14150,6 +14171,7 @@ export class DownloadManager extends EventEmitter {
}); });
const nestedFailureCategories = new Map<string, string>(); const nestedFailureCategories = new Map<string, string>();
const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[]; const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[];
const packageOutputBefore = await this.snapshotPackageOutputFiles(pkg.extractDir);
const nestedResult = await extractPackageArchives({ const nestedResult = await extractPackageArchives({
packageDir: pkg.extractDir, packageDir: pkg.extractDir,
targetDir: pkg.extractDir, targetDir: pkg.extractDir,
@@ -14177,7 +14199,7 @@ export class DownloadManager extends EventEmitter {
} }
}); });
throwIfAborted(); throwIfAborted();
pkg.outputCount = Math.max(pkg.outputCount || 0, await this.countPackageOutputFiles(pkg.extractDir)); await this.recordPackageOutputFiles(pkg, packageOutputBefore);
extractedCount += nestedResult.extracted; extractedCount += nestedResult.extracted;
logger.info(`Deferred Nested-Extraction Ende: extracted=${nestedResult.extracted}, failed=${nestedResult.failed}`); logger.info(`Deferred Nested-Extraction Ende: extracted=${nestedResult.extracted}, failed=${nestedResult.failed}`);
this.logPackageForPackage(pkg, "INFO", "Deferred Nested-Extraction Ende", { this.logPackageForPackage(pkg, "INFO", "Deferred Nested-Extraction Ende", {
+3
View File
@@ -1007,6 +1007,9 @@ export function normalizeLoadedSession(raw: unknown): SessionState {
archiveOperations: normalizeArchiveOperations(pkg.archiveOperations), archiveOperations: normalizeArchiveOperations(pkg.archiveOperations),
remuxOperations: normalizeRemuxOperations(pkg.remuxOperations), remuxOperations: normalizeRemuxOperations(pkg.remuxOperations),
outputCount: clampNumber(pkg.outputCount, 0, 0, 1_000_000), outputCount: clampNumber(pkg.outputCount, 0, 0, 1_000_000),
outputProvenance: Array.isArray(pkg.outputProvenance)
? [...new Set(pkg.outputProvenance.map((value) => asText(value).toLowerCase()).filter((value) => /^[a-f0-9]{64}$/.test(value)))].slice(0, 1_000_000)
: [],
cleanupErrorCategory: asText(pkg.cleanupErrorCategory), cleanupErrorCategory: asText(pkg.cleanupErrorCategory),
resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER), resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER),
createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER), createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER),
+1
View File
@@ -588,6 +588,7 @@ export interface PackageEntry {
archiveOperations?: ArchiveOperationMetric[]; archiveOperations?: ArchiveOperationMetric[];
remuxOperations?: RemuxOperationMetric[]; remuxOperations?: RemuxOperationMetric[];
outputCount?: number; outputCount?: number;
outputProvenance?: string[];
cleanupErrorCategory?: string; cleanupErrorCategory?: string;
resultGeneration?: number; resultGeneration?: number;
createdAt: number; createdAt: number;
+4 -2
View File
@@ -11466,8 +11466,10 @@ describe("download manager", () => {
tempDirs.push(root); tempDirs.push(root);
const outputDir = path.join(root, "downloads", "recovery"); const outputDir = path.join(root, "downloads", "recovery");
const extractDir = path.join(root, "extract", "recovery"); const extractDir = path.join(root, "extract");
fs.mkdirSync(outputDir, { recursive: true }); fs.mkdirSync(outputDir, { recursive: true });
fs.mkdirSync(extractDir, { recursive: true });
fs.writeFileSync(path.join(extractDir, "foreign-package.txt"), "foreign");
const zip = new AdmZip(); const zip = new AdmZip();
zip.addFile("episode.txt", Buffer.from("ok")); zip.addFile("episode.txt", Buffer.from("ok"));
@@ -11518,7 +11520,7 @@ describe("download manager", () => {
token: "rd-token", token: "rd-token",
outputDir: path.join(root, "downloads"), outputDir: path.join(root, "downloads"),
extractDir: path.join(root, "extract"), extractDir: path.join(root, "extract"),
createExtractSubfolder: true, createExtractSubfolder: false,
autoExtract: true, autoExtract: true,
enableIntegrityCheck: false, enableIntegrityCheck: false,
cleanupMode: "none" cleanupMode: "none"
+2
View File
@@ -1200,6 +1200,7 @@ describe("settings storage", () => {
}], }],
remuxOperations: [], remuxOperations: [],
outputCount: 1, outputCount: 1,
outputProvenance: ["a".repeat(64), "invalid", "a".repeat(64)],
cleanupErrorCategory: "", cleanupErrorCategory: "",
createdAt: 1_000, createdAt: 1_000,
updatedAt: 21_000 updatedAt: 21_000
@@ -1226,6 +1227,7 @@ describe("settings storage", () => {
archiveOperations: [expect.objectContaining({ id: "archive-1", durationMs: 4_000 })], archiveOperations: [expect.objectContaining({ id: "archive-1", durationMs: 4_000 })],
remuxOperations: [], remuxOperations: [],
outputCount: 1, outputCount: 1,
outputProvenance: ["a".repeat(64)],
cleanupErrorCategory: "" cleanupErrorCategory: ""
})); }));
}); });