fix: scope extracted output counts to package provenance
Snapshot non-archive outputs around each main, hybrid, and nested extraction and attribute only files created or changed by that package. Persist deduplicated hashed provenance keys so shared extract directories and restarts cannot count unrelated package outputs. Cover startup extraction in a shared target containing an existing foreign file and validate persisted provenance normalization.
This commit is contained in:
@@ -4363,12 +4363,12 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async countPackageOutputFiles(rootDir: string): Promise<number> {
|
private async snapshotPackageOutputFiles(rootDir: string): Promise<Map<string, string>> {
|
||||||
|
const snapshot = new Map<string, string>();
|
||||||
if (!rootDir) {
|
if (!rootDir) {
|
||||||
return 0;
|
return snapshot;
|
||||||
}
|
}
|
||||||
const stack = [rootDir];
|
const stack = [rootDir];
|
||||||
let count = 0;
|
|
||||||
while (stack.length > 0) {
|
while (stack.length > 0) {
|
||||||
const current = stack.pop() as string;
|
const current = stack.pop() as string;
|
||||||
let entries: fs.Dirent[] = [];
|
let entries: fs.Dirent[] = [];
|
||||||
@@ -4385,11 +4385,29 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (entry.isDirectory()) {
|
if (entry.isDirectory()) {
|
||||||
stack.push(fullPath);
|
stack.push(fullPath);
|
||||||
} else if (entry.isFile() && !isArchiveLikePath(fullPath) && !isIgnorableEmptyDirFileName(entry.name)) {
|
} else if (entry.isFile() && !isArchiveLikePath(fullPath) && !isIgnorableEmptyDirFileName(entry.name)) {
|
||||||
count += 1;
|
try {
|
||||||
|
const stat = await fs.promises.stat(fullPath);
|
||||||
|
const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/");
|
||||||
|
const key = process.platform === "win32" ? relativePath.toLowerCase() : relativePath;
|
||||||
|
snapshot.set(key, `${stat.size}:${stat.mtimeMs}`);
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return count;
|
return snapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async recordPackageOutputFiles(pkg: PackageEntry, before: ReadonlyMap<string, string>): Promise<void> {
|
||||||
|
const after = await this.snapshotPackageOutputFiles(pkg.extractDir);
|
||||||
|
const provenance = new Set(pkg.outputProvenance || []);
|
||||||
|
for (const [relativePath, signature] of after) {
|
||||||
|
if (before.get(relativePath) !== signature) {
|
||||||
|
provenance.add(createHash("sha256").update(relativePath).digest("hex"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pkg.outputProvenance = [...provenance];
|
||||||
|
pkg.outputCount = Math.max(pkg.outputCount || 0, provenance.size);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async removeEmptyDirectoryTree(rootDir: string): Promise<number> {
|
private async removeEmptyDirectoryTree(rootDir: string): Promise<number> {
|
||||||
@@ -12244,6 +12262,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
pkg.archiveOperations = [];
|
pkg.archiveOperations = [];
|
||||||
pkg.remuxOperations = [];
|
pkg.remuxOperations = [];
|
||||||
pkg.outputCount = 0;
|
pkg.outputCount = 0;
|
||||||
|
pkg.outputProvenance = [];
|
||||||
pkg.cleanupErrorCategory = "";
|
pkg.cleanupErrorCategory = "";
|
||||||
}
|
}
|
||||||
return next;
|
return next;
|
||||||
@@ -13221,11 +13240,12 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await this.waitForCompletedArchiveFilesToSettle(pkg, hybridItems, signal, "hybrid");
|
await this.waitForCompletedArchiveFilesToSettle(pkg, hybridItems, signal, "hybrid");
|
||||||
if (signal?.aborted) {
|
if (signal?.aborted) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await extractPackageArchives({
|
const packageOutputBefore = await this.snapshotPackageOutputFiles(pkg.extractDir);
|
||||||
|
const result = await extractPackageArchives({
|
||||||
packageDir: pkg.outputDir,
|
packageDir: pkg.outputDir,
|
||||||
targetDir: pkg.extractDir,
|
targetDir: pkg.extractDir,
|
||||||
cleanupMode: this.settings.cleanupMode,
|
cleanupMode: this.settings.cleanupMode,
|
||||||
@@ -13384,7 +13404,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
pkg.outputCount = Math.max(pkg.outputCount || 0, await this.countPackageOutputFiles(pkg.extractDir));
|
await this.recordPackageOutputFiles(pkg, packageOutputBefore);
|
||||||
|
|
||||||
logger.info(`Hybrid-Extract Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}`);
|
logger.info(`Hybrid-Extract Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}`);
|
||||||
this.logPackageForPackage(pkg, "INFO", "Hybrid-Extract abgeschlossen", {
|
this.logPackageForPackage(pkg, "INFO", "Hybrid-Extract abgeschlossen", {
|
||||||
@@ -13799,9 +13819,10 @@ export class DownloadManager extends EventEmitter {
|
|||||||
preExtractStatuses.set(entry.id, String(entry.fullStatus || "").trim());
|
preExtractStatuses.set(entry.id, String(entry.fullStatus || "").trim());
|
||||||
entry.fullStatus = "Entpacken - Ausstehend";
|
entry.fullStatus = "Entpacken - Ausstehend";
|
||||||
entry.updatedAt = pendingAt;
|
entry.updatedAt = pendingAt;
|
||||||
}
|
}
|
||||||
this.emitState();
|
this.emitState();
|
||||||
const result = await extractPackageArchives({
|
const packageOutputBefore = await this.snapshotPackageOutputFiles(pkg.extractDir);
|
||||||
|
const result = await extractPackageArchives({
|
||||||
packageDir: pkg.outputDir,
|
packageDir: pkg.outputDir,
|
||||||
targetDir: pkg.extractDir,
|
targetDir: pkg.extractDir,
|
||||||
cleanupMode: this.settings.cleanupMode,
|
cleanupMode: this.settings.cleanupMode,
|
||||||
@@ -13943,7 +13964,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
emitExtractStatus(overallLabel);
|
emitExtractStatus(overallLabel);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
pkg.outputCount = Math.max(pkg.outputCount || 0, await this.countPackageOutputFiles(pkg.extractDir));
|
await this.recordPackageOutputFiles(pkg, packageOutputBefore);
|
||||||
logger.info(`Post-Processing Entpacken Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}, lastError=${result.lastError || ""}`);
|
logger.info(`Post-Processing Entpacken Ende: pkg=${pkg.name}, extracted=${result.extracted}, failed=${result.failed}, lastError=${result.lastError || ""}`);
|
||||||
this.logPackageForPackage(pkg, "INFO", "Post-Processing Entpacken Ende", {
|
this.logPackageForPackage(pkg, "INFO", "Post-Processing Entpacken Ende", {
|
||||||
extracted: result.extracted,
|
extracted: result.extracted,
|
||||||
@@ -14150,6 +14171,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
});
|
});
|
||||||
const nestedFailureCategories = new Map<string, string>();
|
const nestedFailureCategories = new Map<string, string>();
|
||||||
const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[];
|
const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[];
|
||||||
|
const packageOutputBefore = await this.snapshotPackageOutputFiles(pkg.extractDir);
|
||||||
const nestedResult = await extractPackageArchives({
|
const nestedResult = await extractPackageArchives({
|
||||||
packageDir: pkg.extractDir,
|
packageDir: pkg.extractDir,
|
||||||
targetDir: pkg.extractDir,
|
targetDir: pkg.extractDir,
|
||||||
@@ -14177,7 +14199,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
throwIfAborted();
|
throwIfAborted();
|
||||||
pkg.outputCount = Math.max(pkg.outputCount || 0, await this.countPackageOutputFiles(pkg.extractDir));
|
await this.recordPackageOutputFiles(pkg, packageOutputBefore);
|
||||||
extractedCount += nestedResult.extracted;
|
extractedCount += nestedResult.extracted;
|
||||||
logger.info(`Deferred Nested-Extraction Ende: extracted=${nestedResult.extracted}, failed=${nestedResult.failed}`);
|
logger.info(`Deferred Nested-Extraction Ende: extracted=${nestedResult.extracted}, failed=${nestedResult.failed}`);
|
||||||
this.logPackageForPackage(pkg, "INFO", "Deferred Nested-Extraction Ende", {
|
this.logPackageForPackage(pkg, "INFO", "Deferred Nested-Extraction Ende", {
|
||||||
|
|||||||
@@ -1007,6 +1007,9 @@ export function normalizeLoadedSession(raw: unknown): SessionState {
|
|||||||
archiveOperations: normalizeArchiveOperations(pkg.archiveOperations),
|
archiveOperations: normalizeArchiveOperations(pkg.archiveOperations),
|
||||||
remuxOperations: normalizeRemuxOperations(pkg.remuxOperations),
|
remuxOperations: normalizeRemuxOperations(pkg.remuxOperations),
|
||||||
outputCount: clampNumber(pkg.outputCount, 0, 0, 1_000_000),
|
outputCount: clampNumber(pkg.outputCount, 0, 0, 1_000_000),
|
||||||
|
outputProvenance: Array.isArray(pkg.outputProvenance)
|
||||||
|
? [...new Set(pkg.outputProvenance.map((value) => asText(value).toLowerCase()).filter((value) => /^[a-f0-9]{64}$/.test(value)))].slice(0, 1_000_000)
|
||||||
|
: [],
|
||||||
cleanupErrorCategory: asText(pkg.cleanupErrorCategory),
|
cleanupErrorCategory: asText(pkg.cleanupErrorCategory),
|
||||||
resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER),
|
resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER),
|
||||||
createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER),
|
createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER),
|
||||||
|
|||||||
@@ -588,6 +588,7 @@ export interface PackageEntry {
|
|||||||
archiveOperations?: ArchiveOperationMetric[];
|
archiveOperations?: ArchiveOperationMetric[];
|
||||||
remuxOperations?: RemuxOperationMetric[];
|
remuxOperations?: RemuxOperationMetric[];
|
||||||
outputCount?: number;
|
outputCount?: number;
|
||||||
|
outputProvenance?: string[];
|
||||||
cleanupErrorCategory?: string;
|
cleanupErrorCategory?: string;
|
||||||
resultGeneration?: number;
|
resultGeneration?: number;
|
||||||
createdAt: number;
|
createdAt: number;
|
||||||
|
|||||||
@@ -11466,8 +11466,10 @@ describe("download manager", () => {
|
|||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
|
|
||||||
const outputDir = path.join(root, "downloads", "recovery");
|
const outputDir = path.join(root, "downloads", "recovery");
|
||||||
const extractDir = path.join(root, "extract", "recovery");
|
const extractDir = path.join(root, "extract");
|
||||||
fs.mkdirSync(outputDir, { recursive: true });
|
fs.mkdirSync(outputDir, { recursive: true });
|
||||||
|
fs.mkdirSync(extractDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(extractDir, "foreign-package.txt"), "foreign");
|
||||||
|
|
||||||
const zip = new AdmZip();
|
const zip = new AdmZip();
|
||||||
zip.addFile("episode.txt", Buffer.from("ok"));
|
zip.addFile("episode.txt", Buffer.from("ok"));
|
||||||
@@ -11518,7 +11520,7 @@ describe("download manager", () => {
|
|||||||
token: "rd-token",
|
token: "rd-token",
|
||||||
outputDir: path.join(root, "downloads"),
|
outputDir: path.join(root, "downloads"),
|
||||||
extractDir: path.join(root, "extract"),
|
extractDir: path.join(root, "extract"),
|
||||||
createExtractSubfolder: true,
|
createExtractSubfolder: false,
|
||||||
autoExtract: true,
|
autoExtract: true,
|
||||||
enableIntegrityCheck: false,
|
enableIntegrityCheck: false,
|
||||||
cleanupMode: "none"
|
cleanupMode: "none"
|
||||||
|
|||||||
@@ -1200,6 +1200,7 @@ describe("settings storage", () => {
|
|||||||
}],
|
}],
|
||||||
remuxOperations: [],
|
remuxOperations: [],
|
||||||
outputCount: 1,
|
outputCount: 1,
|
||||||
|
outputProvenance: ["a".repeat(64), "invalid", "a".repeat(64)],
|
||||||
cleanupErrorCategory: "",
|
cleanupErrorCategory: "",
|
||||||
createdAt: 1_000,
|
createdAt: 1_000,
|
||||||
updatedAt: 21_000
|
updatedAt: 21_000
|
||||||
@@ -1226,6 +1227,7 @@ describe("settings storage", () => {
|
|||||||
archiveOperations: [expect.objectContaining({ id: "archive-1", durationMs: 4_000 })],
|
archiveOperations: [expect.objectContaining({ id: "archive-1", durationMs: 4_000 })],
|
||||||
remuxOperations: [],
|
remuxOperations: [],
|
||||||
outputCount: 1,
|
outputCount: 1,
|
||||||
|
outputProvenance: ["a".repeat(64)],
|
||||||
cleanupErrorCategory: ""
|
cleanupErrorCategory: ""
|
||||||
}));
|
}));
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user