fix: scope extraction follow-up work
Remove .rd-output staging and merge handling, parse validated native extractor output, and drive nested extraction, rename, audio, cleanup, residual handling, and library collection from package-owned output records. Preserve provenance migration, package pruning, and run-owned waiter isolation while allowing one bounded legacy adoption scan only for provably exclusive package roots.
This commit is contained in:
Sucukdeluxe
+125
-16
@@ -8,6 +8,53 @@ async function yieldToLoop(): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
async function isDownloadLinkArtifact(filePath: string): Promise<boolean> {
|
||||
const fileName = path.basename(filePath);
|
||||
const ext = path.extname(fileName).toLowerCase();
|
||||
const name = fileName.toLowerCase();
|
||||
if (LINK_ARTIFACT_EXTENSIONS.has(ext)) {
|
||||
return true;
|
||||
}
|
||||
if (![".txt", ".html", ".htm", ".nfo"].includes(ext)
|
||||
|| !/[._\- ](links?|downloads?|urls?|dlc)([._\- ]|$)/i.test(name)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const stat = await fs.promises.lstat(filePath);
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_LINK_ARTIFACT_BYTES) {
|
||||
return false;
|
||||
}
|
||||
const text = await fs.promises.readFile(filePath, "utf8");
|
||||
return /https?:\/\//i.test(text);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function removeEmptyParentChains(rootDir: string, parents: ReadonlySet<string>): Promise<number> {
|
||||
const rootPath = path.resolve(rootDir);
|
||||
const candidates = new Set<string>();
|
||||
for (const parent of parents) {
|
||||
let current = path.resolve(parent);
|
||||
while (current !== rootPath && current.startsWith(`${rootPath}${path.sep}`)) {
|
||||
candidates.add(current);
|
||||
current = path.dirname(current);
|
||||
}
|
||||
}
|
||||
let removed = 0;
|
||||
for (const directory of [...candidates].sort((left, right) => right.length - left.length)) {
|
||||
try {
|
||||
const entries = await fs.promises.readdir(directory);
|
||||
if (entries.length === 0) {
|
||||
await fs.promises.rmdir(directory);
|
||||
removed += 1;
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function isArchiveOrTempFile(filePath: string): boolean {
|
||||
const lowerName = path.basename(filePath).toLowerCase();
|
||||
const ext = path.extname(lowerName);
|
||||
@@ -126,22 +173,7 @@ export async function removeDownloadLinkArtifacts(
|
||||
continue;
|
||||
}
|
||||
|
||||
const ext = path.extname(entry.name).toLowerCase();
|
||||
const name = entry.name.toLowerCase();
|
||||
let shouldDelete = LINK_ARTIFACT_EXTENSIONS.has(ext);
|
||||
if (!shouldDelete && [".txt", ".html", ".htm", ".nfo"].includes(ext)) {
|
||||
if (/[._\- ](links?|downloads?|urls?|dlc)([._\- ]|$)/i.test(name)) {
|
||||
try {
|
||||
const stat = await fs.promises.stat(full);
|
||||
if (stat.size <= MAX_LINK_ARTIFACT_BYTES) {
|
||||
const text = await fs.promises.readFile(full, "utf8");
|
||||
shouldDelete = /https?:\/\//i.test(text);
|
||||
}
|
||||
} catch {
|
||||
shouldDelete = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
const shouldDelete = await isDownloadLinkArtifact(full);
|
||||
|
||||
if (shouldDelete) {
|
||||
try {
|
||||
@@ -155,6 +187,32 @@ export async function removeDownloadLinkArtifacts(
|
||||
return removed;
|
||||
}
|
||||
|
||||
export async function removeDownloadLinkArtifactsFromScope(
|
||||
outputFiles: readonly string[],
|
||||
options: { shouldAbort?: () => boolean; rootDir?: string } = {}
|
||||
): Promise<number> {
|
||||
let removed = 0;
|
||||
const parents = new Set<string>();
|
||||
for (const outputFile of outputFiles) {
|
||||
if (options.shouldAbort?.()) {
|
||||
return removed;
|
||||
}
|
||||
if (!await isDownloadLinkArtifact(outputFile)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await fs.promises.rm(outputFile, { force: true });
|
||||
parents.add(path.dirname(outputFile));
|
||||
removed += 1;
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
if (options.rootDir) {
|
||||
await removeEmptyParentChains(options.rootDir, parents);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export async function removeSampleArtifacts(
|
||||
extractDir: string,
|
||||
options: { shouldAbort?: () => boolean } = {}
|
||||
@@ -264,3 +322,54 @@ export async function removeSampleArtifacts(
|
||||
|
||||
return { files: removedFiles, dirs: removedDirs };
|
||||
}
|
||||
|
||||
export async function removeSampleArtifactsFromScope(
|
||||
outputFiles: readonly string[],
|
||||
options: { shouldAbort?: () => boolean; rootDir?: string } = {}
|
||||
): Promise<{ files: number; dirs: number }> {
|
||||
let removedFiles = 0;
|
||||
const candidateParents = new Set<string>();
|
||||
for (const outputFile of outputFiles) {
|
||||
if (options.shouldAbort?.()) {
|
||||
return { files: removedFiles, dirs: 0 };
|
||||
}
|
||||
const fileName = path.basename(outputFile);
|
||||
const stem = path.parse(fileName).name.toLowerCase();
|
||||
const ext = path.extname(fileName).toLowerCase();
|
||||
const parentDir = path.dirname(outputFile);
|
||||
const inSampleDir = SAMPLE_DIR_NAMES.has(path.basename(parentDir).toLowerCase());
|
||||
if (!inSampleDir && !(SAMPLE_VIDEO_EXTENSIONS.has(ext) && SAMPLE_TOKEN_RE.test(stem))) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const stat = await fs.promises.lstat(outputFile);
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
await fs.promises.rm(outputFile, { force: true });
|
||||
removedFiles += 1;
|
||||
if (inSampleDir) {
|
||||
candidateParents.add(parentDir);
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
let removedDirs = 0;
|
||||
for (const parentDir of candidateParents) {
|
||||
if (options.shouldAbort?.()) {
|
||||
return { files: removedFiles, dirs: removedDirs };
|
||||
}
|
||||
try {
|
||||
const entries = await fs.promises.readdir(parentDir);
|
||||
if (entries.length === 0) {
|
||||
await fs.promises.rmdir(parentDir);
|
||||
removedDirs += 1;
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
if (options.rootDir) {
|
||||
removedDirs += await removeEmptyParentChains(options.rootDir, candidateParents);
|
||||
}
|
||||
return { files: removedFiles, dirs: removedDirs };
|
||||
}
|
||||
|
||||
+256
-281
@@ -61,7 +61,7 @@ function releaseTlsSkip(): void {
|
||||
delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
||||
}
|
||||
}
|
||||
import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup";
|
||||
import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifactsFromScope, removeSampleArtifactsFromScope } from "./cleanup";
|
||||
import { planDownloadCompletion, reconcileFinalizedSize, validateDownloadedFileCompletion } from "./download-completion";
|
||||
import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkDdownloadOnline, checkOneFichierLinks, checkRapidgatorOnline, fetchAllDebridHostInfo, filenameFromDdownloadUrlPath, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, isDdownloadLink, isOneFichierLink, isProviderDisabledForSelection, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown, type DdownloadCheckResult, type OneFichierCheckResult } from "./debrid";
|
||||
import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, detectArchiveSignature, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, resetExtractorCachesForPasswordChange, type ExtractArchiveFailureInfo, type ExtractProgressUpdate } from "./extractor";
|
||||
@@ -82,6 +82,7 @@ import { compactErrorText, ensureDirPath, filenameFromUrl, formatEta, humanSize,
|
||||
import { mergeKnownTotalBytes } from "./download-size";
|
||||
import { DiskCapacityError, DiskReservationCoordinator, type DiskReservationLease } from "./disk-space";
|
||||
import { createRendererState } from "./renderer-state";
|
||||
import { PackageOutputScope } from "./package-output-scope";
|
||||
import {
|
||||
RollingAccountStatisticsAccumulator,
|
||||
addStatisticsActiveIntervalInPlace,
|
||||
@@ -477,7 +478,6 @@ type DownloadManagerOptions = {
|
||||
onHistoryEntry?: HistoryEntryCallback;
|
||||
enqueueNotification?: (event: NotificationEvent) => Promise<void>;
|
||||
protectEmptyClobber?: boolean;
|
||||
readOutputDirectory?: (directory: string) => Promise<fs.Dirent[]>;
|
||||
};
|
||||
|
||||
type RunLifecycleContext = {
|
||||
@@ -1930,7 +1930,7 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
private cleanupQueue: Promise<void> = Promise.resolve();
|
||||
|
||||
private packageOutputProvenanceTails = new Map<string, Promise<void>>();
|
||||
private packageOutputScopes = new Map<string, PackageOutputScope>();
|
||||
|
||||
private packagePostProcessQueue: Promise<void> = Promise.resolve();
|
||||
|
||||
@@ -2069,8 +2069,6 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
private enqueueNotificationCallback?: (event: NotificationEvent) => Promise<void>;
|
||||
|
||||
private readOutputDirectoryFn: (directory: string) => Promise<fs.Dirent[]>;
|
||||
|
||||
public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) {
|
||||
super();
|
||||
this.settings = settings;
|
||||
@@ -2100,7 +2098,6 @@ export class DownloadManager extends EventEmitter {
|
||||
this.invalidateMegaSessionFn = options.invalidateMegaSession;
|
||||
this.onHistoryEntryCallback = options.onHistoryEntry;
|
||||
this.enqueueNotificationCallback = options.enqueueNotification;
|
||||
this.readOutputDirectoryFn = options.readOutputDirectory || ((directory) => fs.promises.readdir(directory, { withFileTypes: true }));
|
||||
logger.info(`DownloadManager Init: ${Object.keys(this.session.packages).length} Pakete, ${this.itemCount} Items, cleanupPolicy=${this.settings.completedCleanupPolicy}`);
|
||||
for (const pkg of Object.values(this.session.packages)) {
|
||||
this.ensurePackageLogForPackage(pkg);
|
||||
@@ -3515,7 +3512,7 @@ export class DownloadManager extends EventEmitter {
|
||||
const extractDirKey = pathKey(pkg.extractDir);
|
||||
const hasExtractedFiles = hasFilesByExtractDir.has(extractDirKey)
|
||||
? Boolean(hasFilesByExtractDir.get(extractDirKey))
|
||||
: await this.directoryHasAnyFiles(pkg.extractDir);
|
||||
: this.getPackageOutputScope(pkg).completeFiles().some((filePath) => isPathInsideDir(filePath, pkg.extractDir));
|
||||
if (!hasFilesByExtractDir.has(extractDirKey)) {
|
||||
hasFilesByExtractDir.set(extractDirKey, hasExtractedFiles);
|
||||
}
|
||||
@@ -4268,7 +4265,8 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
const hasExtractMarker = items.some((item) => isExtractedLabel(item.fullStatus));
|
||||
const extractDirIsUnique = (extractDirUsage.get(pathKey(pkg.extractDir)) || 0) === 1;
|
||||
const hasExtractedOutput = extractDirIsUnique && await this.directoryHasAnyFiles(pkg.extractDir);
|
||||
const hasExtractedOutput = extractDirIsUnique
|
||||
&& this.getPackageOutputScope(pkg).completeFiles().some((filePath) => isPathInsideDir(filePath, pkg.extractDir));
|
||||
if (!hasExtractMarker && !hasExtractedOutput) {
|
||||
continue;
|
||||
}
|
||||
@@ -4384,103 +4382,6 @@ export class DownloadManager extends EventEmitter {
|
||||
return false;
|
||||
}
|
||||
|
||||
private async listStagedOutputFiles(stagingDir: string): Promise<string[]> {
|
||||
const files: string[] = [];
|
||||
const stack = [stagingDir];
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop() as string;
|
||||
let entries: fs.Dirent[] = [];
|
||||
try {
|
||||
entries = await this.readOutputDirectoryFn(current);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
stack.push(fullPath);
|
||||
} else if (entry.isFile()) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return files.sort((left, right) => path.relative(stagingDir, left).localeCompare(path.relative(stagingDir, right)));
|
||||
}
|
||||
|
||||
private async resolveStagedOutputDestination(targetDir: string, relativePath: string): Promise<string | null> {
|
||||
const targetRoot = path.resolve(targetDir);
|
||||
const destination = path.resolve(targetRoot, relativePath);
|
||||
if (destination !== targetRoot && !destination.startsWith(`${targetRoot}${path.sep}`)) {
|
||||
throw new Error(`Ungültiger Staging-Ausgabepfad: ${relativePath}`);
|
||||
}
|
||||
let existing: fs.Stats | null = null;
|
||||
try {
|
||||
existing = await fs.promises.lstat(destination);
|
||||
} catch {
|
||||
}
|
||||
if (!existing) {
|
||||
return destination;
|
||||
}
|
||||
if (this.settings.extractConflictMode === "skip" || this.settings.extractConflictMode === "ask") {
|
||||
return null;
|
||||
}
|
||||
if (this.settings.extractConflictMode === "overwrite") {
|
||||
return existing.isFile() ? destination : null;
|
||||
}
|
||||
const parsed = path.parse(destination);
|
||||
for (let index = 1; index <= 10_000; index += 1) {
|
||||
const candidate = path.join(parsed.dir, `${parsed.name} (${index})${parsed.ext}`);
|
||||
try {
|
||||
await fs.promises.lstat(candidate);
|
||||
} catch {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
throw new Error(`Staging-Rename-Limit erreicht für ${relativePath}`);
|
||||
}
|
||||
|
||||
private async moveStagedOutputFile(sourcePath: string, destinationPath: string): Promise<void> {
|
||||
await fs.promises.mkdir(path.dirname(destinationPath), { recursive: true });
|
||||
try {
|
||||
await fs.promises.rename(sourcePath, destinationPath);
|
||||
return;
|
||||
} catch (error) {
|
||||
const code = String((error as NodeJS.ErrnoException)?.code || "");
|
||||
if (this.settings.extractConflictMode !== "overwrite" || (code !== "EEXIST" && code !== "EPERM" && code !== "EACCES")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const displacedPath = path.join(path.dirname(destinationPath), `.rd-replace-${uuidv4()}`);
|
||||
await fs.promises.rename(destinationPath, displacedPath);
|
||||
try {
|
||||
await fs.promises.rename(sourcePath, destinationPath);
|
||||
} catch (error) {
|
||||
await fs.promises.rename(displacedPath, destinationPath);
|
||||
throw error;
|
||||
}
|
||||
await fs.promises.rm(displacedPath, { force: true });
|
||||
}
|
||||
|
||||
private async mergeStagedPackageOutputs(stagingDir: string, targetDir: string): Promise<string[]> {
|
||||
const movedOutputs: string[] = [];
|
||||
const stagedFiles = await this.listStagedOutputFiles(stagingDir);
|
||||
for (const stagedFile of stagedFiles) {
|
||||
const relativePath = path.relative(stagingDir, stagedFile);
|
||||
const destination = await this.resolveStagedOutputDestination(targetDir, relativePath);
|
||||
if (!destination) {
|
||||
continue;
|
||||
}
|
||||
await this.moveStagedOutputFile(stagedFile, destination);
|
||||
if (!isArchiveLikePath(destination) && !isIgnorableEmptyDirFileName(path.basename(destination))) {
|
||||
movedOutputs.push(destination);
|
||||
}
|
||||
}
|
||||
return movedOutputs;
|
||||
}
|
||||
|
||||
private normalizePackageProvenancePath(pkg: PackageEntry, sourcePath: string): string {
|
||||
const absolutePath = path.resolve(sourcePath);
|
||||
for (const rootDir of [pkg.outputDir, pkg.extractDir]) {
|
||||
@@ -4490,82 +4391,129 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
const relativePath = path.relative(path.resolve(root), absolutePath);
|
||||
if (!relativePath.startsWith(`..${path.sep}`) && relativePath !== ".." && !path.isAbsolute(relativePath)) {
|
||||
const segments = relativePath.replace(/\\/g, "/").split("/");
|
||||
if (/^\.rd-output-[^/]+$/i.test(segments[0] || "")) {
|
||||
segments.shift();
|
||||
}
|
||||
return segments.join("/").toLocaleLowerCase("de-DE");
|
||||
return relativePath.replace(/\\/g, "/").toLocaleLowerCase("de-DE");
|
||||
}
|
||||
}
|
||||
return absolutePath.replace(/\\/g, "/").toLocaleLowerCase("de-DE");
|
||||
}
|
||||
|
||||
private recordPackageOutputFiles(pkg: PackageEntry, outputFiles: readonly string[]): void {
|
||||
const provenance = new Set(pkg.outputProvenance || []);
|
||||
for (const outputFile of outputFiles) {
|
||||
private syncPackageOutputScope(pkg: PackageEntry, scope: PackageOutputScope): void {
|
||||
scope.pruneMissing();
|
||||
const provenance = new Set<string>();
|
||||
for (const outputFile of scope.files()) {
|
||||
const key = this.normalizePackageProvenancePath(pkg, outputFile);
|
||||
provenance.add(createHash("sha256").update(key).digest("hex"));
|
||||
}
|
||||
pkg.outputProvenance = [...provenance];
|
||||
pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION;
|
||||
pkg.outputCount = provenance.size;
|
||||
pkg.outputRecords = scope.records();
|
||||
}
|
||||
|
||||
private async runWithPackageOutputProvenance<T>(pkg: PackageEntry, operation: (targetDir: string) => Promise<T>): Promise<T> {
|
||||
const key = pathKey(pkg.extractDir);
|
||||
const packageWasInSession = this.session.packages[pkg.id] === pkg;
|
||||
const previous = this.packageOutputProvenanceTails.get(key) || Promise.resolve();
|
||||
let release!: () => void;
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
private getPackageOutputScope(pkg: PackageEntry): PackageOutputScope {
|
||||
const current = this.packageOutputScopes.get(pkg.id);
|
||||
if (current) {
|
||||
return current;
|
||||
}
|
||||
const roots = [pkg.outputDir, pkg.extractDir, String(this.settings.mkvLibraryDir || "").trim()].filter(Boolean);
|
||||
const scope = new PackageOutputScope(roots);
|
||||
if (pkg.outputProvenanceVersion === PACKAGE_OUTPUT_PROVENANCE_VERSION) {
|
||||
for (const record of pkg.outputRecords || []) {
|
||||
try {
|
||||
scope.add(record);
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
}
|
||||
this.packageOutputScopes.set(pkg.id, scope);
|
||||
return scope;
|
||||
}
|
||||
|
||||
private async adoptLegacyPackageOutputsIfExclusive(pkg: PackageEntry, scope: PackageOutputScope): Promise<void> {
|
||||
if (pkg.outputScopeAdopted || scope.records().length > 0) {
|
||||
pkg.outputScopeAdopted = true;
|
||||
return;
|
||||
}
|
||||
pkg.outputScopeAdopted = true;
|
||||
if (pkg.outputProvenanceVersion !== undefined
|
||||
&& pkg.outputProvenanceVersion !== PACKAGE_OUTPUT_PROVENANCE_VERSION) {
|
||||
return;
|
||||
}
|
||||
const packageExclusive = (this.settings.createExtractSubfolder || this.isPackageSpecificExtractDir(pkg))
|
||||
&& !this.isExtractDirSharedWithOtherPackages(pkg.id, pkg.extractDir);
|
||||
if (!packageExclusive || !await this.existsAsync(pkg.extractDir)) {
|
||||
return;
|
||||
}
|
||||
const candidates: string[] = [];
|
||||
const stack = [pkg.extractDir];
|
||||
let inspected = 0;
|
||||
let overflow = false;
|
||||
while (stack.length > 0 && !overflow) {
|
||||
const current = stack.pop() as string;
|
||||
let entries: fs.Dirent[] = [];
|
||||
try {
|
||||
entries = await fs.promises.readdir(current, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
inspected += 1;
|
||||
if (inspected > 100_000) {
|
||||
overflow = true;
|
||||
break;
|
||||
}
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
if (!/^\.rd-(?:output|replace)-/i.test(entry.name) && !/^\.rd-trash$/i.test(entry.name)) {
|
||||
stack.push(fullPath);
|
||||
}
|
||||
} else if (entry.isFile()
|
||||
&& !/^\.rd-(?:output|replace)-/i.test(entry.name)
|
||||
&& !/^\.rd_extract_progress(?:_[^.]+)?\.json$/i.test(entry.name)
|
||||
&& !isIgnorableEmptyDirFileName(entry.name)) {
|
||||
candidates.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (overflow) {
|
||||
return;
|
||||
}
|
||||
const archivePath = path.resolve(pkg.outputDir || pkg.extractDir, ".legacy-output-scope");
|
||||
for (const outputPath of candidates) {
|
||||
const entryPath = path.relative(pkg.extractDir, outputPath).replace(/\\/g, "/");
|
||||
try {
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath,
|
||||
entryPath,
|
||||
outputPath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
this.packageOutputProvenanceTails.set(key, current);
|
||||
let stagingDir = "";
|
||||
let result: T | undefined;
|
||||
let operationError: unknown;
|
||||
let mergeError: unknown;
|
||||
let mergeTurnReached = false;
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
this.syncPackageOutputScope(pkg, scope);
|
||||
}
|
||||
|
||||
private async runWithPackageOutputProvenance<T>(
|
||||
pkg: PackageEntry,
|
||||
operation: (targetDir: string, scope: PackageOutputScope) => Promise<T>
|
||||
): Promise<T> {
|
||||
const packageWasInSession = this.session.packages[pkg.id] === pkg;
|
||||
const scope = this.getPackageOutputScope(pkg);
|
||||
try {
|
||||
await fs.promises.mkdir(pkg.extractDir, { recursive: true });
|
||||
stagingDir = await fs.promises.mkdtemp(path.join(pkg.extractDir, ".rd-output-"));
|
||||
try {
|
||||
result = await operation(stagingDir);
|
||||
} catch (error) {
|
||||
operationError = error;
|
||||
}
|
||||
await previous;
|
||||
mergeTurnReached = true;
|
||||
try {
|
||||
return await operation(pkg.extractDir, scope);
|
||||
} finally {
|
||||
if (!packageWasInSession || this.session.packages[pkg.id] === pkg) {
|
||||
const outputFiles = await this.mergeStagedPackageOutputs(stagingDir, pkg.extractDir);
|
||||
this.recordPackageOutputFiles(pkg, outputFiles);
|
||||
}
|
||||
} catch (error) {
|
||||
mergeError = error;
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
if (stagingDir) {
|
||||
await fs.promises.rm(stagingDir, { recursive: true, force: true });
|
||||
}
|
||||
} finally {
|
||||
if (!mergeTurnReached) {
|
||||
await previous;
|
||||
}
|
||||
release();
|
||||
if (this.packageOutputProvenanceTails.get(key) === current) {
|
||||
this.packageOutputProvenanceTails.delete(key);
|
||||
this.syncPackageOutputScope(pkg, scope);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (operationError) {
|
||||
throw operationError;
|
||||
}
|
||||
if (mergeError) {
|
||||
throw mergeError;
|
||||
}
|
||||
return result as T;
|
||||
}
|
||||
|
||||
private async removeEmptyDirectoryTree(rootDir: string): Promise<number> {
|
||||
if (!rootDir) {
|
||||
@@ -4622,13 +4570,8 @@ export class DownloadManager extends EventEmitter {
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async collectFilesByExtensions(rootDir: string, extensions: Set<string>): Promise<string[]> {
|
||||
if (!rootDir || extensions.size === 0) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
await fs.promises.access(rootDir);
|
||||
} catch {
|
||||
private async collectFilesByExtensions(scope: PackageOutputScope, extensions: Set<string>): Promise<string[]> {
|
||||
if (extensions.size === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -4644,46 +4587,24 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
const files: string[] = [];
|
||||
const stack = [rootDir];
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop() as string;
|
||||
let entries: fs.Dirent[] = [];
|
||||
for (const filePath of scope.completeFiles()) {
|
||||
const fileName = path.basename(filePath);
|
||||
if (fileName.startsWith("~rd") || !normalizedExtensions.has(path.extname(fileName).toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
entries = await fs.promises.readdir(current, { withFileTypes: true });
|
||||
const stat = await fs.promises.lstat(filePath);
|
||||
if (stat.isFile() && !stat.isSymbolicLink()) {
|
||||
files.push(filePath);
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
stack.push(fullPath);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
// Never collect our own remux temp/orphan sidecars (~rd<token>.<ext>): a
|
||||
// partial file left by a crash mid-remux must not be swept into the library.
|
||||
if (entry.name.startsWith("~rd")) {
|
||||
continue;
|
||||
}
|
||||
const extension = path.extname(entry.name).toLowerCase();
|
||||
if (!normalizedExtensions.has(extension)) {
|
||||
continue;
|
||||
}
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
private async collectVideoFiles(rootDir: string): Promise<string[]> {
|
||||
return await this.collectFilesByExtensions(rootDir, SAMPLE_VIDEO_EXTENSIONS);
|
||||
private async collectVideoFiles(scope: PackageOutputScope): Promise<string[]> {
|
||||
return await this.collectFilesByExtensions(scope, SAMPLE_VIDEO_EXTENSIONS);
|
||||
}
|
||||
|
||||
private async existsAsync(filePath: string): Promise<boolean> {
|
||||
@@ -4780,6 +4701,7 @@ export class DownloadManager extends EventEmitter {
|
||||
private async renameCompanionFiles(
|
||||
sourceVideoPath: string,
|
||||
targetVideoPath: string,
|
||||
scope: PackageOutputScope,
|
||||
pkg?: PackageEntry
|
||||
): Promise<void> {
|
||||
const COMPANION_EXTENSIONS = new Set([".srt", ".ass", ".ssa", ".sub", ".idx", ".vtt", ".smi", ".nfo"]);
|
||||
@@ -4790,17 +4712,9 @@ export class DownloadManager extends EventEmitter {
|
||||
if (!sourceVideoBase || !targetVideoBase || sourceVideoBase === targetVideoBase) {
|
||||
return;
|
||||
}
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = await fs.promises.readdir(sourceDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
const entryName = entry.name;
|
||||
const companionFiles = scope.completeFiles().filter((filePath) => pathKey(path.dirname(filePath)) === pathKey(sourceDir));
|
||||
for (const sourceCompanionPath of companionFiles) {
|
||||
const entryName = path.basename(sourceCompanionPath);
|
||||
const entryExt = path.extname(entryName).toLowerCase();
|
||||
if (!COMPANION_EXTENSIONS.has(entryExt)) {
|
||||
continue;
|
||||
@@ -4813,13 +4727,13 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
const suffixAfterBase = isExactMatch ? "" : entryBase.slice(sourceVideoBase.length);
|
||||
const newCompanionName = `${targetVideoBase}${suffixAfterBase}${entryExt}`;
|
||||
const sourceCompanionPath = path.join(sourceDir, entryName);
|
||||
const targetCompanionPath = path.join(targetDir, newCompanionName);
|
||||
if (sourceCompanionPath === targetCompanionPath) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await this.renamePathWithExdevFallback(sourceCompanionPath, targetCompanionPath, { label: "companion" });
|
||||
scope.replacePath(sourceCompanionPath, targetCompanionPath);
|
||||
logger.info(`Auto-Rename Companion: ${entryName} -> ${newCompanionName}`);
|
||||
if (pkg) {
|
||||
this.logPackageForPackage(pkg, "INFO", "Auto-Rename Companion umbenannt", {
|
||||
@@ -4836,6 +4750,7 @@ export class DownloadManager extends EventEmitter {
|
||||
private async moveCompanionFiles(
|
||||
sourceVideoPath: string,
|
||||
targetVideoPath: string,
|
||||
scope: PackageOutputScope,
|
||||
pkg?: PackageEntry
|
||||
): Promise<void> {
|
||||
const COMPANION_EXTENSIONS = new Set([".srt", ".ass", ".ssa", ".sub", ".idx", ".vtt", ".smi"]);
|
||||
@@ -4846,17 +4761,9 @@ export class DownloadManager extends EventEmitter {
|
||||
if (!sourceVideoBase || !targetVideoBase) {
|
||||
return;
|
||||
}
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = await fs.promises.readdir(sourceDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
const entryName = entry.name;
|
||||
const companionFiles = scope.completeFiles().filter((filePath) => pathKey(path.dirname(filePath)) === pathKey(sourceDir));
|
||||
for (const sourceCompanionPath of companionFiles) {
|
||||
const entryName = path.basename(sourceCompanionPath);
|
||||
const entryExt = path.extname(entryName).toLowerCase();
|
||||
if (!COMPANION_EXTENSIONS.has(entryExt)) {
|
||||
continue;
|
||||
@@ -4869,13 +4776,13 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
const suffixAfterBase = isExactMatch ? "" : entryBase.slice(sourceVideoBase.length);
|
||||
const newCompanionName = `${targetVideoBase}${suffixAfterBase}${entryExt}`;
|
||||
const sourceCompanionPath = path.join(sourceDir, entryName);
|
||||
const targetCompanionPath = path.join(targetDir, newCompanionName);
|
||||
if (sourceCompanionPath === targetCompanionPath) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await this.moveFileWithExdevFallback(sourceCompanionPath, targetCompanionPath);
|
||||
scope.replacePath(sourceCompanionPath, targetCompanionPath);
|
||||
logger.info(`MKV-Move Companion: ${entryName} -> ${newCompanionName}`);
|
||||
if (pkg) {
|
||||
this.logPackageForPackage(pkg, "INFO", "Companion mit-verschoben", {
|
||||
@@ -5007,29 +4914,31 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
private async autoRenameExtractedVideoFiles(
|
||||
extractDir: string,
|
||||
scope: PackageOutputScope,
|
||||
pkg?: PackageEntry,
|
||||
shouldAbort?: () => boolean,
|
||||
treatFilesAsStable = false
|
||||
): Promise<number> {
|
||||
if (!pkg) {
|
||||
return this.autoRenameExtractedVideoFilesImpl(extractDir, undefined, shouldAbort, treatFilesAsStable);
|
||||
return this.autoRenameExtractedVideoFilesImpl(extractDir, scope, undefined, shouldAbort, treatFilesAsStable);
|
||||
}
|
||||
return this.chainPackageFileOp(pkg.id, () =>
|
||||
this.autoRenameExtractedVideoFilesImpl(extractDir, pkg, shouldAbort, treatFilesAsStable)
|
||||
this.autoRenameExtractedVideoFilesImpl(extractDir, scope, pkg, shouldAbort, treatFilesAsStable)
|
||||
);
|
||||
}
|
||||
|
||||
private async keepGermanAudioOnly(
|
||||
extractDir: string,
|
||||
scope: PackageOutputScope,
|
||||
pkg?: PackageEntry,
|
||||
shouldAbort?: () => boolean,
|
||||
signal?: AbortSignal
|
||||
): Promise<number> {
|
||||
if (!pkg) {
|
||||
return this.keepGermanAudioOnlyImpl(extractDir, undefined, shouldAbort, signal);
|
||||
return this.keepGermanAudioOnlyImpl(extractDir, scope, undefined, shouldAbort, signal);
|
||||
}
|
||||
return this.chainPackageFileOp(pkg.id, () =>
|
||||
this.keepGermanAudioOnlyImpl(extractDir, pkg, shouldAbort, signal)
|
||||
this.keepGermanAudioOnlyImpl(extractDir, scope, pkg, shouldAbort, signal)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -5039,6 +4948,7 @@ export class DownloadManager extends EventEmitter {
|
||||
// error never fails the package. Original is never lost (see video-processor).
|
||||
private async keepGermanAudioOnlyImpl(
|
||||
extractDir: string,
|
||||
scope: PackageOutputScope,
|
||||
pkg?: PackageEntry,
|
||||
shouldAbort?: () => boolean,
|
||||
signal?: AbortSignal
|
||||
@@ -5052,7 +4962,7 @@ export class DownloadManager extends EventEmitter {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const videoFiles = await this.collectVideoFiles(extractDir);
|
||||
const videoFiles = await this.collectVideoFiles(scope);
|
||||
const sampleTokenRe = /(^|[._\-\s])sample([._\-\s]|$)/i;
|
||||
const targets = videoFiles.filter((p) => {
|
||||
const name = path.basename(p);
|
||||
@@ -5208,7 +5118,7 @@ export class DownloadManager extends EventEmitter {
|
||||
// already single-track. Skips/errors leave the file fully untouched so the
|
||||
// unprocessed state stays visible.
|
||||
if (result.action === "remuxed" || result.action === "kept-single") {
|
||||
await this.stripDualLangFromFileName(sourcePath, pkg);
|
||||
await this.stripDualLangFromFileName(sourcePath, scope, pkg);
|
||||
}
|
||||
if (pkg) {
|
||||
const completedAt = nowMs();
|
||||
@@ -5237,7 +5147,7 @@ export class DownloadManager extends EventEmitter {
|
||||
return processed;
|
||||
}
|
||||
|
||||
private async stripDualLangFromFileName(sourcePath: string, pkg?: PackageEntry): Promise<void> {
|
||||
private async stripDualLangFromFileName(sourcePath: string, scope: PackageOutputScope, pkg?: PackageEntry): Promise<void> {
|
||||
const dir = path.dirname(sourcePath);
|
||||
const name = path.basename(sourcePath);
|
||||
const newName = stripDualLangMarker(name);
|
||||
@@ -5251,7 +5161,8 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
try {
|
||||
await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "audio-strip" });
|
||||
await this.renameCompanionFiles(sourcePath, targetPath, pkg);
|
||||
scope.replacePath(sourcePath, targetPath);
|
||||
await this.renameCompanionFiles(sourcePath, targetPath, scope, pkg);
|
||||
if (pkg) {
|
||||
const resolved = this.inferItemForMediaLog(pkg, targetPath, path.basename(targetPath));
|
||||
this.logRenameProcess(pkg, "INFO", "audio-strip", ".DL. aus Dateiname entfernt", { sourcePath, targetPath }, resolved.item, resolved.matchedBy);
|
||||
@@ -5263,6 +5174,7 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
private async autoRenameExtractedVideoFilesImpl(
|
||||
extractDir: string,
|
||||
scope: PackageOutputScope,
|
||||
pkg?: PackageEntry,
|
||||
shouldAbort?: () => boolean,
|
||||
treatFilesAsStable = false
|
||||
@@ -5287,7 +5199,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
const videoFiles = await this.collectVideoFiles(extractDir);
|
||||
const videoFiles = await this.collectVideoFiles(scope);
|
||||
logger.info(`Auto-Rename: ${videoFiles.length} Video-Dateien gefunden in ${extractDir}`);
|
||||
if (pkg) {
|
||||
this.logPackageForPackage(pkg, "INFO", "Auto-Rename Scan gestartet", {
|
||||
@@ -5532,6 +5444,7 @@ export class DownloadManager extends EventEmitter {
|
||||
if (pathKey(targetPath) === pathKey(sourcePath) && targetPath !== sourcePath) {
|
||||
try {
|
||||
await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "auto-rename (Schreibweise)" });
|
||||
scope.replacePath(sourcePath, targetPath);
|
||||
renamed += 1;
|
||||
if (pkg) {
|
||||
const resolved = resolveRenameItem(targetPath);
|
||||
@@ -5593,6 +5506,7 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
try {
|
||||
await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "auto-rename" });
|
||||
scope.replacePath(sourcePath, targetPath);
|
||||
if (pkg) {
|
||||
this.logPackageForPackage(pkg, "INFO", "Auto-Rename durchgeführt", {
|
||||
sourcePath,
|
||||
@@ -5610,7 +5524,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
logger.info(`Auto-Rename: ${sourceName} -> ${path.basename(targetPath)}`);
|
||||
renamed += 1;
|
||||
await this.renameCompanionFiles(sourcePath, targetPath, pkg);
|
||||
await this.renameCompanionFiles(sourcePath, targetPath, scope, pkg);
|
||||
} catch (error) {
|
||||
if (this.isPathLengthRenameError(error)) {
|
||||
const fallbackCandidates = [
|
||||
@@ -5628,6 +5542,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
try {
|
||||
await this.renamePathWithExdevFallback(sourcePath, fallbackPath, { label: "auto-rename (Pfadlaenge-Fallback)" });
|
||||
scope.replacePath(sourcePath, fallbackPath);
|
||||
logger.warn(`Auto-Rename Fallback wegen Pfadlänge: ${sourceName} -> ${path.basename(fallbackPath)}`);
|
||||
renamed += 1;
|
||||
if (pkg) {
|
||||
@@ -5687,46 +5602,58 @@ export class DownloadManager extends EventEmitter {
|
||||
await this.renamePathWithExdevFallback(sourcePath, targetPath, { label: "mkv-move" });
|
||||
}
|
||||
|
||||
private async cleanupNonMkvResidualFiles(rootDir: string, targetDir: string): Promise<number> {
|
||||
if (!rootDir || !await this.existsAsync(rootDir)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
private async cleanupNonMkvResidualFiles(
|
||||
scope: PackageOutputScope,
|
||||
targetDir: string,
|
||||
touchedParents: Set<string>
|
||||
): Promise<number> {
|
||||
let removed = 0;
|
||||
const stack = [rootDir];
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop() as string;
|
||||
let entries: fs.Dirent[] = [];
|
||||
try {
|
||||
entries = await fs.promises.readdir(current, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
for (const fullPath of scope.completeFiles()) {
|
||||
if (isPathInsideDir(fullPath, targetDir)) {
|
||||
continue;
|
||||
}
|
||||
stack.push(fullPath);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
const extension = path.extname(entry.name).toLowerCase();
|
||||
if (SAMPLE_VIDEO_EXTENSIONS.has(extension)) {
|
||||
if (SAMPLE_VIDEO_EXTENSIONS.has(path.extname(fullPath).toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await fs.promises.rm(toWindowsLongPathIfNeeded(fullPath), { force: true });
|
||||
touchedParents.add(path.dirname(fullPath));
|
||||
scope.removePath(fullPath);
|
||||
removed += 1;
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async removeEmptyScopedParentChains(rootDir: string, parents: ReadonlySet<string>): Promise<number> {
|
||||
const rootPath = path.resolve(rootDir);
|
||||
const candidates = new Set<string>();
|
||||
for (const parent of parents) {
|
||||
let current = path.resolve(parent);
|
||||
while (isPathInsideDir(current, rootPath) && pathKey(current) !== pathKey(rootPath)) {
|
||||
candidates.add(current);
|
||||
const next = path.dirname(current);
|
||||
if (next === current) {
|
||||
break;
|
||||
}
|
||||
current = next;
|
||||
}
|
||||
}
|
||||
if (parents.size > 0) {
|
||||
candidates.add(rootPath);
|
||||
}
|
||||
let removed = 0;
|
||||
for (const directory of [...candidates].sort((left, right) => right.length - left.length)) {
|
||||
try {
|
||||
const entries = await fs.promises.readdir(directory);
|
||||
if (entries.length === 0) {
|
||||
await fs.promises.rmdir(directory);
|
||||
removed += 1;
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
@@ -5965,6 +5892,26 @@ export class DownloadManager extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
const targetDir = path.resolve(targetDirRaw);
|
||||
const scope = this.getPackageOutputScope(pkg);
|
||||
await this.adoptLegacyPackageOutputsIfExclusive(pkg, scope);
|
||||
for (const itemId of pkg.itemIds) {
|
||||
const item = this.session.items[itemId];
|
||||
const outputPath = String(item?.targetPath || "").trim();
|
||||
if (!item || item.status !== "completed" || !outputPath || !isPathInsideDir(outputPath, pkg.outputDir)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.resolve(outputPath),
|
||||
entryPath: path.basename(outputPath),
|
||||
outputPath: path.resolve(outputPath),
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
|
||||
const sourceDirs: string[] = [];
|
||||
for (const dir of sourceDirsAll) {
|
||||
@@ -6000,14 +5947,16 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
const seenBasenames = new Set<string>();
|
||||
const collected: { filePath: string; sourceRoot: string }[] = [];
|
||||
for (const dir of sourceDirs) {
|
||||
const filesInDir = await this.collectFilesByExtensions(dir, SAMPLE_VIDEO_EXTENSIONS);
|
||||
for (const filePath of filesInDir) {
|
||||
const scopedVideoFiles = await this.collectFilesByExtensions(scope, SAMPLE_VIDEO_EXTENSIONS);
|
||||
for (const filePath of scopedVideoFiles) {
|
||||
const sourceRoot = sourceDirs.find((dir) => isPathInsideDir(filePath, dir));
|
||||
if (!sourceRoot) {
|
||||
continue;
|
||||
}
|
||||
const baseLower = path.basename(filePath).toLowerCase();
|
||||
if (seenBasenames.has(baseLower)) continue;
|
||||
seenBasenames.add(baseLower);
|
||||
collected.push({ filePath, sourceRoot: dir });
|
||||
}
|
||||
collected.push({ filePath, sourceRoot });
|
||||
}
|
||||
if (collected.length === 0) {
|
||||
logger.info(`MKV-Sammelordner: pkg=${pkg.name}, keine MKV gefunden`);
|
||||
@@ -6059,6 +6008,7 @@ export class DownloadManager extends EventEmitter {
|
||||
let failed = 0;
|
||||
let sourceArtifactsChanged = false;
|
||||
let sourceCleanupRelevant = false;
|
||||
const touchedParents = new Set<string>();
|
||||
|
||||
for (const { filePath: sourcePath, sourceRoot } of mkvFiles) {
|
||||
if (shouldAbort?.()) {
|
||||
@@ -6131,6 +6081,8 @@ export class DownloadManager extends EventEmitter {
|
||||
}, resolved.item, resolved.matchedBy);
|
||||
try {
|
||||
await fs.promises.unlink(sourcePath);
|
||||
touchedParents.add(path.dirname(sourcePath));
|
||||
scope.removePath(sourcePath);
|
||||
sourceArtifactsChanged = true;
|
||||
} catch {
|
||||
}
|
||||
@@ -6149,6 +6101,8 @@ export class DownloadManager extends EventEmitter {
|
||||
|
||||
try {
|
||||
await this.moveFileWithExdevFallback(sourcePath, targetPath);
|
||||
touchedParents.add(path.dirname(sourcePath));
|
||||
scope.replacePath(sourcePath, targetPath);
|
||||
moved += 1;
|
||||
sourceArtifactsChanged = true;
|
||||
sourceCleanupRelevant = true;
|
||||
@@ -6163,7 +6117,7 @@ export class DownloadManager extends EventEmitter {
|
||||
targetPath,
|
||||
sourceSize
|
||||
}, resolved.item, resolved.matchedBy);
|
||||
await this.moveCompanionFiles(sourcePath, targetPath, pkg);
|
||||
await this.moveCompanionFiles(sourcePath, targetPath, scope, pkg);
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
logger.warn(`MKV verschieben fehlgeschlagen: ${sourcePath} -> ${targetPath} (${compactErrorText(error)})`);
|
||||
@@ -6183,11 +6137,11 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
if ((sourceArtifactsChanged || sourceCleanupRelevant) && cleanupDir && await this.existsAsync(cleanupDir)) {
|
||||
const removedResidual = await this.cleanupNonMkvResidualFiles(cleanupDir, targetDir);
|
||||
const removedResidual = await this.cleanupNonMkvResidualFiles(scope, targetDir, touchedParents);
|
||||
if (removedResidual > 0) {
|
||||
logger.info(`MKV-Sammelordner entfernte Restdateien: pkg=${pkg.name}, dir=${cleanupDir}, entfernt=${removedResidual}`);
|
||||
}
|
||||
const removedDirs = await this.removeEmptyDirectoryTree(cleanupDir);
|
||||
const removedDirs = await this.removeEmptyScopedParentChains(cleanupDir, touchedParents);
|
||||
if (removedDirs > 0) {
|
||||
logger.info(`MKV-Sammelordner entfernte leere Ordner: pkg=${pkg.name}, dir=${cleanupDir}, entfernt=${removedDirs}`);
|
||||
}
|
||||
@@ -9140,6 +9094,8 @@ export class DownloadManager extends EventEmitter {
|
||||
pkg.outputCount = 0;
|
||||
pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION;
|
||||
pkg.outputProvenance = [];
|
||||
pkg.outputRecords = [];
|
||||
pkg.outputScopeAdopted = false;
|
||||
}
|
||||
for (const itemId of itemIds) {
|
||||
this.retryAfterByItem.delete(itemId);
|
||||
@@ -12453,6 +12409,9 @@ export class DownloadManager extends EventEmitter {
|
||||
pkg.outputCount = 0;
|
||||
pkg.outputProvenanceVersion = PACKAGE_OUTPUT_PROVENANCE_VERSION;
|
||||
pkg.outputProvenance = [];
|
||||
pkg.outputRecords = [];
|
||||
pkg.outputScopeAdopted = false;
|
||||
this.packageOutputScopes.delete(packageId);
|
||||
pkg.cleanupErrorCategory = "";
|
||||
}
|
||||
this.pruneFinalizedPackageResults();
|
||||
@@ -12477,6 +12436,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
private pruneRemovedPackageResultState(packageId: string): void {
|
||||
this.packageOutputScopes.delete(packageId);
|
||||
const prefix = `${packageId}:`;
|
||||
for (const key of [...this.finalizedPackageResults.keys()]) {
|
||||
if (key.startsWith(prefix)) {
|
||||
@@ -13482,7 +13442,7 @@ export class DownloadManager extends EventEmitter {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const result = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({
|
||||
const result = await this.runWithPackageOutputProvenance(pkg, (targetDir, scope) => extractPackageArchives({
|
||||
packageDir: pkg.outputDir,
|
||||
targetDir,
|
||||
cleanupMode: this.settings.cleanupMode,
|
||||
@@ -13498,6 +13458,7 @@ export class DownloadManager extends EventEmitter {
|
||||
maxParallel: this.settings.maxParallelExtract || 2,
|
||||
extractCpuPriority: "high",
|
||||
onLog: (level, message) => this.logExtractionForItems(pkg, items, "Hybrid-Extractor", level, message),
|
||||
onOutput: (event) => scope.add(event),
|
||||
onArchiveFailure: (failure) => {
|
||||
failedArchiveCategories.set(String(failure.archiveName || "").toLowerCase(), failure.category);
|
||||
const failedArchiveKey = readyArchiveKeyByName.get(String(failure.archiveName || "").toLowerCase());
|
||||
@@ -13681,18 +13642,22 @@ export class DownloadManager extends EventEmitter {
|
||||
this.packageHybridPostProcessControllers.set(packageId, hybridSet);
|
||||
}
|
||||
hybridSet.add(hybridController);
|
||||
const hybridOutputScope = this.getPackageOutputScope(pkg);
|
||||
const hybridShouldAbort = (): boolean => hybridController.signal.aborted || this.session.packages[packageId] !== pkg;
|
||||
const hybridHandle: { task?: Promise<void> } = {};
|
||||
const hybridTask = (async () => {
|
||||
try {
|
||||
await this.chainPackageFileOp(pkg.id, async () => {
|
||||
await this.autoRenameExtractedVideoFilesImpl(pkg.extractDir, pkg, hybridShouldAbort);
|
||||
await this.keepGermanAudioOnlyImpl(pkg.extractDir, pkg, hybridShouldAbort, hybridController.signal);
|
||||
await this.autoRenameExtractedVideoFilesImpl(pkg.extractDir, hybridOutputScope, pkg, hybridShouldAbort);
|
||||
await this.keepGermanAudioOnlyImpl(pkg.extractDir, hybridOutputScope, pkg, hybridShouldAbort, hybridController.signal);
|
||||
await this.collectMkvFilesToLibrary(packageId, pkg, hybridShouldAbort, true);
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(`Hybrid Post-Extract (Rename+Collect) Fehler: pkg=${pkg.name}, reason=${compactErrorText(err)}`);
|
||||
} finally {
|
||||
if (this.session.packages[packageId] === pkg) {
|
||||
this.syncPackageOutputScope(pkg, hybridOutputScope);
|
||||
}
|
||||
const set = this.packageHybridPostProcessControllers.get(packageId);
|
||||
if (set) {
|
||||
set.delete(hybridController);
|
||||
@@ -13772,6 +13737,8 @@ export class DownloadManager extends EventEmitter {
|
||||
if (signal?.aborted) {
|
||||
return;
|
||||
}
|
||||
const packageOutputScope = this.getPackageOutputScope(pkg);
|
||||
await this.adoptLegacyPackageOutputsIfExclusive(pkg, packageOutputScope);
|
||||
const items = pkg.itemIds.map((id) => this.session.items[id]).filter(Boolean) as DownloadItem[];
|
||||
|
||||
const recoveryStart = nowMs();
|
||||
@@ -14057,7 +14024,7 @@ export class DownloadManager extends EventEmitter {
|
||||
entry.updatedAt = pendingAt;
|
||||
}
|
||||
this.emitState();
|
||||
const result = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({
|
||||
const result = await this.runWithPackageOutputProvenance(pkg, (targetDir, scope) => extractPackageArchives({
|
||||
packageDir: pkg.outputDir,
|
||||
targetDir,
|
||||
cleanupMode: this.settings.cleanupMode,
|
||||
@@ -14072,6 +14039,7 @@ export class DownloadManager extends EventEmitter {
|
||||
maxParallel: this.settings.maxParallelExtract || 2,
|
||||
extractCpuPriority: "high",
|
||||
onLog: (level, message) => this.logExtractionForItems(pkg, completedItems, "Extractor", level, message),
|
||||
onOutput: (event) => scope.add(event),
|
||||
onArchiveFailure: (failure) => {
|
||||
fullFailedArchiveCategories.set(failure.archiveName.toLowerCase(), failure.category);
|
||||
if (autoRecoveredArchives.has(failure.archiveName)) {
|
||||
@@ -14240,7 +14208,8 @@ export class DownloadManager extends EventEmitter {
|
||||
);
|
||||
pkg.status = "failed";
|
||||
} else {
|
||||
const hasExtractedOutput = await this.directoryHasAnyFiles(pkg.extractDir);
|
||||
const hasExtractedOutput = this.getPackageOutputScope(pkg).completeFiles()
|
||||
.some((filePath) => isPathInsideDir(filePath, pkg.extractDir));
|
||||
const sourceExists = await this.existsAsync(pkg.outputDir);
|
||||
let finalStatusText = "";
|
||||
|
||||
@@ -14388,13 +14357,15 @@ export class DownloadManager extends EventEmitter {
|
||||
const item = this.session.items[itemId];
|
||||
return Boolean(item && item.status === "completed" && isExtractErrorLabel(item.fullStatus || ""));
|
||||
});
|
||||
const outputScope = this.getPackageOutputScope(pkg);
|
||||
await this.adoptLegacyPackageOutputsIfExclusive(pkg, outputScope);
|
||||
|
||||
try {
|
||||
throwIfAborted();
|
||||
if ((extractedCount > 0 || alreadyMarkedExtracted) && failed === 0 && this.settings.autoExtract) {
|
||||
const nestedBlacklist = /\.(iso|img|bin|dmg|vhd|vhdx|vmdk|wim)$/i;
|
||||
const nestedCandidates = (await findArchiveCandidates(pkg.extractDir))
|
||||
.filter((p) => !nestedBlacklist.test(p));
|
||||
const nestedCandidates = outputScope.archiveFiles()
|
||||
.filter((candidate) => isPathInsideDir(candidate, pkg.extractDir) && !nestedBlacklist.test(candidate));
|
||||
if (nestedCandidates.length > 0) {
|
||||
pkg.postProcessLabel = "Nested Entpacken...";
|
||||
this.emitState();
|
||||
@@ -14405,7 +14376,7 @@ export class DownloadManager extends EventEmitter {
|
||||
});
|
||||
const nestedFailureCategories = new Map<string, string>();
|
||||
const nestedItems = pkg.itemIds.map((itemId) => this.session.items[itemId]).filter(Boolean) as DownloadItem[];
|
||||
const nestedResult = await this.runWithPackageOutputProvenance(pkg, (targetDir) => extractPackageArchives({
|
||||
const nestedResult = await this.runWithPackageOutputProvenance(pkg, (targetDir, scope) => extractPackageArchives({
|
||||
packageDir: pkg.extractDir,
|
||||
targetDir,
|
||||
cleanupMode: this.settings.cleanupMode,
|
||||
@@ -14419,6 +14390,7 @@ export class DownloadManager extends EventEmitter {
|
||||
maxParallel: this.settings.maxParallelExtract || 2,
|
||||
extractCpuPriority: this.settings.extractCpuPriority,
|
||||
onLog: (level, message) => this.logPackageForPackage(pkg, level, `Nested-Extractor: ${message}`),
|
||||
onOutput: (event) => scope.add(event),
|
||||
onArchiveFailure: (failure) => {
|
||||
nestedFailureCategories.set(failure.archiveName.toLowerCase(), failure.category);
|
||||
},
|
||||
@@ -14448,12 +14420,12 @@ export class DownloadManager extends EventEmitter {
|
||||
extractDir: pkg.extractDir
|
||||
});
|
||||
throwIfAborted();
|
||||
await this.autoRenameExtractedVideoFiles(pkg.extractDir, pkg, shouldAbort, true);
|
||||
await this.autoRenameExtractedVideoFiles(pkg.extractDir, outputScope, pkg, shouldAbort, true);
|
||||
if (this.settings.keepGermanAudioOnly) {
|
||||
pkg.postProcessLabel = "Tonspur...";
|
||||
this.emitState();
|
||||
throwIfAborted();
|
||||
await this.keepGermanAudioOnly(pkg.extractDir, pkg, shouldAbort, deferredController.signal);
|
||||
await this.keepGermanAudioOnly(pkg.extractDir, outputScope, pkg, shouldAbort, deferredController.signal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14488,13 +14460,15 @@ export class DownloadManager extends EventEmitter {
|
||||
if (extractedCount > 0 || alreadyMarkedExtracted) {
|
||||
throwIfAborted();
|
||||
if (this.settings.removeLinkFilesAfterExtract) {
|
||||
const removedLinks = await removeDownloadLinkArtifacts(pkg.extractDir, { shouldAbort });
|
||||
const removedLinks = await removeDownloadLinkArtifactsFromScope(outputScope.completeFiles(), { shouldAbort, rootDir: pkg.extractDir });
|
||||
outputScope.pruneMissing();
|
||||
if (removedLinks > 0) {
|
||||
logger.info(`Deferred Link-Cleanup: pkg=${pkg.name}, entfernt=${removedLinks}`);
|
||||
}
|
||||
}
|
||||
if (this.settings.removeSamplesAfterExtract) {
|
||||
const removedSamples = await removeSampleArtifacts(pkg.extractDir, { shouldAbort });
|
||||
const removedSamples = await removeSampleArtifactsFromScope(outputScope.completeFiles(), { shouldAbort, rootDir: pkg.extractDir });
|
||||
outputScope.pruneMissing();
|
||||
if (removedSamples.files > 0 || removedSamples.dirs > 0) {
|
||||
logger.info(`Deferred Sample-Cleanup: pkg=${pkg.name}, files=${removedSamples.files}, dirs=${removedSamples.dirs}`);
|
||||
}
|
||||
@@ -14554,6 +14528,7 @@ export class DownloadManager extends EventEmitter {
|
||||
this.packageDeferredPostProcessAbortControllers.delete(packageId);
|
||||
}
|
||||
if (this.session.packages[packageId] === pkg && this.getPackagePostProcessVersion(packageId) === deferredVersion) {
|
||||
this.syncPackageOutputScope(pkg, outputScope);
|
||||
pkg.postProcessLabel = undefined;
|
||||
pkg.updatedAt = nowMs();
|
||||
this.persistSoon();
|
||||
|
||||
+137
-44
@@ -5,7 +5,7 @@ import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
||||
import AdmZip from "adm-zip";
|
||||
import { CleanupMode, ConflictMode } from "../shared/types";
|
||||
import { logger } from "./logger";
|
||||
import { removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup";
|
||||
import { removeDownloadLinkArtifactsFromScope, removeSampleArtifactsFromScope } from "./cleanup";
|
||||
import { PackageOutputScope, type ExtractOutputEvent } from "./package-output-scope";
|
||||
|
||||
export type { ExtractOutputEvent } from "./package-output-scope";
|
||||
@@ -170,7 +170,7 @@ type ExtractResumeMember = {
|
||||
type ExtractResumeOutput = {
|
||||
entryPath: string;
|
||||
path: string;
|
||||
disposition: ExtractOutputEvent["disposition"];
|
||||
disposition: Exclude<ExtractOutputEvent["disposition"], "skipped">;
|
||||
};
|
||||
|
||||
type ExtractResumeArchive = {
|
||||
@@ -369,7 +369,7 @@ export async function findArchiveCandidates(packageDir: string): Promise<string[
|
||||
continue;
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
if (!/^\.rd-(?:output|replace)-/i.test(entry.name)) {
|
||||
if (!/^\.rd-(?:output|replace)-/i.test(entry.name) && !/^\.rd-trash$/i.test(entry.name)) {
|
||||
stack.push(fullPath);
|
||||
}
|
||||
} else if (entry.isFile()) {
|
||||
@@ -2155,7 +2155,98 @@ export function buildExternalExtractArgs(
|
||||
|
||||
const overwrite = mode === "overwrite" ? "-aoa" : mode === "rename" ? "-aou" : "-aos";
|
||||
const pass = password ? `-p${password}` : "-p";
|
||||
return ["x", "-y", overwrite, pass, archivePath, `-o${targetDir}`];
|
||||
return ["x", "-y", "-bb1", "-sccUTF-8", overwrite, pass, archivePath, `-o${targetDir}`];
|
||||
}
|
||||
|
||||
export function parseNativeExtractOutput(
|
||||
command: string,
|
||||
line: string,
|
||||
archivePath: string,
|
||||
targetDir: string,
|
||||
conflictMode: ConflictMode
|
||||
): ExtractOutputEvent[] {
|
||||
const trimmed = String(line || "").trim();
|
||||
let reportedPath = "";
|
||||
if (extractorCommandKind(command) === "seven_zip") {
|
||||
const match = trimmed.match(/^[-+]\s+(.+)$/);
|
||||
reportedPath = match?.[1]?.trim() || "";
|
||||
} else if (isRarNativeCommand(command)) {
|
||||
const match = trimmed.match(/^Extracting\s+(.+?)(?:\s+OK)?$/i);
|
||||
reportedPath = match?.[1]?.trim() || "";
|
||||
}
|
||||
if (!reportedPath) {
|
||||
return [];
|
||||
}
|
||||
const targetRoot = path.resolve(targetDir);
|
||||
const rawPath = reportedPath.replace(/^"|"$/g, "");
|
||||
const outputPath = path.isAbsolute(rawPath) ? path.resolve(rawPath) : path.resolve(targetRoot, rawPath);
|
||||
const relativePath = path.relative(targetRoot, outputPath);
|
||||
if (!relativePath
|
||||
|| relativePath === ".."
|
||||
|| relativePath.startsWith(`..${path.sep}`)
|
||||
|| path.isAbsolute(relativePath)) {
|
||||
return [];
|
||||
}
|
||||
const entryPath = relativePath.replace(/\\/g, "/");
|
||||
if (entryPath.split("/").some((segment) => !segment || segment === "..")) {
|
||||
return [];
|
||||
}
|
||||
const mode = effectiveConflictMode(conflictMode);
|
||||
if (mode === "rename" && !/ \(\d+\)(?=\.[^./]+$|$)/.test(path.basename(outputPath))) {
|
||||
return [];
|
||||
}
|
||||
const event: ExtractOutputEvent = {
|
||||
version: 1,
|
||||
archivePath: path.resolve(archivePath),
|
||||
entryPath,
|
||||
outputPath,
|
||||
state: "complete",
|
||||
disposition: mode === "rename" ? "renamed" : mode === "overwrite" ? "overwritten" : "written"
|
||||
};
|
||||
try {
|
||||
const scope = new PackageOutputScope([targetRoot]);
|
||||
scope.add(event);
|
||||
return [event];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function createNativeOutputCollector(
|
||||
command: string,
|
||||
archivePath: string,
|
||||
targetDir: string,
|
||||
conflictMode: ConflictMode,
|
||||
onOutput?: (event: ExtractOutputEvent) => void
|
||||
): { push: (chunk: string) => void; finish: (state: ExtractOutputEvent["state"]) => void } {
|
||||
let buffer = "";
|
||||
const lines = new Set<string>();
|
||||
const collectLine = (value: string): void => {
|
||||
const trimmed = value.trim();
|
||||
if ((extractorCommandKind(command) === "seven_zip" && /^[-+]\s+/.test(trimmed))
|
||||
|| (isRarNativeCommand(command) && /^Extracting\s+/i.test(trimmed))) {
|
||||
lines.add(trimmed);
|
||||
}
|
||||
};
|
||||
return {
|
||||
push: (chunk) => {
|
||||
buffer += chunk;
|
||||
const parts = buffer.split(/[\r\n]+/);
|
||||
buffer = parts.pop() || "";
|
||||
for (const part of parts) {
|
||||
collectLine(part);
|
||||
}
|
||||
},
|
||||
finish: (state) => {
|
||||
collectLine(buffer);
|
||||
buffer = "";
|
||||
for (const outputLine of lines) {
|
||||
for (const event of parseNativeExtractOutput(command, outputLine, archivePath, targetDir, conflictMode)) {
|
||||
onOutput?.({ ...event, state });
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const extractRetryDelay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
@@ -2191,6 +2282,23 @@ async function runExternalExtractInner(
|
||||
const summarizeResultError = (errorText: string): string => cleanErrorText(errorText);
|
||||
let createErrorText = "";
|
||||
let createErrorPassword = "";
|
||||
const runNativeAttempt = async (args: string[]): Promise<ExtractSpawnResult> => {
|
||||
const outputs = createNativeOutputCollector(command, archivePath, targetDir, conflictMode, onOutput);
|
||||
const result = await runExtractCommand(command, args, (chunk) => {
|
||||
outputs.push(chunk);
|
||||
const parsed = parseProgressPercent(chunk);
|
||||
if (parsed === null) {
|
||||
return;
|
||||
}
|
||||
const next = nextArchivePercent(bestPercent, parsed);
|
||||
if (next !== bestPercent) {
|
||||
bestPercent = next;
|
||||
onArchiveProgress?.(bestPercent);
|
||||
}
|
||||
}, signal, timeoutMs);
|
||||
outputs.finish(result.ok ? "complete" : "partial");
|
||||
return result;
|
||||
};
|
||||
|
||||
if (forceFlatMode) {
|
||||
logger.info(`Flat-Modus direkt (gespeichert vom vorherigen Archiv): ${path.basename(archivePath)}`);
|
||||
@@ -2201,12 +2309,7 @@ async function runExternalExtractInner(
|
||||
onLog?.("INFO", `Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length}: archive=${path.basename(archivePath)}, password=<redacted>`);
|
||||
logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length} für ${path.basename(archivePath)} (password=<redacted>)`);
|
||||
const args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, usePerformanceFlags, hybridMode, true);
|
||||
const result = await runExtractCommand(command, args, (chunk) => {
|
||||
const parsed = parseProgressPercent(chunk);
|
||||
if (parsed === null) return;
|
||||
const next = nextArchivePercent(bestPercent, parsed);
|
||||
if (next !== bestPercent) { bestPercent = next; onArchiveProgress?.(bestPercent); }
|
||||
}, signal, timeoutMs);
|
||||
const result = await runNativeAttempt(args);
|
||||
logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length}: ok=${result.ok}, bestPercent=${bestPercent}`);
|
||||
onLog?.("INFO", `Flach-Extraktion Ergebnis ${passwordAttempt}/${passwords.length}: archive=${path.basename(archivePath)}, ok=${result.ok}, timedOut=${result.timedOut}, missingCommand=${result.missingCommand}, bestPercent=${bestPercent}`);
|
||||
if (result.ok) { if (flatModeResult) flatModeResult.needed = true; onArchiveProgress?.(100); return password; }
|
||||
@@ -2233,17 +2336,7 @@ async function runExternalExtractInner(
|
||||
onPasswordAttempt?.(passwordAttempt, passwords.length);
|
||||
}
|
||||
let args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, usePerformanceFlags, hybridMode);
|
||||
let result = await runExtractCommand(command, args, (chunk) => {
|
||||
const parsed = parseProgressPercent(chunk);
|
||||
if (parsed === null) {
|
||||
return;
|
||||
}
|
||||
const next = nextArchivePercent(bestPercent, parsed);
|
||||
if (next !== bestPercent) {
|
||||
bestPercent = next;
|
||||
onArchiveProgress?.(bestPercent);
|
||||
}
|
||||
}, signal, timeoutMs);
|
||||
let result = await runNativeAttempt(args);
|
||||
|
||||
if (!result.ok && usePerformanceFlags && isUnsupportedExtractorSwitchError(result.errorText)) {
|
||||
usePerformanceFlags = false;
|
||||
@@ -2251,17 +2344,7 @@ async function runExternalExtractInner(
|
||||
onLog?.("WARN", `Entpacker ohne Performance-Flags fortgesetzt: ${path.basename(archivePath)}`);
|
||||
logger.warn(`Entpacker ohne Performance-Flags fortgesetzt: ${path.basename(archivePath)}`);
|
||||
args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, false, hybridMode);
|
||||
result = await runExtractCommand(command, args, (chunk) => {
|
||||
const parsed = parseProgressPercent(chunk);
|
||||
if (parsed === null) {
|
||||
return;
|
||||
}
|
||||
const next = nextArchivePercent(bestPercent, parsed);
|
||||
if (next !== bestPercent) {
|
||||
bestPercent = next;
|
||||
onArchiveProgress?.(bestPercent);
|
||||
}
|
||||
}, signal, timeoutMs);
|
||||
result = await runNativeAttempt(args);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
@@ -2324,12 +2407,7 @@ async function runExternalExtractInner(
|
||||
logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length} für ${path.basename(archivePath)} (password=<redacted>)`);
|
||||
onLog?.("INFO", `Flach-Extraktion Versuch ${passwordAttempt}/${flatPasswords.length}: archive=${path.basename(archivePath)}, password=<redacted>`);
|
||||
const args = buildExternalExtractArgs(command, archivePath, targetDir, conflictMode, password, usePerformanceFlags, hybridMode, true);
|
||||
const result = await runExtractCommand(command, args, (chunk) => {
|
||||
const parsed = parseProgressPercent(chunk);
|
||||
if (parsed === null) return;
|
||||
const next = nextArchivePercent(bestPercent, parsed);
|
||||
if (next !== bestPercent) { bestPercent = next; onArchiveProgress?.(bestPercent); }
|
||||
}, signal, timeoutMs);
|
||||
const result = await runNativeAttempt(args);
|
||||
logger.info(`Flach-Extraktion Versuch ${passwordAttempt}/${passwords.length}: ok=${result.ok}, bestPercent=${bestPercent}`);
|
||||
onLog?.("INFO", `Flach-Extraktion Ergebnis ${passwordAttempt}/${flatPasswords.length}: archive=${path.basename(archivePath)}, ok=${result.ok}, timedOut=${result.timedOut}, missingCommand=${result.missingCommand}, bestPercent=${bestPercent}`);
|
||||
if (result.ok) { if (flatModeResult) flatModeResult.needed = true; onArchiveProgress?.(100); return password; }
|
||||
@@ -2439,6 +2517,12 @@ async function runExternalExtract(
|
||||
} else {
|
||||
onLog?.("INFO", `Legacy-Zielpfad unveraendert: archive=${archiveName}, effectiveTargetDir=${effectiveTargetDir}`);
|
||||
}
|
||||
const legacyOnOutput = subst && onOutput
|
||||
? (event: ExtractOutputEvent): void => onOutput({
|
||||
...event,
|
||||
outputPath: path.resolve(targetDir, ...event.entryPath.split("/"))
|
||||
})
|
||||
: onOutput;
|
||||
|
||||
const command = await resolveExtractorCommand(archivePath);
|
||||
const legacyStartedAt = Date.now();
|
||||
@@ -2449,7 +2533,7 @@ async function runExternalExtract(
|
||||
password = await runExternalExtractInner(
|
||||
command, archivePath, effectiveTargetDir, conflictMode, passwordCandidates,
|
||||
onArchiveProgress, signal, timeoutMs, hybridMode, onPasswordAttempt,
|
||||
forceFlatMode, flatModeResult, onLog
|
||||
forceFlatMode, flatModeResult, onLog, legacyOnOutput
|
||||
);
|
||||
} catch (primaryError) {
|
||||
const isRar = /\.rar$/i.test(archiveName) || /\.r\d{2,3}$/i.test(archiveName);
|
||||
@@ -2465,7 +2549,7 @@ async function runExternalExtract(
|
||||
password = await runExternalExtractInner(
|
||||
alt, archivePath, effectiveTargetDir, conflictMode, passwordCandidates,
|
||||
onArchiveProgress, signal, timeoutMs, hybridMode, onPasswordAttempt,
|
||||
forceFlatMode, flatModeResult, onLog
|
||||
forceFlatMode, flatModeResult, onLog, legacyOnOutput
|
||||
);
|
||||
} else {
|
||||
throw primaryError;
|
||||
@@ -2507,7 +2591,8 @@ async function runExternalExtract(
|
||||
onPasswordAttempt,
|
||||
forceFlatMode,
|
||||
flatModeResult,
|
||||
onLog
|
||||
onLog,
|
||||
legacyOnOutput
|
||||
);
|
||||
logger.info(`Legacy-Retry erfolgreich: ${archiveName}`);
|
||||
onLog?.("INFO", `Legacy-Retry erfolgreich: ${archiveName}`);
|
||||
@@ -3660,7 +3745,7 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<E
|
||||
}
|
||||
|
||||
if (extracted > 0) {
|
||||
const hasOutputAfter = await hasAnyFilesRecursive(options.targetDir);
|
||||
const hasOutputAfter = outputScope.completeFiles().length > 0;
|
||||
const hadResumeProgress = resumeCompletedAtStart > 0;
|
||||
if (!hasOutputAfter && conflictMode !== "skip" && !hadResumeProgress) {
|
||||
lastError = "Keine entpackten Dateien erkannt";
|
||||
@@ -3682,11 +3767,19 @@ export async function extractPackageArchives(options: ExtractOptions): Promise<E
|
||||
logger.info(`Archive-Cleanup abgeschlossen: ${removedArchives} Datei(en) entfernt`);
|
||||
}
|
||||
if (options.removeLinks) {
|
||||
const removedLinks = await removeDownloadLinkArtifacts(options.targetDir);
|
||||
const removedLinks = await removeDownloadLinkArtifactsFromScope(outputScope.completeFiles(), {
|
||||
shouldAbort: () => options.signal?.aborted === true,
|
||||
rootDir: options.targetDir
|
||||
});
|
||||
outputScope.pruneMissing();
|
||||
logger.info(`Link-Artefakt-Cleanup: ${removedLinks} Datei(en) entfernt`);
|
||||
}
|
||||
if (options.removeSamples) {
|
||||
const removedSamples = await removeSampleArtifacts(options.targetDir);
|
||||
const removedSamples = await removeSampleArtifactsFromScope(outputScope.completeFiles(), {
|
||||
shouldAbort: () => options.signal?.aborted === true,
|
||||
rootDir: options.targetDir
|
||||
});
|
||||
outputScope.pruneMissing();
|
||||
logger.info(`Sample-Cleanup: ${removedSamples.files} Datei(en), ${removedSamples.dirs} Ordner entfernt`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ export class PackageOutputScope {
|
||||
}
|
||||
|
||||
public archiveFiles(): string[] {
|
||||
return this.completeFiles().filter((filePath) => /\.(?:7z|rar|zip|tar|gz|bz2|xz|001)$/i.test(filePath));
|
||||
return this.completeFiles().filter((filePath) => /\.(?:7z|rar|zip|tar|gz|bz2|xz|tgz|tbz2|txz|001)$/i.test(filePath));
|
||||
}
|
||||
|
||||
public replacePath(sourcePath: string, targetPath: string, state?: ExtractOutputState): boolean {
|
||||
|
||||
+54
-3
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getDebridLinkApiKeyIds } from "../shared/debrid-link-keys";
|
||||
import { getMegaDebridAccountIds, mergeMegaDebridCredentialPools, parseMegaDebridAccounts } from "../shared/mega-debrid-accounts";
|
||||
import { AppSettings, ArchiveOperationMetric, AudioStripSummary, BandwidthScheduleEntry, DailyStartOutcome, DebridAccountStatus, DebridFallbackProvider, DebridProvider, DownloadItem, DownloadStatus, FailurePhase, HistoryEntry, HistoryRetentionMode, LogStorageLocation, PACKAGE_OUTPUT_PROVENANCE_VERSION, PackageEntry, PackagePriority, RemuxOperationMetric, SessionState } from "../shared/types";
|
||||
import { AppSettings, ArchiveOperationMetric, AudioStripSummary, BandwidthScheduleEntry, DailyStartOutcome, DebridAccountStatus, DebridFallbackProvider, DebridProvider, DownloadItem, DownloadStatus, FailurePhase, HistoryEntry, HistoryRetentionMode, LogStorageLocation, PACKAGE_OUTPUT_PROVENANCE_VERSION, PackageEntry, PackageOutputRecord, PackagePriority, RemuxOperationMetric, SessionState } from "../shared/types";
|
||||
import { getProviderUsageDayKey } from "../shared/provider-daily-limits";
|
||||
import { getRealDebridAccountIds, normalizeRealDebridWebAccountIds, parseRealDebridApiAccounts, serializeRealDebridApiAccounts } from "../shared/real-debrid-accounts";
|
||||
import { defaultSettings } from "./constants";
|
||||
@@ -838,6 +838,44 @@ function normalizeRemuxOperations(raw: unknown): RemuxOperationMetric[] {
|
||||
});
|
||||
}
|
||||
|
||||
function normalizePackageOutputRecords(raw: unknown): PackageOutputRecord[] {
|
||||
if (!Array.isArray(raw)) {
|
||||
return [];
|
||||
}
|
||||
const records = new Map<string, PackageOutputRecord>();
|
||||
for (const value of raw.slice(0, 1_000_000)) {
|
||||
const record = asRecord(value);
|
||||
if (!record || Number(record.version) !== 1) {
|
||||
continue;
|
||||
}
|
||||
const archivePath = asText(record.archivePath);
|
||||
const outputPath = asText(record.outputPath);
|
||||
const entryPath = asText(record.entryPath).replace(/\\/g, "/");
|
||||
const state = asText(record.state);
|
||||
const disposition = asText(record.disposition);
|
||||
if (!path.isAbsolute(archivePath)
|
||||
|| !path.isAbsolute(outputPath)
|
||||
|| !entryPath
|
||||
|| entryPath.startsWith("/")
|
||||
|| /^[a-zA-Z]:/.test(entryPath)
|
||||
|| entryPath.split("/").some((segment) => !segment || segment === "..")
|
||||
|| (state !== "complete" && state !== "partial")
|
||||
|| !["written", "overwritten", "renamed", "skipped"].includes(disposition)) {
|
||||
continue;
|
||||
}
|
||||
const key = path.resolve(outputPath).toLocaleLowerCase("en-US");
|
||||
records.set(key, {
|
||||
version: 1,
|
||||
archivePath: path.resolve(archivePath),
|
||||
entryPath,
|
||||
outputPath: path.resolve(outputPath),
|
||||
state,
|
||||
disposition: disposition as PackageOutputRecord["disposition"]
|
||||
});
|
||||
}
|
||||
return [...records.values()];
|
||||
}
|
||||
|
||||
function optionalClampedNumber(record: Record<string, unknown>, key: string, max = Number.MAX_SAFE_INTEGER): number | undefined {
|
||||
return Object.prototype.hasOwnProperty.call(record, key)
|
||||
? clampNumber(record[key], 0, 0, max)
|
||||
@@ -974,9 +1012,18 @@ export function normalizeLoadedSession(raw: unknown): SessionState {
|
||||
const statusRaw = asText(pkg.status) as DownloadStatus;
|
||||
const status: DownloadStatus = VALID_DOWNLOAD_STATUSES.has(statusRaw) ? statusRaw : "queued";
|
||||
const rawItemIds = Array.isArray(pkg.itemIds) ? pkg.itemIds : [];
|
||||
const outputProvenance = Array.isArray(pkg.outputProvenance)
|
||||
const normalizedOutputProvenance = Array.isArray(pkg.outputProvenance)
|
||||
? [...new Set(pkg.outputProvenance.map((value) => asText(value).toLowerCase()).filter((value) => /^[a-f0-9]{64}$/.test(value)))].slice(0, 1_000_000)
|
||||
: [];
|
||||
const hasOutputProvenanceVersion = pkg.outputProvenanceVersion !== undefined && pkg.outputProvenanceVersion !== null;
|
||||
const rawOutputProvenanceVersion = Number(pkg.outputProvenanceVersion);
|
||||
const unknownOutputProvenanceVersion = hasOutputProvenanceVersion
|
||||
&& rawOutputProvenanceVersion !== PACKAGE_OUTPUT_PROVENANCE_VERSION;
|
||||
const outputProvenance = unknownOutputProvenanceVersion ? [] : normalizedOutputProvenance;
|
||||
const outputRecords = unknownOutputProvenanceVersion
|
||||
|| (!hasOutputProvenanceVersion && outputProvenance.length === 0)
|
||||
? []
|
||||
: normalizePackageOutputRecords(pkg.outputRecords);
|
||||
packagesById[id] = {
|
||||
id,
|
||||
name: asText(pkg.name) || "Paket",
|
||||
@@ -1010,8 +1057,12 @@ export function normalizeLoadedSession(raw: unknown): SessionState {
|
||||
archiveOperations: normalizeArchiveOperations(pkg.archiveOperations),
|
||||
remuxOperations: normalizeRemuxOperations(pkg.remuxOperations),
|
||||
outputCount: outputProvenance.length,
|
||||
outputProvenanceVersion: PACKAGE_OUTPUT_PROVENANCE_VERSION,
|
||||
outputProvenanceVersion: unknownOutputProvenanceVersion
|
||||
? rawOutputProvenanceVersion
|
||||
: PACKAGE_OUTPUT_PROVENANCE_VERSION,
|
||||
outputProvenance,
|
||||
outputRecords,
|
||||
outputScopeAdopted: Boolean(pkg.outputScopeAdopted),
|
||||
cleanupErrorCategory: asText(pkg.cleanupErrorCategory),
|
||||
resultGeneration: clampNumber(pkg.resultGeneration, 1, 1, Number.MAX_SAFE_INTEGER),
|
||||
createdAt: clampNumber(pkg.createdAt, now, 0, Number.MAX_SAFE_INTEGER),
|
||||
|
||||
@@ -497,6 +497,15 @@ export type PackageResultStatus = "completed" | "partial" | "failed" | "cancelle
|
||||
export type FailurePhase = "download" | "extract" | "remux" | "cleanup" | null;
|
||||
export const PACKAGE_OUTPUT_PROVENANCE_VERSION = 1;
|
||||
|
||||
export interface PackageOutputRecord {
|
||||
version: 1;
|
||||
archivePath: string;
|
||||
entryPath: string;
|
||||
outputPath: string;
|
||||
state: "complete" | "partial";
|
||||
disposition: "written" | "overwritten" | "renamed" | "skipped";
|
||||
}
|
||||
|
||||
export interface ArchiveOperationMetric {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -591,6 +600,8 @@ export interface PackageEntry {
|
||||
outputCount?: number;
|
||||
outputProvenanceVersion?: number;
|
||||
outputProvenance?: string[];
|
||||
outputRecords?: PackageOutputRecord[];
|
||||
outputScopeAdopted?: boolean;
|
||||
cleanupErrorCategory?: string;
|
||||
resultGeneration?: number;
|
||||
createdAt: number;
|
||||
|
||||
+33
-1
@@ -2,7 +2,13 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { cleanupCancelledPackageArtifacts, removeDownloadLinkArtifacts, removeSampleArtifacts } from "../src/main/cleanup";
|
||||
import {
|
||||
cleanupCancelledPackageArtifacts,
|
||||
removeDownloadLinkArtifacts,
|
||||
removeDownloadLinkArtifactsFromScope,
|
||||
removeSampleArtifacts,
|
||||
removeSampleArtifactsFromScope
|
||||
} from "../src/main/cleanup";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
@@ -97,4 +103,30 @@ describe("cleanup", () => {
|
||||
expect(result.files).toBe(0);
|
||||
expect(fs.existsSync(outsideFile)).toBe(true);
|
||||
});
|
||||
|
||||
it("removes only scoped link and sample outputs from a shared root", async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rd-clean-scope-"));
|
||||
tempDirs.push(dir);
|
||||
const sampleDir = path.join(dir, "Samples");
|
||||
fs.mkdirSync(sampleDir, { recursive: true });
|
||||
const ownedLink = path.join(dir, "owned.url");
|
||||
const foreignLink = path.join(dir, "foreign.url");
|
||||
const ownedSample = path.join(sampleDir, "owned-sample.mkv");
|
||||
const foreignSample = path.join(sampleDir, "foreign-sample.mkv");
|
||||
fs.writeFileSync(ownedLink, "owned");
|
||||
fs.writeFileSync(foreignLink, "foreign");
|
||||
fs.writeFileSync(ownedSample, "owned");
|
||||
fs.writeFileSync(foreignSample, "foreign");
|
||||
|
||||
const removedLinks = await removeDownloadLinkArtifactsFromScope([ownedLink]);
|
||||
const removedSamples = await removeSampleArtifactsFromScope([ownedSample]);
|
||||
|
||||
expect(removedLinks).toBe(1);
|
||||
expect(removedSamples).toEqual({ files: 1, dirs: 0 });
|
||||
expect(fs.existsSync(ownedLink)).toBe(false);
|
||||
expect(fs.existsSync(ownedSample)).toBe(false);
|
||||
expect(fs.existsSync(foreignLink)).toBe(true);
|
||||
expect(fs.existsSync(foreignSample)).toBe(true);
|
||||
expect(fs.existsSync(sampleDir)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -587,7 +587,16 @@ describe("disk write recovery", () => {
|
||||
})
|
||||
});
|
||||
|
||||
const processed = await (manager as any).keepGermanAudioOnlyImpl(extractDir, pkg);
|
||||
const outputScope = (manager as any).getPackageOutputScope(pkg);
|
||||
outputScope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(pkg.outputDir, "archive.rar"),
|
||||
entryPath: path.basename(sourcePath),
|
||||
outputPath: sourcePath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
const processed = await (manager as any).keepGermanAudioOnlyImpl(extractDir, outputScope, pkg);
|
||||
|
||||
expect(processed).toBe(0);
|
||||
expect(fs.existsSync(sourcePath)).toBe(true);
|
||||
@@ -12758,7 +12767,7 @@ describe("download manager", () => {
|
||||
void manager;
|
||||
}, 20000);
|
||||
|
||||
it("collect cleans a raw file sitting OUTSIDE extractDir (Downloader-Unfertig case) AND its .srt follows the rename", async () => {
|
||||
it("does not collect unscoped raw files outside the package extract directory", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
|
||||
@@ -12808,10 +12817,9 @@ describe("download manager", () => {
|
||||
|
||||
await (manager as any).collectMkvFilesToLibrary(packageId, session.packages[packageId], undefined, false);
|
||||
|
||||
const cleanBase = "Fritzie.-.Der.Himmel.muss.warten.S04E01.GERMAN.720p.WEB.AVC-4SF";
|
||||
expect(fs.existsSync(path.join(mkvLibraryDir, `${cleanBase}.mkv`))).toBe(true);
|
||||
expect(fs.existsSync(path.join(mkvLibraryDir, rawName))).toBe(false);
|
||||
expect(fs.existsSync(path.join(mkvLibraryDir, `${cleanBase}.de.srt`))).toBe(true);
|
||||
expect(fs.existsSync(path.join(outputDir, rawName))).toBe(true);
|
||||
expect(fs.existsSync(path.join(outputDir, rawSrt))).toBe(true);
|
||||
expect(fs.existsSync(mkvLibraryDir) ? fs.readdirSync(mkvLibraryDir) : []).toEqual([]);
|
||||
|
||||
void manager;
|
||||
}, 20000);
|
||||
@@ -13321,6 +13329,51 @@ describe("download manager", () => {
|
||||
void manager;
|
||||
}, 20000);
|
||||
|
||||
it("does not adopt unscoped files from an extraction root shared by packages", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-shared-adoption-"));
|
||||
tempDirs.push(root);
|
||||
const sharedExtractDir = path.join(root, "shared");
|
||||
const libraryDir = path.join(root, "library");
|
||||
fs.mkdirSync(sharedExtractDir, { recursive: true });
|
||||
const foreignPath = path.join(sharedExtractDir, "foreign.mkv");
|
||||
fs.writeFileSync(foreignPath, "foreign");
|
||||
const session = emptySession();
|
||||
for (const id of ["package-a", "package-b"]) {
|
||||
session.packages[id] = {
|
||||
id,
|
||||
name: id,
|
||||
outputDir: path.join(root, "downloads", id),
|
||||
extractDir: sharedExtractDir,
|
||||
status: "completed",
|
||||
itemIds: [],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
outputProvenanceVersion: 1,
|
||||
outputRecords: [],
|
||||
createdAt: 1_000,
|
||||
updatedAt: 1_000
|
||||
};
|
||||
}
|
||||
session.packageOrder = ["package-a", "package-b"];
|
||||
const manager = new DownloadManager(
|
||||
{
|
||||
...defaultSettings(),
|
||||
autoExtract: true,
|
||||
createExtractSubfolder: true,
|
||||
collectMkvToLibrary: true,
|
||||
mkvLibraryDir: libraryDir
|
||||
},
|
||||
session,
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
|
||||
await (manager as any).collectMkvFilesToLibrary("package-a", session.packages["package-a"]);
|
||||
|
||||
expect(fs.existsSync(foreignPath)).toBe(true);
|
||||
expect(fs.existsSync(path.join(libraryDir, "foreign.mkv"))).toBe(false);
|
||||
expect(session.packages["package-a"].outputRecords).toEqual([]);
|
||||
});
|
||||
|
||||
it("does NOT move bonus files from Extras subdirectory to flat library", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
@@ -15114,7 +15167,7 @@ describe("package priority ordering", () => {
|
||||
});
|
||||
|
||||
describe("package lifecycle telemetry boundaries", () => {
|
||||
it("captures shared-root provenance from package staging without scanning unrelated files", async () => {
|
||||
it("captures direct package output scopes concurrently without scanning a shared root", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-provenance-lock-"));
|
||||
tempDirs.push(root);
|
||||
const extractDir = path.join(root, "extract");
|
||||
@@ -15122,17 +15175,10 @@ describe("package lifecycle telemetry boundaries", () => {
|
||||
for (let index = 0; index < 2_000; index += 1) {
|
||||
fs.writeFileSync(path.join(extractDir, `foreign-${index}.txt`), "foreign");
|
||||
}
|
||||
const traversedDirectories: string[] = [];
|
||||
const manager = new DownloadManager(
|
||||
{ ...defaultSettings(), extractConflictMode: "overwrite" },
|
||||
emptySession(),
|
||||
createStoragePaths(path.join(root, "state")),
|
||||
{
|
||||
readOutputDirectory: async (directory: string) => {
|
||||
traversedDirectories.push(path.resolve(directory));
|
||||
return fs.promises.readdir(directory, { withFileTypes: true });
|
||||
}
|
||||
} as any
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
const createPackage = (id: string): PackageEntry => ({
|
||||
id,
|
||||
@@ -15155,79 +15201,51 @@ describe("package lifecycle telemetry boundaries", () => {
|
||||
let enteredB = false;
|
||||
const state = manager as any;
|
||||
|
||||
const first = state.runWithPackageOutputProvenance(packageA, async (operationTarget = extractDir) => {
|
||||
fs.writeFileSync(path.join(operationTarget, "package-a.mkv"), "a");
|
||||
const first = state.runWithPackageOutputProvenance(packageA, async (operationTarget: string, scope: any) => {
|
||||
const outputPath = path.join(operationTarget, "package-a.mkv");
|
||||
fs.writeFileSync(outputPath, "a");
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(packageA.outputDir, "archive.rar"),
|
||||
entryPath: "package-a.mkv",
|
||||
outputPath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
await gateA;
|
||||
});
|
||||
await vi.waitFor(() => expect(enteredB).toBe(false));
|
||||
const second = state.runWithPackageOutputProvenance(packageB, async (operationTarget = extractDir) => {
|
||||
const second = state.runWithPackageOutputProvenance(packageB, async (operationTarget: string, scope: any) => {
|
||||
enteredB = true;
|
||||
fs.writeFileSync(path.join(operationTarget, "package-b.mkv"), "b");
|
||||
const outputPath = path.join(operationTarget, "package-b.mkv");
|
||||
fs.writeFileSync(outputPath, "b");
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(packageB.outputDir, "archive.rar"),
|
||||
entryPath: "package-b.mkv",
|
||||
outputPath,
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
});
|
||||
});
|
||||
await vi.waitFor(() => expect(enteredB).toBe(true));
|
||||
releaseA();
|
||||
await Promise.all([first, second]);
|
||||
expect(packageA.outputCount).toBe(1);
|
||||
expect(packageB.outputCount).toBe(1);
|
||||
expect(traversedDirectories.length).toBeGreaterThan(0);
|
||||
expect(traversedDirectories).not.toContain(path.resolve(extractDir));
|
||||
expect(traversedDirectories.length).toBeLessThanOrEqual(4);
|
||||
expect(packageA.outputRecords).toEqual([expect.objectContaining({ outputPath: path.join(extractDir, "package-a.mkv") })]);
|
||||
expect(packageB.outputRecords).toEqual([expect.objectContaining({ outputPath: path.join(extractDir, "package-b.mkv") })]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["overwrite", "package", ["episode.mkv"]],
|
||||
["skip", "foreign", ["episode.mkv"]],
|
||||
["rename", "foreign", ["episode (1).mkv", "episode.mkv"]]
|
||||
] as const)("preserves %s conflicts while merging staged package outputs", async (conflictMode, expectedOriginal, expectedFiles) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), `rd-output-${conflictMode}-`));
|
||||
tempDirs.push(root);
|
||||
const extractDir = path.join(root, "extract");
|
||||
fs.mkdirSync(extractDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(extractDir, "episode.mkv"), "foreign");
|
||||
const manager = new DownloadManager(
|
||||
{ ...defaultSettings(), extractConflictMode: conflictMode },
|
||||
emptySession(),
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
const pkg: PackageEntry = {
|
||||
id: `conflict-${conflictMode}`,
|
||||
name: `conflict-${conflictMode}`,
|
||||
outputDir: path.join(root, "downloads"),
|
||||
extractDir,
|
||||
status: "completed",
|
||||
itemIds: [],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
createdAt: 1_000,
|
||||
updatedAt: 1_000
|
||||
};
|
||||
const state = manager as any;
|
||||
|
||||
await state.runWithPackageOutputProvenance(pkg, async (operationTarget = extractDir) => {
|
||||
fs.writeFileSync(path.join(operationTarget, "episode.mkv"), "package");
|
||||
});
|
||||
|
||||
expect(fs.readFileSync(path.join(extractDir, "episode.mkv"), "utf8")).toBe(expectedOriginal);
|
||||
expect(fs.readdirSync(extractDir).filter((name) => name.endsWith(".mkv")).sort()).toEqual([...expectedFiles]);
|
||||
expect(pkg.outputCount).toBe(conflictMode === "skip" ? 0 : 1);
|
||||
});
|
||||
|
||||
it("retains partial staged outputs deterministically when extraction aborts", async () => {
|
||||
it("retains directly reported partial outputs when extraction aborts", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-output-abort-"));
|
||||
tempDirs.push(root);
|
||||
const extractDir = path.join(root, "extract");
|
||||
fs.mkdirSync(extractDir, { recursive: true });
|
||||
const traversedDirectories: string[] = [];
|
||||
const manager = new DownloadManager(
|
||||
defaultSettings(),
|
||||
emptySession(),
|
||||
createStoragePaths(path.join(root, "state")),
|
||||
{
|
||||
readOutputDirectory: async (directory: string) => {
|
||||
traversedDirectories.push(path.resolve(directory));
|
||||
return fs.promises.readdir(directory, { withFileTypes: true });
|
||||
}
|
||||
} as any
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
const pkg: PackageEntry = {
|
||||
id: "aborted-output",
|
||||
@@ -15243,16 +15261,23 @@ describe("package lifecycle telemetry boundaries", () => {
|
||||
};
|
||||
const state = manager as any;
|
||||
|
||||
await expect(state.runWithPackageOutputProvenance(pkg, async (operationTarget = extractDir) => {
|
||||
fs.writeFileSync(path.join(operationTarget, "partial.mkv"), "partial");
|
||||
await expect(state.runWithPackageOutputProvenance(pkg, async (operationTarget: string, scope: any) => {
|
||||
const outputPath = path.join(operationTarget, "partial.mkv");
|
||||
fs.writeFileSync(outputPath, "partial");
|
||||
scope.add({
|
||||
version: 1,
|
||||
archivePath: path.join(pkg.outputDir, "archive.rar"),
|
||||
entryPath: "partial.mkv",
|
||||
outputPath,
|
||||
state: "partial",
|
||||
disposition: "written"
|
||||
});
|
||||
throw new Error("aborted:extract");
|
||||
})).rejects.toThrow("aborted:extract");
|
||||
|
||||
expect(fs.readFileSync(path.join(extractDir, "partial.mkv"), "utf8")).toBe("partial");
|
||||
expect(pkg.outputCount).toBe(1);
|
||||
expect(traversedDirectories.length).toBeGreaterThan(0);
|
||||
expect(traversedDirectories).not.toContain(path.resolve(extractDir));
|
||||
expect(fs.readdirSync(extractDir).filter((name) => name.startsWith(".rd-output-"))).toEqual([]);
|
||||
expect(pkg.outputRecords).toEqual([expect.objectContaining({ state: "partial", outputPath: path.join(extractDir, "partial.mkv") })]);
|
||||
});
|
||||
|
||||
it("uses normalized nested item paths for archive identity and leaves empty item provenance at zero", () => {
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
shouldSerialRetryParallelFailures,
|
||||
findArchiveCandidates,
|
||||
orderExtractorCandidatesForArchive,
|
||||
parseNativeExtractOutput,
|
||||
resolveExtractorBackendModeForArchive,
|
||||
resolveExtractorBackendMode,
|
||||
shouldFallbackLegacyRarToJvm,
|
||||
@@ -1467,5 +1468,32 @@ describe("extractor", () => {
|
||||
expect(fs.existsSync(path.join(targetDir, "second.txt"))).toBe(false);
|
||||
});
|
||||
|
||||
it("strictly parses native output paths and fails closed for ambiguous rename output", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-native-output-"));
|
||||
tempDirs.push(root);
|
||||
const targetDir = path.join(root, "out");
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
const archivePath = path.join(root, "archive.7z");
|
||||
const exactPath = path.join(targetDir, "folder", "episode.mkv");
|
||||
fs.mkdirSync(path.dirname(exactPath), { recursive: true });
|
||||
fs.writeFileSync(exactPath, "video");
|
||||
|
||||
expect(parseNativeExtractOutput("7z.exe", "- folder\\episode.mkv", archivePath, targetDir, "overwrite")).toEqual([
|
||||
expect.objectContaining({ entryPath: "folder/episode.mkv", outputPath: exactPath, disposition: "overwritten" })
|
||||
]);
|
||||
expect(parseNativeExtractOutput("UnRAR.exe", `Extracting ${exactPath} OK`, archivePath, targetDir, "overwrite")).toEqual([
|
||||
expect.objectContaining({ entryPath: "folder/episode.mkv", outputPath: exactPath })
|
||||
]);
|
||||
expect(parseNativeExtractOutput("7z.exe", "- ..\\foreign.mkv", archivePath, targetDir, "overwrite")).toEqual([]);
|
||||
|
||||
const renamedPath = path.join(targetDir, "episode (1).mkv");
|
||||
fs.writeFileSync(path.join(targetDir, "episode.mkv"), "foreign");
|
||||
fs.writeFileSync(renamedPath, "owned");
|
||||
expect(parseNativeExtractOutput("7z.exe", "- episode.mkv", archivePath, targetDir, "rename")).toEqual([]);
|
||||
expect(parseNativeExtractOutput("7z.exe", "- episode (1).mkv", archivePath, targetDir, "rename")).toEqual([
|
||||
expect.objectContaining({ outputPath: renamedPath, disposition: "renamed" })
|
||||
]);
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1271,6 +1271,101 @@ describe("settings storage", () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it("fails closed for unknown package output provenance versions without relabeling", () => {
|
||||
const normalized = normalizeLoadedSession({
|
||||
version: 2,
|
||||
packageOrder: ["future-output"],
|
||||
packages: {
|
||||
"future-output": {
|
||||
id: "future-output",
|
||||
name: "Future output",
|
||||
outputDir: "C:\\Downloads\\Future",
|
||||
extractDir: "C:\\Downloads\\Shared",
|
||||
status: "completed",
|
||||
itemIds: [],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
outputCount: 2,
|
||||
outputProvenanceVersion: 99,
|
||||
outputProvenance: ["a".repeat(64), "b".repeat(64)],
|
||||
outputRecords: [{
|
||||
version: 1,
|
||||
archivePath: "C:\\Downloads\\Future\\archive.rar",
|
||||
entryPath: "episode.mkv",
|
||||
outputPath: "C:\\Downloads\\Shared\\episode.mkv",
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
}],
|
||||
createdAt: 1_000,
|
||||
updatedAt: 2_000
|
||||
}
|
||||
},
|
||||
items: {},
|
||||
runStartedAt: 0,
|
||||
totalDownloadedBytes: 0,
|
||||
summaryText: "",
|
||||
reconnectUntil: 0,
|
||||
reconnectReason: "",
|
||||
paused: false,
|
||||
running: false,
|
||||
updatedAt: 2_000
|
||||
});
|
||||
|
||||
expect(normalized.packages["future-output"]).toEqual(expect.objectContaining({
|
||||
outputCount: 0,
|
||||
outputProvenanceVersion: 99,
|
||||
outputProvenance: [],
|
||||
outputRecords: []
|
||||
}));
|
||||
});
|
||||
|
||||
it("migrates unversioned valid provenance hashes and concrete output records to v1", () => {
|
||||
const normalized = normalizeLoadedSession({
|
||||
version: 2,
|
||||
packageOrder: ["legacy-hashes"],
|
||||
packages: {
|
||||
"legacy-hashes": {
|
||||
id: "legacy-hashes",
|
||||
name: "Legacy hashes",
|
||||
outputDir: "C:\\Downloads\\Legacy",
|
||||
extractDir: "C:\\Downloads\\Shared",
|
||||
status: "completed",
|
||||
itemIds: [],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
outputCount: 40_000,
|
||||
outputProvenance: ["a".repeat(64)],
|
||||
outputRecords: [{
|
||||
version: 1,
|
||||
archivePath: "C:\\Downloads\\Legacy\\archive.rar",
|
||||
entryPath: "episode.mkv",
|
||||
outputPath: "C:\\Downloads\\Shared\\episode.mkv",
|
||||
state: "complete",
|
||||
disposition: "written"
|
||||
}],
|
||||
createdAt: 1_000,
|
||||
updatedAt: 2_000
|
||||
}
|
||||
},
|
||||
items: {},
|
||||
runStartedAt: 0,
|
||||
totalDownloadedBytes: 0,
|
||||
summaryText: "",
|
||||
reconnectUntil: 0,
|
||||
reconnectReason: "",
|
||||
paused: false,
|
||||
running: false,
|
||||
updatedAt: 2_000
|
||||
});
|
||||
|
||||
expect(normalized.packages["legacy-hashes"]).toEqual(expect.objectContaining({
|
||||
outputCount: 1,
|
||||
outputProvenanceVersion: 1,
|
||||
outputProvenance: ["a".repeat(64)],
|
||||
outputRecords: [expect.objectContaining({ entryPath: "episode.mkv", state: "complete" })]
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips adding persisted history entries when history retention is never", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rd-store-"));
|
||||
tempDirs.push(dir);
|
||||
|
||||
Reference in New Issue
Block a user