Fix archive underflow and extraction readiness
This commit is contained in:
parent
0f2e8d5567
commit
16bfbfc106
@ -4007,12 +4007,15 @@ export class DownloadManager extends EventEmitter {
|
|||||||
* old 50% recovery threshold). Reset to "queued" so it gets re-downloaded. */
|
* old 50% recovery threshold). Reset to "queued" so it gets re-downloaded. */
|
||||||
private revalidateCompletedItems(): void {
|
private revalidateCompletedItems(): void {
|
||||||
let fixed = 0;
|
let fixed = 0;
|
||||||
|
const touchedPackageIds = new Set<string>();
|
||||||
for (const item of Object.values(this.session.items)) {
|
for (const item of Object.values(this.session.items)) {
|
||||||
if (item.status !== "completed") continue;
|
if (item.status !== "completed") continue;
|
||||||
if (!item.targetPath || !item.totalBytes || item.totalBytes <= 0) continue;
|
if (!item.targetPath || !item.totalBytes || item.totalBytes <= 0) continue;
|
||||||
try {
|
try {
|
||||||
const stat = fs.statSync(item.targetPath);
|
const stat = fs.statSync(item.targetPath);
|
||||||
if (stat.size < item.totalBytes - ALLOCATION_UNIT_SIZE) {
|
const expectedMinSize = item.totalBytes - ALLOCATION_UNIT_SIZE;
|
||||||
|
const persistedShortfall = item.downloadedBytes < expectedMinSize && stat.size >= expectedMinSize;
|
||||||
|
if (stat.size < expectedMinSize) {
|
||||||
logger.warn(`revalidateCompleted: ${item.fileName} ist nur ${humanSize(stat.size)} statt ${humanSize(item.totalBytes)}, setze auf queued`);
|
logger.warn(`revalidateCompleted: ${item.fileName} ist nur ${humanSize(stat.size)} statt ${humanSize(item.totalBytes)}, setze auf queued`);
|
||||||
item.status = "queued";
|
item.status = "queued";
|
||||||
item.fullStatus = "Wartet";
|
item.fullStatus = "Wartet";
|
||||||
@ -4020,6 +4023,15 @@ export class DownloadManager extends EventEmitter {
|
|||||||
item.progressPercent = Math.floor((stat.size / item.totalBytes) * 100);
|
item.progressPercent = Math.floor((stat.size / item.totalBytes) * 100);
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
fixed += 1;
|
fixed += 1;
|
||||||
|
touchedPackageIds.add(item.packageId);
|
||||||
|
} else if (persistedShortfall) {
|
||||||
|
logger.warn(`revalidateCompleted: ${item.fileName} wirkt pre-alloc/unvollständig (stat=${humanSize(stat.size)}, bytes=${humanSize(item.downloadedBytes)}, total=${humanSize(item.totalBytes)}), setze auf queued`);
|
||||||
|
item.status = "queued";
|
||||||
|
item.fullStatus = "Wartet (Auto-Recovery: pre-alloc)";
|
||||||
|
item.progressPercent = Math.max(0, Math.min(99, Math.floor((Math.max(0, item.downloadedBytes) / item.totalBytes) * 100)));
|
||||||
|
item.speedBps = 0;
|
||||||
|
fixed += 1;
|
||||||
|
touchedPackageIds.add(item.packageId);
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// file doesn't exist — reset to queued so it gets re-downloaded
|
// file doesn't exist — reset to queued so it gets re-downloaded
|
||||||
@ -4030,9 +4042,16 @@ export class DownloadManager extends EventEmitter {
|
|||||||
item.progressPercent = 0;
|
item.progressPercent = 0;
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
fixed += 1;
|
fixed += 1;
|
||||||
|
touchedPackageIds.add(item.packageId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (fixed > 0) {
|
if (fixed > 0) {
|
||||||
|
for (const packageId of touchedPackageIds) {
|
||||||
|
const pkg = this.session.packages[packageId];
|
||||||
|
if (pkg) {
|
||||||
|
this.refreshPackageStatus(pkg);
|
||||||
|
}
|
||||||
|
}
|
||||||
logger.info(`revalidateCompletedItems: ${fixed} Items korrigiert`);
|
logger.info(`revalidateCompletedItems: ${fixed} Items korrigiert`);
|
||||||
this.persistSoon();
|
this.persistSoon();
|
||||||
}
|
}
|
||||||
@ -6530,6 +6549,23 @@ export class DownloadManager extends EventEmitter {
|
|||||||
throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`);
|
throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const exactLengthRequired = isLargeBinaryLikePath(item.fileName || effectiveTargetPath);
|
||||||
|
if (item.totalBytes && item.totalBytes > 0 && written < item.totalBytes) {
|
||||||
|
const shortfall = item.totalBytes - written;
|
||||||
|
if (preAllocated) {
|
||||||
|
try {
|
||||||
|
await fs.promises.truncate(effectiveTargetPath, written);
|
||||||
|
} catch { /* best-effort */ }
|
||||||
|
}
|
||||||
|
logger.warn(`Download-Underflow: erwartet=${item.totalBytes}, erhalten=${written}, shortfall=${shortfall} fuer ${item.fileName}`);
|
||||||
|
if (exactLengthRequired || shortfall > ALLOCATION_UNIT_SIZE) {
|
||||||
|
item.downloadedBytes = written;
|
||||||
|
item.progressPercent = Math.max(0, Math.min(99, Math.floor((written / item.totalBytes) * 100)));
|
||||||
|
item.speedBps = 0;
|
||||||
|
throw new Error(`download_underflow:${written}/${item.totalBytes}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Truncate pre-allocated files to actual bytes written to prevent zero-padded tail
|
// Truncate pre-allocated files to actual bytes written to prevent zero-padded tail
|
||||||
if (preAllocated && item.totalBytes && written < item.totalBytes) {
|
if (preAllocated && item.totalBytes && written < item.totalBytes) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@ -536,8 +536,8 @@ export function classifyExtractionError(errorText: string): ExtractErrorCategory
|
|||||||
const text = String(errorText || "").toLowerCase();
|
const text = String(errorText || "").toLowerCase();
|
||||||
if (text.includes("aborted:extract") || text.includes("extract_aborted")) return "aborted";
|
if (text.includes("aborted:extract") || text.includes("extract_aborted")) return "aborted";
|
||||||
if (text.includes("timeout")) return "timeout";
|
if (text.includes("timeout")) return "timeout";
|
||||||
if (text.includes("wrong password") || text.includes("falsches passwort") || text.includes("incorrect password")) return "wrong_password";
|
|
||||||
if (text.includes("crc failed") || text.includes("checksum error") || text.includes("crc error")) return "crc_error";
|
if (text.includes("crc failed") || text.includes("checksum error") || text.includes("crc error")) return "crc_error";
|
||||||
|
if (text.includes("wrong password") || text.includes("falsches passwort") || text.includes("incorrect password")) return "wrong_password";
|
||||||
if (text.includes("missing volume") || text.includes("next volume") || text.includes("unexpected end of archive") || text.includes("missing parts")) return "missing_parts";
|
if (text.includes("missing volume") || text.includes("next volume") || text.includes("unexpected end of archive") || text.includes("missing parts")) return "missing_parts";
|
||||||
if (text.includes("nicht gefunden") || text.includes("not found") || text.includes("no extractor")) return "no_extractor";
|
if (text.includes("nicht gefunden") || text.includes("not found") || text.includes("no extractor")) return "no_extractor";
|
||||||
if (text.includes("kein rar-archiv") || text.includes("not a rar archive") || text.includes("unsupported") || text.includes("unsupportedmethod")) return "unsupported_format";
|
if (text.includes("kein rar-archiv") || text.includes("not a rar archive") || text.includes("unsupported") || text.includes("unsupportedmethod")) return "unsupported_format";
|
||||||
@ -937,9 +937,12 @@ type JvmExtractResult = {
|
|||||||
backend: string;
|
backend: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
function extractorBackendMode(): ExtractBackendMode {
|
export function resolveExtractorBackendMode(
|
||||||
const defaultMode = "legacy";
|
rawValue?: string | null,
|
||||||
const raw = String(process.env.RD_EXTRACT_BACKEND || defaultMode).trim().toLowerCase();
|
isVitestEnv = Boolean(process.env.VITEST)
|
||||||
|
): ExtractBackendMode {
|
||||||
|
const defaultMode: ExtractBackendMode = isVitestEnv ? "legacy" : "auto";
|
||||||
|
const raw = String(rawValue ?? defaultMode).trim().toLowerCase();
|
||||||
if (raw === "legacy") {
|
if (raw === "legacy") {
|
||||||
return "legacy";
|
return "legacy";
|
||||||
}
|
}
|
||||||
@ -949,6 +952,10 @@ function extractorBackendMode(): ExtractBackendMode {
|
|||||||
return "auto";
|
return "auto";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractorBackendMode(): ExtractBackendMode {
|
||||||
|
return resolveExtractorBackendMode(process.env.RD_EXTRACT_BACKEND);
|
||||||
|
}
|
||||||
|
|
||||||
function isJvmRuntimeMissingError(errorText: string): boolean {
|
function isJvmRuntimeMissingError(errorText: string): boolean {
|
||||||
const text = String(errorText || "").toLowerCase();
|
const text = String(errorText || "").toLowerCase();
|
||||||
return text.includes("could not find or load main class")
|
return text.includes("could not find or load main class")
|
||||||
|
|||||||
@ -288,6 +288,80 @@ describe("download manager", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not mark truncated archive downloads as completed", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const advertised = Buffer.alloc(96 * 1024, 5);
|
||||||
|
const actual = advertised.subarray(0, advertised.length - 2048);
|
||||||
|
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
if ((req.url || "") !== "/short-archive") {
|
||||||
|
res.statusCode = 404;
|
||||||
|
res.end("not-found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.statusCode = 200;
|
||||||
|
res.setHeader("Accept-Ranges", "bytes");
|
||||||
|
res.setHeader("Content-Length", String(actual.length));
|
||||||
|
res.end(actual);
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(0, "127.0.0.1");
|
||||||
|
await once(server, "listening");
|
||||||
|
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") {
|
||||||
|
throw new Error("server address unavailable");
|
||||||
|
}
|
||||||
|
const directUrl = `http://127.0.0.1:${address.port}/short-archive`;
|
||||||
|
|
||||||
|
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||||
|
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||||
|
if (url.includes("/unrestrict/link")) {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
download: directUrl,
|
||||||
|
filename: "broken.part01.rar",
|
||||||
|
filesize: advertised.length
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
status: 200,
|
||||||
|
headers: { "Content-Type": "application/json" }
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return originalFetch(input, init);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const manager = new DownloadManager(
|
||||||
|
{
|
||||||
|
...defaultSettings(),
|
||||||
|
token: "rd-token",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: path.join(root, "extract"),
|
||||||
|
autoExtract: false,
|
||||||
|
autoReconnect: false,
|
||||||
|
retryLimit: 1
|
||||||
|
},
|
||||||
|
emptySession(),
|
||||||
|
createStoragePaths(path.join(root, "state"))
|
||||||
|
);
|
||||||
|
|
||||||
|
manager.addPackages([{ name: "short-archive", links: ["https://dummy/short-archive"] }]);
|
||||||
|
await manager.start();
|
||||||
|
await waitFor(() => !manager.getSnapshot().session.running, 25000);
|
||||||
|
|
||||||
|
const item = Object.values(manager.getSnapshot().session.items)[0];
|
||||||
|
expect(item?.status).toBe("failed");
|
||||||
|
expect(item?.fullStatus || item?.lastError || "").toContain("download_underflow");
|
||||||
|
expect(item?.downloadedBytes).toBe(actual.length);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
await once(server, "close");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("continues downloading while package post-processing is pending", async () => {
|
it("continues downloading while package post-processing is pending", async () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
@ -1810,6 +1884,72 @@ describe("download manager", () => {
|
|||||||
expect(fs.existsSync(targetPath)).toBe(false);
|
expect(fs.existsSync(targetPath)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("requeues preallocated completed archive items automatically on startup", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
|
||||||
|
const session = emptySession();
|
||||||
|
const packageId = "prealloc-pkg";
|
||||||
|
const itemId = "prealloc-item";
|
||||||
|
const createdAt = Date.now() - 20_000;
|
||||||
|
const outputDir = path.join(root, "downloads", "prealloc");
|
||||||
|
const targetPath = path.join(outputDir, "archive.part01.rar");
|
||||||
|
fs.mkdirSync(outputDir, { recursive: true });
|
||||||
|
fs.writeFileSync(targetPath, Buffer.alloc(8192));
|
||||||
|
|
||||||
|
session.packageOrder = [packageId];
|
||||||
|
session.packages[packageId] = {
|
||||||
|
id: packageId,
|
||||||
|
name: "prealloc",
|
||||||
|
outputDir,
|
||||||
|
extractDir: path.join(root, "extract", "prealloc"),
|
||||||
|
status: "completed",
|
||||||
|
itemIds: [itemId],
|
||||||
|
cancelled: false,
|
||||||
|
enabled: true,
|
||||||
|
createdAt,
|
||||||
|
updatedAt: createdAt
|
||||||
|
};
|
||||||
|
session.items[itemId] = {
|
||||||
|
id: itemId,
|
||||||
|
packageId,
|
||||||
|
url: "https://dummy/prealloc",
|
||||||
|
provider: "megadebrid",
|
||||||
|
status: "completed",
|
||||||
|
retries: 0,
|
||||||
|
speedBps: 0,
|
||||||
|
downloadedBytes: 1024,
|
||||||
|
totalBytes: 8192,
|
||||||
|
progressPercent: 100,
|
||||||
|
fileName: "archive.part01.rar",
|
||||||
|
targetPath,
|
||||||
|
resumable: true,
|
||||||
|
attempts: 1,
|
||||||
|
lastError: "",
|
||||||
|
fullStatus: "Fertig (8 KB)",
|
||||||
|
createdAt,
|
||||||
|
updatedAt: createdAt
|
||||||
|
};
|
||||||
|
|
||||||
|
const manager = new DownloadManager(
|
||||||
|
{
|
||||||
|
...defaultSettings(),
|
||||||
|
token: "rd-token",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: path.join(root, "extract"),
|
||||||
|
autoExtract: false
|
||||||
|
},
|
||||||
|
session,
|
||||||
|
createStoragePaths(path.join(root, "state"))
|
||||||
|
);
|
||||||
|
|
||||||
|
const snapshot = manager.getSnapshot();
|
||||||
|
const item = snapshot.session.items[itemId];
|
||||||
|
expect(item?.status).toBe("queued");
|
||||||
|
expect(item?.fullStatus).toContain("pre-alloc");
|
||||||
|
expect(snapshot.session.packages[packageId]?.status).toBe("queued");
|
||||||
|
});
|
||||||
|
|
||||||
it("detects start conflicts when extract output already exists", async () => {
|
it("detects start conflicts when extract output already exists", async () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import {
|
|||||||
detectArchiveSignature,
|
detectArchiveSignature,
|
||||||
classifyExtractionError,
|
classifyExtractionError,
|
||||||
findArchiveCandidates,
|
findArchiveCandidates,
|
||||||
|
resolveExtractorBackendMode,
|
||||||
} from "../src/main/extractor";
|
} from "../src/main/extractor";
|
||||||
|
|
||||||
const tempDirs: string[] = [];
|
const tempDirs: string[] = [];
|
||||||
@ -988,6 +989,10 @@ describe("extractor", () => {
|
|||||||
expect(classifyExtractionError("WinRAR/UnRAR nicht gefunden")).toBe("no_extractor");
|
expect(classifyExtractionError("WinRAR/UnRAR nicht gefunden")).toBe("no_extractor");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("prioritizes checksum errors over embedded wrong-password wording", () => {
|
||||||
|
expect(classifyExtractionError("Checksum error in the encrypted file. Corrupt file or wrong password.")).toBe("crc_error");
|
||||||
|
});
|
||||||
|
|
||||||
it("returns unknown for unrecognized errors", () => {
|
it("returns unknown for unrecognized errors", () => {
|
||||||
expect(classifyExtractionError("something weird happened")).toBe("unknown");
|
expect(classifyExtractionError("something weird happened")).toBe("unknown");
|
||||||
});
|
});
|
||||||
@ -1086,4 +1091,20 @@ describe("extractor", () => {
|
|||||||
expect(result.failed).toBe(0);
|
expect(result.failed).toBe(0);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("backend selection", () => {
|
||||||
|
it("defaults to auto in production when no backend override is set", () => {
|
||||||
|
expect(resolveExtractorBackendMode(undefined, false)).toBe("auto");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults to legacy in vitest when no backend override is set", () => {
|
||||||
|
expect(resolveExtractorBackendMode(undefined, true)).toBe("legacy");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("respects explicit backend overrides", () => {
|
||||||
|
expect(resolveExtractorBackendMode("legacy", false)).toBe("legacy");
|
||||||
|
expect(resolveExtractorBackendMode("jvm", false)).toBe("jvm");
|
||||||
|
expect(resolveExtractorBackendMode("auto", false)).toBe("auto");
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user