Fix archive underflow and extraction readiness
This commit is contained in:
Sucukdeluxe
parent
0f2e8d5567
commit
16bfbfc106
@ -4007,12 +4007,15 @@ export class DownloadManager extends EventEmitter {
|
||||
* old 50% recovery threshold). Reset to "queued" so it gets re-downloaded. */
|
||||
private revalidateCompletedItems(): void {
|
||||
let fixed = 0;
|
||||
const touchedPackageIds = new Set<string>();
|
||||
for (const item of Object.values(this.session.items)) {
|
||||
if (item.status !== "completed") continue;
|
||||
if (!item.targetPath || !item.totalBytes || item.totalBytes <= 0) continue;
|
||||
try {
|
||||
const stat = fs.statSync(item.targetPath);
|
||||
if (stat.size < item.totalBytes - ALLOCATION_UNIT_SIZE) {
|
||||
const expectedMinSize = item.totalBytes - ALLOCATION_UNIT_SIZE;
|
||||
const persistedShortfall = item.downloadedBytes < expectedMinSize && stat.size >= expectedMinSize;
|
||||
if (stat.size < expectedMinSize) {
|
||||
logger.warn(`revalidateCompleted: ${item.fileName} ist nur ${humanSize(stat.size)} statt ${humanSize(item.totalBytes)}, setze auf queued`);
|
||||
item.status = "queued";
|
||||
item.fullStatus = "Wartet";
|
||||
@ -4020,6 +4023,15 @@ export class DownloadManager extends EventEmitter {
|
||||
item.progressPercent = Math.floor((stat.size / item.totalBytes) * 100);
|
||||
item.speedBps = 0;
|
||||
fixed += 1;
|
||||
touchedPackageIds.add(item.packageId);
|
||||
} else if (persistedShortfall) {
|
||||
logger.warn(`revalidateCompleted: ${item.fileName} wirkt pre-alloc/unvollständig (stat=${humanSize(stat.size)}, bytes=${humanSize(item.downloadedBytes)}, total=${humanSize(item.totalBytes)}), setze auf queued`);
|
||||
item.status = "queued";
|
||||
item.fullStatus = "Wartet (Auto-Recovery: pre-alloc)";
|
||||
item.progressPercent = Math.max(0, Math.min(99, Math.floor((Math.max(0, item.downloadedBytes) / item.totalBytes) * 100)));
|
||||
item.speedBps = 0;
|
||||
fixed += 1;
|
||||
touchedPackageIds.add(item.packageId);
|
||||
}
|
||||
} catch {
|
||||
// file doesn't exist — reset to queued so it gets re-downloaded
|
||||
@ -4030,9 +4042,16 @@ export class DownloadManager extends EventEmitter {
|
||||
item.progressPercent = 0;
|
||||
item.speedBps = 0;
|
||||
fixed += 1;
|
||||
touchedPackageIds.add(item.packageId);
|
||||
}
|
||||
}
|
||||
if (fixed > 0) {
|
||||
for (const packageId of touchedPackageIds) {
|
||||
const pkg = this.session.packages[packageId];
|
||||
if (pkg) {
|
||||
this.refreshPackageStatus(pkg);
|
||||
}
|
||||
}
|
||||
logger.info(`revalidateCompletedItems: ${fixed} Items korrigiert`);
|
||||
this.persistSoon();
|
||||
}
|
||||
@ -6530,6 +6549,23 @@ export class DownloadManager extends EventEmitter {
|
||||
throw new Error(`Download zu klein (${written} B) – Hoster-Fehlerseite?${snippet ? ` Inhalt: "${snippet}"` : ""}`);
|
||||
}
|
||||
|
||||
const exactLengthRequired = isLargeBinaryLikePath(item.fileName || effectiveTargetPath);
|
||||
if (item.totalBytes && item.totalBytes > 0 && written < item.totalBytes) {
|
||||
const shortfall = item.totalBytes - written;
|
||||
if (preAllocated) {
|
||||
try {
|
||||
await fs.promises.truncate(effectiveTargetPath, written);
|
||||
} catch { /* best-effort */ }
|
||||
}
|
||||
logger.warn(`Download-Underflow: erwartet=${item.totalBytes}, erhalten=${written}, shortfall=${shortfall} fuer ${item.fileName}`);
|
||||
if (exactLengthRequired || shortfall > ALLOCATION_UNIT_SIZE) {
|
||||
item.downloadedBytes = written;
|
||||
item.progressPercent = Math.max(0, Math.min(99, Math.floor((written / item.totalBytes) * 100)));
|
||||
item.speedBps = 0;
|
||||
throw new Error(`download_underflow:${written}/${item.totalBytes}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate pre-allocated files to actual bytes written to prevent zero-padded tail
|
||||
if (preAllocated && item.totalBytes && written < item.totalBytes) {
|
||||
try {
|
||||
|
||||
@ -536,8 +536,8 @@ export function classifyExtractionError(errorText: string): ExtractErrorCategory
|
||||
const text = String(errorText || "").toLowerCase();
|
||||
if (text.includes("aborted:extract") || text.includes("extract_aborted")) return "aborted";
|
||||
if (text.includes("timeout")) return "timeout";
|
||||
if (text.includes("wrong password") || text.includes("falsches passwort") || text.includes("incorrect password")) return "wrong_password";
|
||||
if (text.includes("crc failed") || text.includes("checksum error") || text.includes("crc error")) return "crc_error";
|
||||
if (text.includes("wrong password") || text.includes("falsches passwort") || text.includes("incorrect password")) return "wrong_password";
|
||||
if (text.includes("missing volume") || text.includes("next volume") || text.includes("unexpected end of archive") || text.includes("missing parts")) return "missing_parts";
|
||||
if (text.includes("nicht gefunden") || text.includes("not found") || text.includes("no extractor")) return "no_extractor";
|
||||
if (text.includes("kein rar-archiv") || text.includes("not a rar archive") || text.includes("unsupported") || text.includes("unsupportedmethod")) return "unsupported_format";
|
||||
@ -937,9 +937,12 @@ type JvmExtractResult = {
|
||||
backend: string;
|
||||
};
|
||||
|
||||
function extractorBackendMode(): ExtractBackendMode {
|
||||
const defaultMode = "legacy";
|
||||
const raw = String(process.env.RD_EXTRACT_BACKEND || defaultMode).trim().toLowerCase();
|
||||
export function resolveExtractorBackendMode(
|
||||
rawValue?: string | null,
|
||||
isVitestEnv = Boolean(process.env.VITEST)
|
||||
): ExtractBackendMode {
|
||||
const defaultMode: ExtractBackendMode = isVitestEnv ? "legacy" : "auto";
|
||||
const raw = String(rawValue ?? defaultMode).trim().toLowerCase();
|
||||
if (raw === "legacy") {
|
||||
return "legacy";
|
||||
}
|
||||
@ -949,6 +952,10 @@ function extractorBackendMode(): ExtractBackendMode {
|
||||
return "auto";
|
||||
}
|
||||
|
||||
function extractorBackendMode(): ExtractBackendMode {
|
||||
return resolveExtractorBackendMode(process.env.RD_EXTRACT_BACKEND);
|
||||
}
|
||||
|
||||
function isJvmRuntimeMissingError(errorText: string): boolean {
|
||||
const text = String(errorText || "").toLowerCase();
|
||||
return text.includes("could not find or load main class")
|
||||
|
||||
@ -288,6 +288,80 @@ describe("download manager", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not mark truncated archive downloads as completed", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
const advertised = Buffer.alloc(96 * 1024, 5);
|
||||
const actual = advertised.subarray(0, advertised.length - 2048);
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
if ((req.url || "") !== "/short-archive") {
|
||||
res.statusCode = 404;
|
||||
res.end("not-found");
|
||||
return;
|
||||
}
|
||||
res.statusCode = 200;
|
||||
res.setHeader("Accept-Ranges", "bytes");
|
||||
res.setHeader("Content-Length", String(actual.length));
|
||||
res.end(actual);
|
||||
});
|
||||
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("server address unavailable");
|
||||
}
|
||||
const directUrl = `http://127.0.0.1:${address.port}/short-archive`;
|
||||
|
||||
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||
if (url.includes("/unrestrict/link")) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
download: directUrl,
|
||||
filename: "broken.part01.rar",
|
||||
filesize: advertised.length
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" }
|
||||
}
|
||||
);
|
||||
}
|
||||
return originalFetch(input, init);
|
||||
};
|
||||
|
||||
try {
|
||||
const manager = new DownloadManager(
|
||||
{
|
||||
...defaultSettings(),
|
||||
token: "rd-token",
|
||||
outputDir: path.join(root, "downloads"),
|
||||
extractDir: path.join(root, "extract"),
|
||||
autoExtract: false,
|
||||
autoReconnect: false,
|
||||
retryLimit: 1
|
||||
},
|
||||
emptySession(),
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
|
||||
manager.addPackages([{ name: "short-archive", links: ["https://dummy/short-archive"] }]);
|
||||
await manager.start();
|
||||
await waitFor(() => !manager.getSnapshot().session.running, 25000);
|
||||
|
||||
const item = Object.values(manager.getSnapshot().session.items)[0];
|
||||
expect(item?.status).toBe("failed");
|
||||
expect(item?.fullStatus || item?.lastError || "").toContain("download_underflow");
|
||||
expect(item?.downloadedBytes).toBe(actual.length);
|
||||
} finally {
|
||||
server.close();
|
||||
await once(server, "close");
|
||||
}
|
||||
});
|
||||
|
||||
it("continues downloading while package post-processing is pending", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
@ -1810,6 +1884,72 @@ describe("download manager", () => {
|
||||
expect(fs.existsSync(targetPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("requeues preallocated completed archive items automatically on startup", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
|
||||
const session = emptySession();
|
||||
const packageId = "prealloc-pkg";
|
||||
const itemId = "prealloc-item";
|
||||
const createdAt = Date.now() - 20_000;
|
||||
const outputDir = path.join(root, "downloads", "prealloc");
|
||||
const targetPath = path.join(outputDir, "archive.part01.rar");
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
fs.writeFileSync(targetPath, Buffer.alloc(8192));
|
||||
|
||||
session.packageOrder = [packageId];
|
||||
session.packages[packageId] = {
|
||||
id: packageId,
|
||||
name: "prealloc",
|
||||
outputDir,
|
||||
extractDir: path.join(root, "extract", "prealloc"),
|
||||
status: "completed",
|
||||
itemIds: [itemId],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
createdAt,
|
||||
updatedAt: createdAt
|
||||
};
|
||||
session.items[itemId] = {
|
||||
id: itemId,
|
||||
packageId,
|
||||
url: "https://dummy/prealloc",
|
||||
provider: "megadebrid",
|
||||
status: "completed",
|
||||
retries: 0,
|
||||
speedBps: 0,
|
||||
downloadedBytes: 1024,
|
||||
totalBytes: 8192,
|
||||
progressPercent: 100,
|
||||
fileName: "archive.part01.rar",
|
||||
targetPath,
|
||||
resumable: true,
|
||||
attempts: 1,
|
||||
lastError: "",
|
||||
fullStatus: "Fertig (8 KB)",
|
||||
createdAt,
|
||||
updatedAt: createdAt
|
||||
};
|
||||
|
||||
const manager = new DownloadManager(
|
||||
{
|
||||
...defaultSettings(),
|
||||
token: "rd-token",
|
||||
outputDir: path.join(root, "downloads"),
|
||||
extractDir: path.join(root, "extract"),
|
||||
autoExtract: false
|
||||
},
|
||||
session,
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
|
||||
const snapshot = manager.getSnapshot();
|
||||
const item = snapshot.session.items[itemId];
|
||||
expect(item?.status).toBe("queued");
|
||||
expect(item?.fullStatus).toContain("pre-alloc");
|
||||
expect(snapshot.session.packages[packageId]?.status).toBe("queued");
|
||||
});
|
||||
|
||||
it("detects start conflicts when extract output already exists", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
|
||||
@ -11,6 +11,7 @@ import {
|
||||
detectArchiveSignature,
|
||||
classifyExtractionError,
|
||||
findArchiveCandidates,
|
||||
resolveExtractorBackendMode,
|
||||
} from "../src/main/extractor";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
@ -988,6 +989,10 @@ describe("extractor", () => {
|
||||
expect(classifyExtractionError("WinRAR/UnRAR nicht gefunden")).toBe("no_extractor");
|
||||
});
|
||||
|
||||
it("prioritizes checksum errors over embedded wrong-password wording", () => {
|
||||
expect(classifyExtractionError("Checksum error in the encrypted file. Corrupt file or wrong password.")).toBe("crc_error");
|
||||
});
|
||||
|
||||
it("returns unknown for unrecognized errors", () => {
|
||||
expect(classifyExtractionError("something weird happened")).toBe("unknown");
|
||||
});
|
||||
@ -1086,4 +1091,20 @@ describe("extractor", () => {
|
||||
expect(result.failed).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("backend selection", () => {
|
||||
it("defaults to auto in production when no backend override is set", () => {
|
||||
expect(resolveExtractorBackendMode(undefined, false)).toBe("auto");
|
||||
});
|
||||
|
||||
it("defaults to legacy in vitest when no backend override is set", () => {
|
||||
expect(resolveExtractorBackendMode(undefined, true)).toBe("legacy");
|
||||
});
|
||||
|
||||
it("respects explicit backend overrides", () => {
|
||||
expect(resolveExtractorBackendMode("legacy", false)).toBe("legacy");
|
||||
expect(resolveExtractorBackendMode("jvm", false)).toBe("jvm");
|
||||
expect(resolveExtractorBackendMode("auto", false)).toBe("auto");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user