Rebuild download completion verification
This commit is contained in:
Sucukdeluxe
parent
1a076c49cb
commit
1a0f49b29c
132
src/main/download-completion.ts
Normal file
132
src/main/download-completion.ts
Normal file
@ -0,0 +1,132 @@
|
||||
import { ALLOCATION_UNIT_SIZE } from "./constants";
|
||||
|
||||
export type DownloadCompletionSource =
|
||||
| "content-range"
|
||||
| "content-length"
|
||||
| "provider-metadata"
|
||||
| "stream-end";
|
||||
|
||||
export type DownloadCompletionPlan = {
|
||||
expectedTotal: number | null;
|
||||
source: DownloadCompletionSource;
|
||||
canFinishEarly: boolean;
|
||||
};
|
||||
|
||||
export function planDownloadCompletion(args: {
|
||||
existingBytes: number;
|
||||
responseStatus: number;
|
||||
contentLength: number;
|
||||
totalFromRange: number | null;
|
||||
knownTotal: number | null;
|
||||
correctedTotal: number | null;
|
||||
}): DownloadCompletionPlan {
|
||||
const existingBytes = Math.max(0, Math.floor(Number(args.existingBytes) || 0));
|
||||
const responseStatus = Math.floor(Number(args.responseStatus) || 0);
|
||||
const contentLength = Math.max(0, Math.floor(Number(args.contentLength) || 0));
|
||||
const totalFromRange = Number.isFinite(args.totalFromRange || NaN)
|
||||
? Math.max(0, Math.floor(args.totalFromRange || 0))
|
||||
: 0;
|
||||
const correctedTotal = Number.isFinite(args.correctedTotal || NaN)
|
||||
? Math.max(0, Math.floor(args.correctedTotal || 0))
|
||||
: 0;
|
||||
const knownTotal = Number.isFinite(args.knownTotal || NaN)
|
||||
? Math.max(0, Math.floor(args.knownTotal || 0))
|
||||
: 0;
|
||||
|
||||
if (correctedTotal > 0) {
|
||||
return {
|
||||
expectedTotal: correctedTotal,
|
||||
source: totalFromRange > 0 ? "content-range" : "content-length",
|
||||
canFinishEarly: true
|
||||
};
|
||||
}
|
||||
|
||||
if (totalFromRange > 0) {
|
||||
return {
|
||||
expectedTotal: totalFromRange,
|
||||
source: "content-range",
|
||||
canFinishEarly: true
|
||||
};
|
||||
}
|
||||
|
||||
if (contentLength > 0) {
|
||||
return {
|
||||
expectedTotal: responseStatus === 206 ? existingBytes + contentLength : contentLength,
|
||||
source: "content-length",
|
||||
canFinishEarly: true
|
||||
};
|
||||
}
|
||||
|
||||
if (knownTotal > 0) {
|
||||
return {
|
||||
expectedTotal: knownTotal,
|
||||
source: "provider-metadata",
|
||||
canFinishEarly: false
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
expectedTotal: null,
|
||||
source: "stream-end",
|
||||
canFinishEarly: false
|
||||
};
|
||||
}
|
||||
|
||||
export function validateDownloadedFileCompletion(args: {
|
||||
actualBytes: number;
|
||||
plan: DownloadCompletionPlan;
|
||||
}): {
|
||||
ok: boolean;
|
||||
totalBytes: number;
|
||||
acceptedMetadataMismatch: boolean;
|
||||
error?: string;
|
||||
} {
|
||||
const actualBytes = Math.max(0, Math.floor(Number(args.actualBytes) || 0));
|
||||
const expectedTotal = Number.isFinite(args.plan.expectedTotal || NaN)
|
||||
? Math.max(0, Math.floor(args.plan.expectedTotal || 0))
|
||||
: 0;
|
||||
|
||||
if (
|
||||
expectedTotal > 0 &&
|
||||
(args.plan.source === "content-range" || args.plan.source === "content-length") &&
|
||||
actualBytes + ALLOCATION_UNIT_SIZE < expectedTotal
|
||||
) {
|
||||
return {
|
||||
ok: false,
|
||||
totalBytes: expectedTotal,
|
||||
acceptedMetadataMismatch: false,
|
||||
error: `download_underflow:${actualBytes}/${expectedTotal}`
|
||||
};
|
||||
}
|
||||
|
||||
if (actualBytes <= 0 && expectedTotal > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
totalBytes: expectedTotal,
|
||||
acceptedMetadataMismatch: false,
|
||||
error: `download_underflow:${actualBytes}/${expectedTotal}`
|
||||
};
|
||||
}
|
||||
|
||||
if (args.plan.source === "provider-metadata") {
|
||||
return {
|
||||
ok: true,
|
||||
totalBytes: actualBytes,
|
||||
acceptedMetadataMismatch: expectedTotal > 0 && Math.abs(actualBytes - expectedTotal) > ALLOCATION_UNIT_SIZE
|
||||
};
|
||||
}
|
||||
|
||||
if (args.plan.source === "stream-end") {
|
||||
return {
|
||||
ok: true,
|
||||
totalBytes: actualBytes,
|
||||
acceptedMetadataMismatch: false
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
totalBytes: Math.max(actualBytes, expectedTotal),
|
||||
acceptedMetadataMismatch: false
|
||||
};
|
||||
}
|
||||
@ -50,6 +50,7 @@ function releaseTlsSkip(): void {
|
||||
}
|
||||
}
|
||||
import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup";
|
||||
import { planDownloadCompletion, validateDownloadedFileCompletion } from "./download-completion";
|
||||
import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkRapidgatorOnline, fetchAllDebridHostInfo, getAvailableDebridLinkApiKeys } from "./debrid";
|
||||
import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, type ExtractArchiveFailureInfo } from "./extractor";
|
||||
import { validateFileAgainstManifest } from "./integrity";
|
||||
@ -8363,6 +8364,14 @@ export class DownloadManager extends EventEmitter {
|
||||
// Only add existingBytes for 206 responses; for 200 the Content-Length is the full file
|
||||
item.totalBytes = response.status === 206 ? existingBytes + contentLength : contentLength;
|
||||
}
|
||||
const completionPlan = planDownloadCompletion({
|
||||
existingBytes,
|
||||
responseStatus: response.status,
|
||||
contentLength,
|
||||
totalFromRange,
|
||||
knownTotal,
|
||||
correctedTotal: correctedRealDebridTotal?.totalBytes || null
|
||||
});
|
||||
|
||||
const writeMode = existingBytes > 0 && response.status === 206 ? "a" : "w";
|
||||
logAttemptEvent("INFO", "HTTP-Antwort akzeptiert", {
|
||||
@ -8723,12 +8732,7 @@ export class DownloadManager extends EventEmitter {
|
||||
// Use totalBytes (from unrestrict or Content-Length header) as
|
||||
// primary check, fall back to raw contentLength for providers
|
||||
// that don't report fileSize (e.g. Mega-Debrid Web).
|
||||
const expectedTotal = (item.totalBytes && item.totalBytes > 0) ? item.totalBytes : 0;
|
||||
const expectedFromResponse = contentLength > 0 ? contentLength : 0;
|
||||
if (expectedTotal > 0 && existingBytes + written >= expectedTotal) {
|
||||
break;
|
||||
}
|
||||
if (expectedTotal === 0 && expectedFromResponse > 0 && written >= expectedFromResponse) {
|
||||
if (completionPlan.canFinishEarly && completionPlan.expectedTotal && written >= completionPlan.expectedTotal) {
|
||||
break;
|
||||
}
|
||||
|
||||
@ -8870,6 +8874,20 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const finalizedStat = await fs.promises.stat(effectiveTargetPath);
|
||||
if (Number.isFinite(finalizedStat.size) && finalizedStat.size >= 0 && finalizedStat.size !== written) {
|
||||
logAttemptEvent("WARN", "Dateigroesse nach Stream-Abschluss korrigiert", {
|
||||
attempt,
|
||||
previousWritten: written,
|
||||
statSize: finalizedStat.size
|
||||
});
|
||||
written = finalizedStat.size;
|
||||
}
|
||||
} catch {
|
||||
// ignore stat race; validation below will handle empty/missing files
|
||||
}
|
||||
|
||||
// Detect tiny error-response files (e.g. hoster returning "Forbidden" with HTTP 200).
|
||||
// No legitimate file-hoster download is < 512 bytes.
|
||||
if (written > 0 && written < 512) {
|
||||
@ -8900,21 +8918,37 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
const exactLengthRequired = isLargeBinaryLikePath(item.fileName || effectiveTargetPath);
|
||||
if (item.totalBytes && item.totalBytes > 0 && written < item.totalBytes) {
|
||||
const shortfall = item.totalBytes - written;
|
||||
const completionValidation = validateDownloadedFileCompletion({
|
||||
actualBytes: written,
|
||||
plan: completionPlan
|
||||
});
|
||||
if (!completionValidation.ok) {
|
||||
const shortfall = Math.max(0, completionValidation.totalBytes - written);
|
||||
if (preAllocated) {
|
||||
try {
|
||||
await fs.promises.truncate(effectiveTargetPath, written);
|
||||
} catch { /* best-effort */ }
|
||||
}
|
||||
logger.warn(`Download-Underflow: erwartet=${item.totalBytes}, erhalten=${written}, shortfall=${shortfall} fuer ${item.fileName}`);
|
||||
if (exactLengthRequired || shortfall > ALLOCATION_UNIT_SIZE) {
|
||||
item.downloadedBytes = written;
|
||||
item.progressPercent = Math.max(0, Math.min(99, Math.floor((written / item.totalBytes) * 100)));
|
||||
item.speedBps = 0;
|
||||
throw new Error(`download_underflow:${written}/${item.totalBytes}`);
|
||||
}
|
||||
logger.warn(`Download-Underflow: erwartet=${completionValidation.totalBytes}, erhalten=${written}, shortfall=${shortfall} fuer ${item.fileName}`);
|
||||
item.downloadedBytes = written;
|
||||
item.progressPercent = completionValidation.totalBytes > 0
|
||||
? Math.max(0, Math.min(99, Math.floor((written / completionValidation.totalBytes) * 100)))
|
||||
: 0;
|
||||
item.speedBps = 0;
|
||||
throw new Error(completionValidation.error || `download_underflow:${written}/${completionValidation.totalBytes}`);
|
||||
}
|
||||
|
||||
if (completionValidation.acceptedMetadataMismatch) {
|
||||
logger.warn(
|
||||
`Provider-Groesseninfo verworfen, HTTP-EOF als vollstaendig akzeptiert: ` +
|
||||
`${item.fileName} erwartet=${completionPlan.expectedTotal}, erhalten=${written}`
|
||||
);
|
||||
logAttemptEvent("WARN", "Provider-Groesseninfo weicht von finaler Dateigroesse ab", {
|
||||
attempt,
|
||||
expectedTotal: completionPlan.expectedTotal,
|
||||
actualBytes: written,
|
||||
source: completionPlan.source
|
||||
});
|
||||
}
|
||||
|
||||
// Truncate pre-allocated files to actual bytes written to prevent zero-padded tail
|
||||
@ -8926,6 +8960,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
item.downloadedBytes = written;
|
||||
item.totalBytes = completionValidation.totalBytes > 0 ? completionValidation.totalBytes : item.totalBytes;
|
||||
item.progressPercent = item.totalBytes ? Math.max(0, Math.min(100, Math.floor((written / item.totalBytes) * 100))) : 100;
|
||||
item.speedBps = 0;
|
||||
item.fullStatus = "Finalisierend...";
|
||||
|
||||
@ -7,6 +7,7 @@ import { EventEmitter, once } from "node:events";
|
||||
import AdmZip from "adm-zip";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { DownloadManager, extractArchiveNameFromExtractorLogMessage, getAuthoritativeRealDebridTotal, resolveArchiveItemsFromList } from "../src/main/download-manager";
|
||||
import { planDownloadCompletion, validateDownloadedFileCompletion } from "../src/main/download-completion";
|
||||
import { defaultSettings } from "../src/main/constants";
|
||||
import { parseDebridLinkApiKeys } from "../src/shared/debrid-link-keys";
|
||||
import { getProviderUsageDayKey } from "../src/shared/provider-daily-limits";
|
||||
@ -45,6 +46,38 @@ describe("resolveArchiveItemsFromList", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("download completion planning", () => {
|
||||
it("does not allow early finish on provider metadata alone", () => {
|
||||
expect(planDownloadCompletion({
|
||||
existingBytes: 0,
|
||||
responseStatus: 200,
|
||||
contentLength: 0,
|
||||
totalFromRange: null,
|
||||
knownTotal: 192 * 1024,
|
||||
correctedTotal: null
|
||||
})).toEqual({
|
||||
expectedTotal: 192 * 1024,
|
||||
source: "provider-metadata",
|
||||
canFinishEarly: false
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts provider metadata mismatches after a clean stream end", () => {
|
||||
expect(validateDownloadedFileCompletion({
|
||||
actualBytes: 256 * 1024,
|
||||
plan: {
|
||||
expectedTotal: 192 * 1024,
|
||||
source: "provider-metadata",
|
||||
canFinishEarly: false
|
||||
}
|
||||
})).toEqual({
|
||||
ok: true,
|
||||
totalBytes: 256 * 1024,
|
||||
acceptedMetadataMismatch: true
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
async function waitFor(predicate: () => boolean, timeoutMs = 15000): Promise<void> {
|
||||
const started = Date.now();
|
||||
while (!predicate()) {
|
||||
@ -1863,6 +1896,215 @@ describe("download manager", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not stop early on provider-only totals when the HTTP response is longer", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-provider-total-mismatch-"));
|
||||
tempDirs.push(root);
|
||||
const actual = Buffer.alloc(256 * 1024, 77);
|
||||
const advertised = 192 * 1024;
|
||||
let unrestrictCalls = 0;
|
||||
|
||||
const server = http.createServer((_req, res) => {
|
||||
res.statusCode = 200;
|
||||
res.setHeader("Transfer-Encoding", "chunked");
|
||||
res.write(actual.subarray(0, 96 * 1024));
|
||||
setTimeout(() => {
|
||||
res.end(actual.subarray(96 * 1024));
|
||||
}, 40);
|
||||
});
|
||||
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("server address unavailable");
|
||||
}
|
||||
const directUrl = `http://127.0.0.1:${address.port}/provider-mismatch`;
|
||||
|
||||
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||
if (url.includes("/unrestrict/link")) {
|
||||
unrestrictCalls += 1;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
download: directUrl,
|
||||
filename: "provider-mismatch.rar",
|
||||
filesize: advertised
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" }
|
||||
}
|
||||
);
|
||||
}
|
||||
return originalFetch(input, init);
|
||||
};
|
||||
|
||||
try {
|
||||
const manager = new DownloadManager(
|
||||
{
|
||||
...defaultSettings(),
|
||||
token: "rd-token",
|
||||
outputDir: path.join(root, "downloads"),
|
||||
extractDir: path.join(root, "extract"),
|
||||
retryLimit: 1,
|
||||
autoExtract: false,
|
||||
autoReconnect: false
|
||||
},
|
||||
emptySession(),
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
|
||||
manager.addPackages([{ name: "provider-mismatch", links: ["https://dummy/provider-mismatch"] }]);
|
||||
await manager.start();
|
||||
await waitFor(() => !manager.getSnapshot().session.running, 25000);
|
||||
|
||||
const item = Object.values(manager.getSnapshot().session.items)[0];
|
||||
expect(item?.status).toBe("completed");
|
||||
expect(item?.downloadedBytes).toBe(actual.length);
|
||||
expect(item?.totalBytes).toBe(actual.length);
|
||||
expect(unrestrictCalls).toBe(1);
|
||||
expect(fs.readFileSync(item.targetPath).equals(actual)).toBe(true);
|
||||
} finally {
|
||||
server.close();
|
||||
await once(server, "close");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not double-count resumed bytes when deciding early completion", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-resume-early-finish-"));
|
||||
tempDirs.push(root);
|
||||
const actual = Buffer.alloc(256 * 1024, 91);
|
||||
const partialSize = 96 * 1024;
|
||||
const pkgDir = path.join(root, "downloads", "resume-early-finish");
|
||||
fs.mkdirSync(pkgDir, { recursive: true });
|
||||
const targetPath = path.join(pkgDir, "resume-early-finish.mkv");
|
||||
fs.writeFileSync(targetPath, actual.subarray(0, partialSize));
|
||||
let unrestrictCalls = 0;
|
||||
const starts: number[] = [];
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
const range = String(req.headers.range || "");
|
||||
const match = range.match(/bytes=(\d+)-/i);
|
||||
const start = match ? Number(match[1]) : 0;
|
||||
starts.push(start);
|
||||
|
||||
if (start <= 0) {
|
||||
res.statusCode = 500;
|
||||
res.end("expected resume");
|
||||
return;
|
||||
}
|
||||
|
||||
const remaining = actual.subarray(start);
|
||||
const firstChunkBytes = 64 * 1024;
|
||||
res.statusCode = 206;
|
||||
res.setHeader("Accept-Ranges", "bytes");
|
||||
res.setHeader("Content-Range", `bytes ${start}-${actual.length - 1}/${actual.length}`);
|
||||
res.setHeader("Content-Length", String(remaining.length));
|
||||
res.write(remaining.subarray(0, firstChunkBytes));
|
||||
setTimeout(() => {
|
||||
res.end(remaining.subarray(firstChunkBytes));
|
||||
}, 50);
|
||||
});
|
||||
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("server address unavailable");
|
||||
}
|
||||
const directUrl = `http://127.0.0.1:${address.port}/resume-early-finish`;
|
||||
|
||||
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||
if (url.includes("/unrestrict/link")) {
|
||||
unrestrictCalls += 1;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
download: directUrl,
|
||||
filename: "resume-early-finish.mkv",
|
||||
filesize: actual.length
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" }
|
||||
}
|
||||
);
|
||||
}
|
||||
return originalFetch(input, init);
|
||||
};
|
||||
|
||||
try {
|
||||
const session = emptySession();
|
||||
const packageId = "resume-early-finish-pkg";
|
||||
const itemId = "resume-early-finish-item";
|
||||
const createdAt = Date.now() - 10_000;
|
||||
|
||||
session.packageOrder = [packageId];
|
||||
session.packages[packageId] = {
|
||||
id: packageId,
|
||||
name: "resume-early-finish",
|
||||
outputDir: pkgDir,
|
||||
extractDir: path.join(root, "extract", "resume-early-finish"),
|
||||
status: "queued",
|
||||
itemIds: [itemId],
|
||||
cancelled: false,
|
||||
enabled: true,
|
||||
createdAt,
|
||||
updatedAt: createdAt
|
||||
};
|
||||
session.items[itemId] = {
|
||||
id: itemId,
|
||||
packageId,
|
||||
url: "https://dummy/resume-early-finish",
|
||||
provider: null,
|
||||
status: "queued",
|
||||
retries: 0,
|
||||
speedBps: 0,
|
||||
downloadedBytes: partialSize,
|
||||
totalBytes: actual.length,
|
||||
progressPercent: Math.floor((partialSize / actual.length) * 100),
|
||||
fileName: "resume-early-finish.mkv",
|
||||
targetPath,
|
||||
resumable: true,
|
||||
attempts: 0,
|
||||
lastError: "",
|
||||
fullStatus: "Wartet",
|
||||
createdAt,
|
||||
updatedAt: createdAt
|
||||
};
|
||||
|
||||
const manager = new DownloadManager(
|
||||
{
|
||||
...defaultSettings(),
|
||||
token: "rd-token",
|
||||
outputDir: path.join(root, "downloads"),
|
||||
extractDir: path.join(root, "extract"),
|
||||
retryLimit: 1,
|
||||
autoExtract: false,
|
||||
autoReconnect: false
|
||||
},
|
||||
session,
|
||||
createStoragePaths(path.join(root, "state"))
|
||||
);
|
||||
|
||||
await manager.start();
|
||||
await waitFor(() => !manager.getSnapshot().session.running, 25000);
|
||||
|
||||
const item = manager.getSnapshot().session.items[itemId];
|
||||
expect(item?.status).toBe("completed");
|
||||
expect(item?.downloadedBytes).toBe(actual.length);
|
||||
expect(item?.totalBytes).toBe(actual.length);
|
||||
expect(unrestrictCalls).toBe(1);
|
||||
expect(starts).toEqual([partialSize]);
|
||||
expect(fs.readFileSync(targetPath).equals(actual)).toBe(true);
|
||||
} finally {
|
||||
server.close();
|
||||
await once(server, "close");
|
||||
}
|
||||
});
|
||||
|
||||
it("assigns unique target paths for same filenames in parallel", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||
tempDirs.push(root);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user