Administrator/Multi-Debrid-Downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 596 Wiki Activity

Harden proxy segment integrity verification

Browse Source
This commit is contained in:
Sucukdeluxe
2026-08-31 16:46:56 +02:00
parent 5cee459d0b
commit 39d1961c92
7 changed files with 503 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/download-manager.ts
+20 -5
View File
@@ -10263,11 +10263,17 @@ export class DownloadManager extends EventEmitter {
}
throw new Error(`Integritätsprüfung fehlgeschlagen (${validation.message})`);
}
// Symmetry: a passed check was previously silent in the item log, so a
// reader could not tell whether integrity ran and passed vs was skipped.
this.logPackageForItem(item, "INFO", "Integritätsprüfung bestanden", {
elapsedMs: integrityElapsedMs
});
if (validation.checked) {
this.logPackageForItem(item, "INFO", "Integritätsprüfung bestanden", {
result: validation.message,
elapsedMs: integrityElapsedMs
});
} else {
this.logPackageForItem(item, "INFO", "Integritätsprüfung nicht ausgeführt", {
reason: validation.message,
elapsedMs: integrityElapsedMs
});
}
}
if (active.abortController.signal.aborted) {
@@ -11088,6 +11094,15 @@ export class DownloadManager extends EventEmitter {
this.emitState();
lastProxyUiEmitAt = nowTick;
}
},
onValidationRetry: (event) => {
logAttemptEvent("WARN", "Proxy-Segmentprüfung fehlgeschlagen, Segment wird neu geladen", {
reason: event.reason,
rangeStart: event.rangeStart,
rangeEnd: event.rangeEnd,
segmentStart: event.segmentStart,
segmentEnd: event.segmentEnd
});
}
});
if (proxyResult.status === "completed") {
src/main/integrity.ts
+19 -16
View File
@@ -139,21 +139,24 @@ async function hashFile(filePath: string, algorithm: "crc32" | "md5" | "sha1"):
});
}
export async function validateFileAgainstManifest(filePath: string, packageDir: string): Promise<{ ok: boolean; message: string }> {
const manifest = readHashManifest(packageDir);
if (manifest.size === 0) {
return { ok: true, message: "Kein Hash verfügbar" };
}
export async function validateFileAgainstManifest(
filePath: string,
packageDir: string
): Promise<{ ok: boolean; checked: boolean; message: string }> {
const manifest = readHashManifest(packageDir);
if (manifest.size === 0) {
return { ok: true, checked: false, message: "Kein Hash verfügbar" };
}
const keyByBaseName = normalizeManifestKey(path.basename(filePath));
const keyByRelativePath = normalizeManifestKey(path.relative(packageDir, filePath));
const entry = manifest.get(keyByRelativePath) || manifest.get(keyByBaseName);
if (!entry) {
return { ok: true, message: "Kein Hash für Datei" };
}
const actual = await hashFile(filePath, entry.algorithm);
if (actual === entry.digest.toLowerCase()) {
return { ok: true, message: `${entry.algorithm.toUpperCase()} ok` };
}
return { ok: false, message: `${entry.algorithm.toUpperCase()} mismatch` };
}
const entry = manifest.get(keyByRelativePath) || manifest.get(keyByBaseName);
if (!entry) {
return { ok: true, checked: false, message: "Kein Hash für Datei" };
}
const actual = await hashFile(filePath, entry.algorithm);
if (actual === entry.digest.toLowerCase()) {
return { ok: true, checked: true, message: `${entry.algorithm.toUpperCase()} ok` };
}
return { ok: false, checked: true, message: `${entry.algorithm.toUpperCase()} mismatch` };
}
src/main/proxy-segmented-download.ts
+269 -10
View File
@@ -2,7 +2,7 @@ import fs from "node:fs";
import http from "node:http";
import https from "node:https";
import path from "node:path";
import { randomUUID } from "node:crypto";
import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
import type { IncomingMessage } from "node:http";
import { HttpsProxyAgent } from "https-proxy-agent";
@@ -18,6 +18,10 @@ const PROXY_PROBE_END = 1;
const MAX_REDIRECTS = 5;
const MAX_SEGMENTS = 4_096;
const MAX_PROXY_FILE_BYTES = 8 * 1024 * 1024;
const SEGMENT_READBACK_BUFFER_BYTES = 256 * 1024;
const SUSPICIOUS_ZERO_RUN_BYTES = 1024;
const MAX_ZERO_RUN_VERIFICATIONS_PER_SEGMENT = 4;
const ZERO_RUN_SENTINEL = Buffer.alloc(SUSPICIOUS_ZERO_RUN_BYTES);
const DISK_ERROR_CODES = new Set(["ENOSPC", "EDQUOT", "EACCES", "EPERM", "EROFS", "EIO", "ENODEV"]);
export interface ProxyEndpoint {
@@ -53,6 +57,19 @@ interface Segment {
end: number;
}
interface SegmentDownloadOutcome {
bytes: number;
suspiciousZeroRanges: Segment[];
}
export interface ProxySegmentValidationRetry {
reason: "readback_mismatch" | "zero_run_mismatch";
rangeStart: number;
rangeEnd: number;
segmentStart: number;
segmentEnd: number;
}
export interface ProxySegmentedDownloadOptions {
directUrl: string;
targetPath: string;
@@ -70,6 +87,7 @@ export interface ProxySegmentedDownloadOptions {
waitWhilePaused?: () => Promise<void>;
onTrafficBytes?: (bytes: number) => void;
onProgress?: (deltaBytes: number, downloadedBytes: number, totalBytes: number) => void;
onValidationRetry?: (event: ProxySegmentValidationRetry) => void;
}
export type ProxySegmentedDownloadResult =
@@ -103,6 +121,17 @@ class OriginHttpError extends Error {
}
}
class SegmentValidationError extends Error {
public constructor(
public readonly reason: ProxySegmentValidationRetry["reason"],
public readonly rangeStart: number,
public readonly rangeEnd: number,
public readonly segment: Segment
) {
super(`proxy_segment_${reason}`);
}
}
interface ProxyLeaseOutcome {
succeeded: boolean;
transferredBytes?: number;
@@ -185,6 +214,18 @@ class SharedProxyCoordinator {
this.drain();
}
public reportContentMismatch(proxy: ProxyEndpoint): void {
const key = proxyEndpointKey(proxy);
const previous = this.performance.get(key);
this.cooldownUntil.set(key, Date.now() + 30_000);
this.performance.set(key, {
ewmaBytesPerSecond: (previous?.ewmaBytesPerSecond || 0) * 0.5,
failures: (previous?.failures || 0) + 1,
samples: previous?.samples || 0
});
this.drain();
}
public acquire(
poolKey: string,
groupId: string,
@@ -759,6 +800,149 @@ async function writeBufferAt(handle: fs.promises.FileHandle, buffer: Buffer, pos
}
}
function appendSuspiciousZeroRange(ranges: Segment[], start: number): void {
if (ranges.length >= MAX_ZERO_RUN_VERIFICATIONS_PER_SEGMENT) {
return;
}
const candidate = {
index: ranges.length,
start,
end: start + SUSPICIOUS_ZERO_RUN_BYTES - 1
};
const previous = ranges[ranges.length - 1];
if (previous && candidate.start <= previous.end + 1) {
return;
}
ranges.push(candidate);
}
function collectSuspiciousZeroRanges(
chunk: Buffer,
chunkStart: number,
previousTrailingZeros: number,
ranges: Segment[]
): number {
let leadingZeros = 0;
while (leadingZeros < chunk.length && chunk[leadingZeros] === 0) {
leadingZeros += 1;
}
if (previousTrailingZeros + leadingZeros >= SUSPICIOUS_ZERO_RUN_BYTES) {
appendSuspiciousZeroRange(ranges, chunkStart - previousTrailingZeros);
}
let searchAt = 0;
while (ranges.length < MAX_ZERO_RUN_VERIFICATIONS_PER_SEGMENT) {
const foundAt = chunk.indexOf(ZERO_RUN_SENTINEL, searchAt);
if (foundAt < 0) {
break;
}
appendSuspiciousZeroRange(ranges, chunkStart + foundAt);
searchAt = foundAt + SUSPICIOUS_ZERO_RUN_BYTES;
}
let trailingZeros = 0;
for (let index = chunk.length - 1; index >= 0 && chunk[index] === 0; index -= 1) {
trailingZeros += 1;
if (trailingZeros >= SUSPICIOUS_ZERO_RUN_BYTES - 1) {
break;
}
}
return trailingZeros;
}
async function hashFileRange(
filePath: string,
start: number,
length: number,
signal: AbortSignal
): Promise<Buffer> {
const handle = await fs.promises.open(filePath, "r");
const hash = createHash("sha256");
const buffer = Buffer.allocUnsafe(Math.min(SEGMENT_READBACK_BUFFER_BYTES, Math.max(1, length)));
let position = start;
let remaining = length;
try {
while (remaining > 0) {
if (signal.aborted) {
throw abortError();
}
const requested = Math.min(buffer.length, remaining);
const { bytesRead } = await handle.read(buffer, 0, requested, position);
if (bytesRead <= 0) {
throw new Error("proxy_segment_readback_underflow");
}
hash.update(buffer.subarray(0, bytesRead));
position += bytesRead;
remaining -= bytesRead;
}
return hash.digest();
} finally {
await handle.close();
}
}
async function readRangeOnce(
target: URL,
proxy: ProxyEndpoint,
range: Segment,
totalBytes: number,
signal: AbortSignal,
options: Required<Pick<ProxySegmentedDownloadOptions, "skipTlsVerify" | "connectTimeoutMs" | "idleTimeoutMs">>,
onTrafficBytes?: (bytes: number) => void
): Promise<Buffer> {
const opened = await openRangeResponse(
target,
proxy,
range.start,
range.end,
signal,
options.skipTlsVerify,
options.connectTimeoutMs,
options.idleTimeoutMs
);
const response = opened.response;
const status = response.statusCode || 0;
if (status >= 400) {
opened.dispose();
throw new OriginHttpError(status);
}
const expectedBytes = range.end - range.start + 1;
const contentRange = parseContentRange(response.headers["content-range"]);
const contentLength = Number(response.headers["content-length"] || 0);
if (status !== 206
|| !contentRange
|| contentRange.start !== range.start
|| contentRange.end !== range.end
|| contentRange.total !== totalBytes
|| (contentLength > 0 && contentLength !== expectedBytes)) {
opened.dispose();
throw new Error("proxy_verification_range_mismatch");
}
const chunks: Buffer[] = [];
let received = 0;
try {
for await (const rawChunk of response) {
if (signal.aborted) {
throw abortError();
}
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
onTrafficBytes?.(chunk.length);
received += chunk.length;
if (received > expectedBytes) {
throw new Error("proxy_verification_range_overflow");
}
chunks.push(chunk);
}
if (received !== expectedBytes) {
throw new Error("proxy_verification_range_underflow");
}
return Buffer.concat(chunks, received);
} finally {
opened.dispose();
}
}
async function downloadSegmentOnce(
target: URL,
tempPath: string,
@@ -769,7 +953,7 @@ async function downloadSegmentOnce(
callbacks: Pick<ProxySegmentedDownloadOptions, "waitWhilePaused" | "onTrafficBytes" | "onProgress">,
totalBytes: number,
currentProgress: () => number
): Promise<number> {
): Promise<SegmentDownloadOutcome> {
const opened = await openRangeResponse(
target,
proxy,
@@ -807,6 +991,9 @@ async function downloadSegmentOnce(
throw error;
}
let received = 0;
const receivedHash = createHash("sha256");
const suspiciousZeroRanges: Segment[] = [];
let trailingZeros = 0;
try {
for await (const rawChunk of response) {
if (signal.aborted) {
@@ -823,17 +1010,29 @@ async function downloadSegmentOnce(
response.setTimeout(options.idleTimeoutMs);
}
await writeBufferAt(handle, chunk, segment.start + received);
receivedHash.update(chunk);
trailingZeros = collectSuspiciousZeroRanges(
chunk,
segment.start + received,
trailingZeros,
suspiciousZeroRanges
);
received += chunk.length;
callbacks.onProgress?.(chunk.length, currentProgress() + chunk.length, totalBytes);
}
if (received !== expectedBytes) {
throw new Error("proxy_segment_underflow");
}
return received;
} finally {
opened.dispose();
await handle.close();
}
const writtenHash = await hashFileRange(tempPath, segment.start, expectedBytes, signal);
const networkHash = receivedHash.digest();
if (!timingSafeEqual(networkHash, writtenHash)) {
throw new SegmentValidationError("readback_mismatch", segment.start, segment.end, segment);
}
return { bytes: received, suspiciousZeroRanges };
}
export async function downloadWithProxySegments(options: ProxySegmentedDownloadOptions): Promise<ProxySegmentedDownloadResult> {
@@ -943,19 +1142,20 @@ export async function downloadWithProxySegments(options: ProxySegmentedDownloadO
if (signal.aborted) {
throw abortError();
}
const lease = await sharedProxyCoordinator.acquire(poolKey, groupId, segmentProxies, excluded, signal);
let lease = await sharedProxyCoordinator.acquire(poolKey, groupId, segmentProxies, excluded, signal);
if (!lease) {
break;
}
excluded.add(proxyEndpointKey(lease.proxy));
const primaryProxy = lease.proxy;
excluded.add(proxyEndpointKey(primaryProxy));
let attemptProgress = 0;
const startedAt = Date.now();
try {
const bytes = await downloadSegmentOnce(
const outcome = await downloadSegmentOnce(
target,
tempPath,
segment,
lease.proxy,
primaryProxy,
signal,
normalized,
{
@@ -971,13 +1171,60 @@ export async function downloadWithProxySegments(options: ProxySegmentedDownloadO
);
lease.release({
succeeded: true,
transferredBytes: bytes,
transferredBytes: outcome.bytes,
durationMs: Math.max(1, Date.now() - startedAt)
});
return bytes;
lease = null;
if (outcome.suspiciousZeroRanges.length > 0) {
lease = await sharedProxyCoordinator.acquire(poolKey, groupId, segmentProxies, excluded, signal);
if (!lease) {
throw new Error("proxy_zero_run_verification_unavailable");
}
const verificationStartedAt = Date.now();
let verificationBytes = 0;
for (const suspiciousRange of outcome.suspiciousZeroRanges) {
const verification = await readRangeOnce(
target,
lease.proxy,
suspiciousRange,
probe.totalBytes,
signal,
normalized,
(bytes) => {
verificationBytes += bytes;
options.onTrafficBytes?.(bytes);
}
);
if (!timingSafeEqual(verification, ZERO_RUN_SENTINEL)) {
lease.release({
succeeded: true,
transferredBytes: verificationBytes,
durationMs: Math.max(1, Date.now() - verificationStartedAt)
});
lease = null;
sharedProxyCoordinator.reportContentMismatch(primaryProxy);
throw new SegmentValidationError(
"zero_run_mismatch",
suspiciousRange.start,
suspiciousRange.end,
segment
);
}
}
lease.release({
succeeded: true,
transferredBytes: verificationBytes,
durationMs: Math.max(1, Date.now() - verificationStartedAt)
});
lease = null;
}
return outcome.bytes;
} catch (error) {
const originError = error instanceof OriginHttpError;
lease.release({ succeeded: originError || signal.aborted });
lease?.release({ succeeded: originError || signal.aborted });
lease = null;
if (originError) {
sharedProxyCoordinator.reportOriginStatus(error.statusCode);
}
@@ -987,6 +1234,18 @@ export async function downloadWithProxySegments(options: ProxySegmentedDownloadO
if (options.signal.aborted || isDiskError(error)) {
throw error;
}
if (error instanceof SegmentValidationError) {
try {
options.onValidationRetry?.({
reason: error.reason,
rangeStart: error.rangeStart,
rangeEnd: error.rangeEnd,
segmentStart: error.segment.start,
segmentEnd: error.segment.end
});
} catch {
}
}
lastError = error;
}
}