feat: parallele Proxy-Segmentdownloads
This commit is contained in:
Sucukdeluxe
@@ -0,0 +1,274 @@
|
||||
import fs from "node:fs";
|
||||
import http from "node:http";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { downloadWithProxySegments, parseProxyList } from "../src/main/proxy-segmented-download";
|
||||
|
||||
interface RunningServer {
|
||||
port: number;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
const cleanups: Array<() => Promise<void>> = [];
|
||||
|
||||
afterEach(async () => {
|
||||
while (cleanups.length > 0) {
|
||||
await cleanups.pop()?.();
|
||||
}
|
||||
});
|
||||
|
||||
async function listen(server: http.Server): Promise<RunningServer> {
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("Testserver konnte nicht gestartet werden");
|
||||
}
|
||||
const sockets = new Set<net.Socket>();
|
||||
server.on("connection", (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.once("close", () => sockets.delete(socket));
|
||||
});
|
||||
const close = async (): Promise<void> => {
|
||||
for (const socket of sockets) socket.destroy();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
};
|
||||
cleanups.push(close);
|
||||
return { port: address.port, close };
|
||||
}
|
||||
|
||||
async function createRangeServer(content: Buffer, supportsRanges = true): Promise<RunningServer & { ranges: string[] }> {
|
||||
const ranges: string[] = [];
|
||||
const server = http.createServer((request, response) => {
|
||||
const rangeHeader = String(request.headers.range || "");
|
||||
ranges.push(rangeHeader);
|
||||
if (!supportsRanges || !rangeHeader) {
|
||||
response.writeHead(200, { "Content-Length": content.length });
|
||||
response.end(content);
|
||||
return;
|
||||
}
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(rangeHeader);
|
||||
if (!match) {
|
||||
response.writeHead(416, { "Content-Range": `bytes */${content.length}` });
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
const start = Number(match[1]);
|
||||
const end = Math.min(Number(match[2]), content.length - 1);
|
||||
const body = content.subarray(start, end + 1);
|
||||
response.writeHead(206, {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Range": `bytes ${start}-${end}/${content.length}`,
|
||||
"Content-Length": body.length
|
||||
});
|
||||
response.end(body);
|
||||
});
|
||||
const running = await listen(server);
|
||||
return { ...running, ranges };
|
||||
}
|
||||
|
||||
async function createConnectProxy(username: string, password: string, acceptedConnections: { value: number }): Promise<RunningServer> {
|
||||
const server = http.createServer();
|
||||
server.on("connect", (request, clientSocket, head) => {
|
||||
const expectedAuth = `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`;
|
||||
if (request.headers["proxy-authorization"] !== expectedAuth) {
|
||||
clientSocket.end("HTTP/1.1 407 Proxy Authentication Required\r\n\r\n");
|
||||
return;
|
||||
}
|
||||
const target = new URL(`http://${request.url}`);
|
||||
const upstream = net.connect(Number(target.port), target.hostname, () => {
|
||||
acceptedConnections.value += 1;
|
||||
clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
|
||||
if (head.length > 0) upstream.write(head);
|
||||
clientSocket.pipe(upstream);
|
||||
upstream.pipe(clientSocket);
|
||||
});
|
||||
upstream.once("error", () => clientSocket.destroy());
|
||||
clientSocket.once("error", () => upstream.destroy());
|
||||
});
|
||||
return listen(server);
|
||||
}
|
||||
|
||||
async function createRejectingProxy(): Promise<RunningServer> {
|
||||
const server = http.createServer();
|
||||
server.on("connect", (_request, socket) => socket.end("HTTP/1.1 407 Proxy Authentication Required\r\n\r\n"));
|
||||
return listen(server);
|
||||
}
|
||||
|
||||
async function createTempDirectory(): Promise<string> {
|
||||
const directory = await fs.promises.mkdtemp(path.join(os.tmpdir(), "mdd-proxy-test-"));
|
||||
cleanups.push(() => fs.promises.rm(directory, { recursive: true, force: true }));
|
||||
return directory;
|
||||
}
|
||||
|
||||
describe("proxy segmented download", () => {
|
||||
it("accepts authenticated premium proxy list formats without exposing entries", () => {
|
||||
const secret = "private-password";
|
||||
const count = parseProxyList([
|
||||
`user:${secret}@127.0.0.1:8080`,
|
||||
"http://name:password@example.test:3128",
|
||||
"127.0.0.1:9000:name:password",
|
||||
"invalid",
|
||||
"# ignored"
|
||||
].join("\n"));
|
||||
|
||||
expect(count).toBe(3);
|
||||
expect(String(count)).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("downloads exact byte segments through different authenticated proxies", async () => {
|
||||
const content = Buffer.allocUnsafe(256 * 1024);
|
||||
for (let index = 0; index < content.length; index += 1) content[index] = index % 251;
|
||||
const targetServer = await createRangeServer(content);
|
||||
const proxyCounters = Array.from({ length: 4 }, () => ({ value: 0 }));
|
||||
const proxies = await Promise.all(proxyCounters.map((counter, index) => createConnectProxy(`user${index}`, `pass${index}`, counter)));
|
||||
const directory = await createTempDirectory();
|
||||
const proxyFile = path.join(directory, "proxies.txt");
|
||||
const targetFile = path.join(directory, "archive.bin");
|
||||
await fs.promises.writeFile(
|
||||
proxyFile,
|
||||
proxies.map((proxy, index) => `user${index}:pass${index}@127.0.0.1:${proxy.port}`).join("\n"),
|
||||
"utf8"
|
||||
);
|
||||
const progress: number[] = [];
|
||||
let trafficBytes = 0;
|
||||
|
||||
const result = await downloadWithProxySegments({
|
||||
directUrl: `http://127.0.0.1:${targetServer.port}/archive.bin`,
|
||||
targetPath: targetFile,
|
||||
proxyListPath: proxyFile,
|
||||
connections: 4,
|
||||
signal: new AbortController().signal,
|
||||
minSegmentBytes: 1,
|
||||
onTrafficBytes: (bytes) => { trafficBytes += bytes; },
|
||||
onProgress: (_delta, downloaded) => progress.push(downloaded)
|
||||
});
|
||||
|
||||
expect(result).toEqual({ status: "completed", totalBytes: content.length, connections: 4 });
|
||||
expect(await fs.promises.readFile(targetFile)).toEqual(content);
|
||||
expect(proxyCounters.every((counter) => counter.value > 0)).toBe(true);
|
||||
expect(targetServer.ranges).toContain("bytes=0-1");
|
||||
expect(new Set(targetServer.ranges.filter((range) => range !== "bytes=0-1")).size).toBe(4);
|
||||
expect(progress.at(-1)).toBe(content.length);
|
||||
expect(trafficBytes).toBe(content.length + 2);
|
||||
expect((await fs.promises.readdir(directory)).filter((name) => name.includes(".proxy-")).length).toBe(0);
|
||||
});
|
||||
|
||||
it("uses another proxy after a rejected connection", async () => {
|
||||
const content = Buffer.alloc(64 * 1024, 7);
|
||||
const targetServer = await createRangeServer(content);
|
||||
const rejected = await createRejectingProxy();
|
||||
const counters = Array.from({ length: 3 }, () => ({ value: 0 }));
|
||||
const working = await Promise.all(counters.map((counter, index) => createConnectProxy(`ok${index}`, `secret${index}`, counter)));
|
||||
const directory = await createTempDirectory();
|
||||
const proxyFile = path.join(directory, "proxies.txt");
|
||||
const targetFile = path.join(directory, "retry.bin");
|
||||
await fs.promises.writeFile(proxyFile, [
|
||||
`bad:bad@127.0.0.1:${rejected.port}`,
|
||||
...working.map((proxy, index) => `ok${index}:secret${index}@127.0.0.1:${proxy.port}`)
|
||||
].join("\n"));
|
||||
|
||||
const result = await downloadWithProxySegments({
|
||||
directUrl: `http://127.0.0.1:${targetServer.port}/retry.bin`,
|
||||
targetPath: targetFile,
|
||||
proxyListPath: proxyFile,
|
||||
connections: 2,
|
||||
signal: new AbortController().signal,
|
||||
minSegmentBytes: 1
|
||||
});
|
||||
|
||||
expect(result.status).toBe("completed");
|
||||
expect(await fs.promises.readFile(targetFile)).toEqual(content);
|
||||
expect(counters.some((counter) => counter.value > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back cleanly when the origin ignores byte ranges", async () => {
|
||||
const content = Buffer.alloc(4096, 3);
|
||||
const targetServer = await createRangeServer(content, false);
|
||||
const counter = { value: 0 };
|
||||
const proxy = await createConnectProxy("user", "password", counter);
|
||||
const directory = await createTempDirectory();
|
||||
const proxyFile = path.join(directory, "proxies.txt");
|
||||
const targetFile = path.join(directory, "fallback.bin");
|
||||
await fs.promises.writeFile(proxyFile, `user:password@127.0.0.1:${proxy.port}`);
|
||||
|
||||
const result = await downloadWithProxySegments({
|
||||
directUrl: `http://127.0.0.1:${targetServer.port}/fallback.bin`,
|
||||
targetPath: targetFile,
|
||||
proxyListPath: proxyFile,
|
||||
connections: 4,
|
||||
signal: new AbortController().signal,
|
||||
minSegmentBytes: 1
|
||||
});
|
||||
|
||||
expect(result).toEqual({ status: "fallback", reason: "range_unsupported" });
|
||||
expect(fs.existsSync(targetFile)).toBe(false);
|
||||
expect((await fs.promises.readdir(directory)).filter((name) => name.includes(".proxy-")).length).toBe(0);
|
||||
});
|
||||
|
||||
it("cancels all segments and removes temporary data", async () => {
|
||||
const content = Buffer.alloc(256 * 1024, 5);
|
||||
const server = http.createServer((request, response) => {
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(String(request.headers.range || ""));
|
||||
if (!match) {
|
||||
response.writeHead(416);
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
const start = Number(match[1]);
|
||||
const end = Number(match[2]);
|
||||
const body = content.subarray(start, end + 1);
|
||||
response.writeHead(206, {
|
||||
"Content-Range": `bytes ${start}-${end}/${content.length}`,
|
||||
"Content-Length": body.length
|
||||
});
|
||||
if (start === 0 && end === 1) {
|
||||
response.end(body);
|
||||
return;
|
||||
}
|
||||
let offset = 0;
|
||||
const timer = setInterval(() => {
|
||||
if (offset >= body.length) {
|
||||
clearInterval(timer);
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
const next = Math.min(body.length, offset + 1024);
|
||||
response.write(body.subarray(offset, next));
|
||||
offset = next;
|
||||
}, 5);
|
||||
response.once("close", () => clearInterval(timer));
|
||||
});
|
||||
const targetServer = await listen(server);
|
||||
const counters = Array.from({ length: 2 }, () => ({ value: 0 }));
|
||||
const proxies = await Promise.all(counters.map((counter, index) => createConnectProxy(`cancel${index}`, `pass${index}`, counter)));
|
||||
const directory = await createTempDirectory();
|
||||
const proxyFile = path.join(directory, "proxies.txt");
|
||||
const targetFile = path.join(directory, "cancel.bin");
|
||||
await fs.promises.writeFile(
|
||||
proxyFile,
|
||||
proxies.map((proxy, index) => `cancel${index}:pass${index}@127.0.0.1:${proxy.port}`).join("\n")
|
||||
);
|
||||
const controller = new AbortController();
|
||||
const progress: number[] = [];
|
||||
|
||||
const download = downloadWithProxySegments({
|
||||
directUrl: `http://127.0.0.1:${targetServer.port}/cancel.bin`,
|
||||
targetPath: targetFile,
|
||||
proxyListPath: proxyFile,
|
||||
connections: 2,
|
||||
signal: controller.signal,
|
||||
minSegmentBytes: 1,
|
||||
onProgress: (_delta, downloaded) => {
|
||||
progress.push(downloaded);
|
||||
if (downloaded >= 4096) controller.abort("test_cancel");
|
||||
}
|
||||
});
|
||||
|
||||
await expect(download).rejects.toThrow("aborted:proxy_download");
|
||||
expect(progress.at(-1)).toBe(0);
|
||||
expect(fs.existsSync(targetFile)).toBe(false);
|
||||
expect((await fs.promises.readdir(directory)).filter((name) => name.includes(".proxy-")).length).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -642,6 +642,43 @@ describe("settings views", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("projects proxy segmentation controls with safe defaults and connection totals", () => {
|
||||
const form = buildSettingsFormViewModel({
|
||||
settings: {
|
||||
...createRendererSettings({
|
||||
...defaultSettings(),
|
||||
maxParallel: 5,
|
||||
proxyDownloadEnabled: true,
|
||||
proxyListPath: "C:\\proxy-list.txt",
|
||||
proxyConnectionsPerDownload: 16
|
||||
}),
|
||||
archivePasswordList: "",
|
||||
notifyUrl: ""
|
||||
},
|
||||
section: "speed",
|
||||
speedLimitInput: "0",
|
||||
scheduleSpeedInputs: {}
|
||||
});
|
||||
const proxyGroup = form.groups.find((group) => group.id === "speed-proxy");
|
||||
|
||||
expect(proxyGroup?.fields.map((field) => field.id)).toEqual([
|
||||
"proxyDownloadEnabled",
|
||||
"proxyListPath",
|
||||
"proxyConnectionsPerDownload"
|
||||
]);
|
||||
expect(proxyGroup?.fields.find((field) => field.id === "proxyListPath")).toEqual(expect.objectContaining({
|
||||
value: "C:\\proxy-list.txt",
|
||||
actionLabel: "Datei wählen",
|
||||
disabled: false
|
||||
}));
|
||||
expect(proxyGroup?.fields.find((field) => field.id === "proxyConnectionsPerDownload")).toEqual(expect.objectContaining({
|
||||
value: "16",
|
||||
min: 2,
|
||||
max: 32,
|
||||
help: expect.stringContaining("80 Proxy-Verbindungen")
|
||||
}));
|
||||
});
|
||||
|
||||
it("projects every notification control in the required order with exact bounds", () => {
|
||||
const form = buildSettingsFormViewModel({
|
||||
settings: {
|
||||
|
||||
+19
-7
@@ -579,9 +579,12 @@ describe("settings storage", () => {
|
||||
completedCleanupPolicy: "broken" as unknown as AppSettings["completedCleanupPolicy"],
|
||||
speedLimitMode: "broken" as unknown as AppSettings["speedLimitMode"],
|
||||
maxParallel: 0,
|
||||
retryLimit: 999,
|
||||
reconnectWaitSeconds: 9999,
|
||||
speedLimitKbps: -1,
|
||||
retryLimit: 999,
|
||||
reconnectWaitSeconds: 9999,
|
||||
speedLimitKbps: -1,
|
||||
proxyDownloadEnabled: true,
|
||||
proxyListPath: " C:\\proxies.txt ",
|
||||
proxyConnectionsPerDownload: 999,
|
||||
outputDir: " ",
|
||||
extractDir: " ",
|
||||
mkvLibraryDir: " ",
|
||||
@@ -598,7 +601,10 @@ describe("settings storage", () => {
|
||||
expect(normalized.maxParallel).toBe(1);
|
||||
expect(normalized.retryLimit).toBe(99);
|
||||
expect(normalized.reconnectWaitSeconds).toBe(600);
|
||||
expect(normalized.speedLimitKbps).toBe(0);
|
||||
expect(normalized.speedLimitKbps).toBe(0);
|
||||
expect(normalized.proxyDownloadEnabled).toBe(true);
|
||||
expect(normalized.proxyListPath).toBe("C:\\proxies.txt");
|
||||
expect(normalized.proxyConnectionsPerDownload).toBe(32);
|
||||
expect(normalized.outputDir).toBe(defaultSettings().outputDir);
|
||||
expect(normalized.extractDir).toBe(defaultSettings().extractDir);
|
||||
expect(normalized.mkvLibraryDir).toBe(defaultSettings().mkvLibraryDir);
|
||||
@@ -629,8 +635,11 @@ describe("settings storage", () => {
|
||||
completedCleanupPolicy: "not-valid",
|
||||
maxParallel: "999",
|
||||
retryLimit: "-3",
|
||||
reconnectWaitSeconds: "1",
|
||||
speedLimitMode: "not-valid",
|
||||
reconnectWaitSeconds: "1",
|
||||
speedLimitMode: "not-valid",
|
||||
proxyDownloadEnabled: 1,
|
||||
proxyListPath: " C:\\proxy-list.txt ",
|
||||
proxyConnectionsPerDownload: "1",
|
||||
updateRepo: "",
|
||||
autoSortPackagesByProgress: false
|
||||
}),
|
||||
@@ -643,7 +652,10 @@ describe("settings storage", () => {
|
||||
expect(loaded.maxParallel).toBe(50);
|
||||
expect(loaded.retryLimit).toBe(0);
|
||||
expect(loaded.reconnectWaitSeconds).toBe(10);
|
||||
expect(loaded.speedLimitMode).toBe("global");
|
||||
expect(loaded.speedLimitMode).toBe("global");
|
||||
expect(loaded.proxyDownloadEnabled).toBe(true);
|
||||
expect(loaded.proxyListPath).toBe("C:\\proxy-list.txt");
|
||||
expect(loaded.proxyConnectionsPerDownload).toBe(2);
|
||||
expect(loaded.updateRepo).toBe(defaultSettings().updateRepo);
|
||||
expect(loaded.autoSortPackagesByProgress).toBe(false);
|
||||
});
|
||||
|
||||
@@ -114,9 +114,12 @@ function createSettings(): AppSettings {
|
||||
maxParallel: 4,
|
||||
maxParallelExtract: 2,
|
||||
retryLimit: 3,
|
||||
speedLimitEnabled: false,
|
||||
speedLimitKbps: 0,
|
||||
speedLimitMode: "global",
|
||||
speedLimitEnabled: false,
|
||||
speedLimitKbps: 0,
|
||||
speedLimitMode: "global",
|
||||
proxyDownloadEnabled: false,
|
||||
proxyListPath: "",
|
||||
proxyConnectionsPerDownload: 16,
|
||||
updateRepo: "Sucukdeluxe/Multi-Debrid-Downloader",
|
||||
autoUpdateCheck: true,
|
||||
clipboardWatch: true,
|
||||
|
||||
@@ -168,6 +168,7 @@ export function createVisualElectronApi(
|
||||
},
|
||||
writeClipboardText: async () => true,
|
||||
pickFolder: async () => "C:\\Visual\\Selected",
|
||||
pickProxyList: async () => "C:\\Visual\\proxy-list.txt",
|
||||
pickContainers: async () => ["C:\\Visual\\Containers\\visual.dlc"],
|
||||
getSessionStats: async () => ({
|
||||
bandwidth: {
|
||||
|
||||
Reference in New Issue
Block a user