feat(ddownload): resolve metadata before queueing
Inspect public ddownload.com and ddl.to links before download startup to resolve canonical filenames, exact sizes and availability without consuming a debrid account. Keep removed files offline, protected pages retryable, restored queues recheckable and previously resolved names safe from later download.bin responses. Reuse the metadata path in the link collector, queue import and direct DDownload login flow.
This commit is contained in:
Sucukdeluxe
@@ -4,11 +4,12 @@ import { serializeCollectorPackages } from "../shared/collector";
|
||||
import type { CollectorInspectionRequest, CollectorInspectionResult, CollectorLink, CollectorPackage } from "../shared/collector";
|
||||
import type { AppSettings, ParsedPackageInput } from "../shared/types";
|
||||
import { extractHosterFromUrl } from "../shared/hoster";
|
||||
import { checkOneFichierLinks, checkRapidgatorOnline, DebridService, isOneFichierLink, type OneFichierCheckResult } from "./debrid";
|
||||
import { checkDdownloadOnline, checkOneFichierLinks, checkRapidgatorOnline, DebridService, isDdownloadLink, isOneFichierLink, type OneFichierCheckResult } from "./debrid";
|
||||
import { parseCollectorInput } from "./link-parser";
|
||||
import { filenameFromUrl, isHttpLink, looksLikeOpaqueFilename, sanitizeFilename } from "./utils";
|
||||
|
||||
interface CollectorInspectionDependencies {
|
||||
checkDdownload?: typeof checkDdownloadOnline;
|
||||
checkOneFichier?: (links: string[]) => Promise<Map<string, OneFichierCheckResult>>;
|
||||
checkRapidgator?: typeof checkRapidgatorOnline;
|
||||
resolveFilenames?: (links: string[]) => Promise<Map<string, string>>;
|
||||
@@ -160,11 +161,13 @@ export async function inspectCollectorPackages(
|
||||
|
||||
const oneFichierLinks = sourceLinks.map((link) => link.url).filter(isOneFichierLink);
|
||||
const rapidgatorLinks = sourceLinks.map((link) => link.url).filter((url) => extractHosterFromUrl(url) === "rapidgator");
|
||||
const ddownloadLinks = sourceLinks.map((link) => link.url).filter(isDdownloadLink);
|
||||
const genericLinks = sourceLinks
|
||||
.filter((source) => !source.explicitFileName && looksLikeOpaqueFilename(source.fileName))
|
||||
.map((source) => source.url)
|
||||
.filter((url) => !oneFichierLinks.includes(url) && !rapidgatorLinks.includes(url));
|
||||
.filter((url) => !oneFichierLinks.includes(url) && !rapidgatorLinks.includes(url) && !ddownloadLinks.includes(url));
|
||||
|
||||
const checkDdownload = dependencies.checkDdownload ?? checkDdownloadOnline;
|
||||
const checkOneFichier = dependencies.checkOneFichier ?? checkOneFichierLinks;
|
||||
const checkRapidgator = dependencies.checkRapidgator ?? checkRapidgatorOnline;
|
||||
const resolveFilenames = dependencies.resolveFilenames ?? ((urls) => new DebridService(settings).resolveFilenames(urls));
|
||||
@@ -179,9 +182,20 @@ export async function inspectCollectorPackages(
|
||||
if (result.fileName) link.fileName = sanitizeFilename(result.fileName);
|
||||
if (result.fileSizeBytes !== null && result.fileSizeBytes >= 0) link.fileSizeBytes = result.fileSizeBytes;
|
||||
});
|
||||
const genericPromise = resolveFilenames(genericLinks).catch(() => new Map<string, string>());
|
||||
const ddownloadPromise = runWithConcurrency(ddownloadLinks, 4, async (url) => {
|
||||
const result = await checkDdownload(url).catch(() => null);
|
||||
const link = linksByUrl.get(url);
|
||||
if (!link || !result) return;
|
||||
link.availability = result.online ? "online" : "offline";
|
||||
link.status = result.online ? (result.fileName ? "ready" : "unknown") : "offline";
|
||||
if (result.fileName) link.fileName = sanitizeFilename(result.fileName);
|
||||
if (result.fileSizeBytes !== null && result.fileSizeBytes >= 0) link.fileSizeBytes = result.fileSizeBytes;
|
||||
});
|
||||
const genericPromise = genericLinks.length > 0
|
||||
? resolveFilenames(genericLinks).catch(() => new Map<string, string>())
|
||||
: Promise.resolve(new Map<string, string>());
|
||||
|
||||
const [oneFichierResults, genericResults] = await Promise.all([oneFichierPromise, genericPromise, rapidgatorPromise]).then(([one, generic]) => [one, generic] as const);
|
||||
const [oneFichierResults, genericResults] = await Promise.all([oneFichierPromise, genericPromise, rapidgatorPromise, ddownloadPromise]).then(([one, generic]) => [one, generic] as const);
|
||||
for (const [url, result] of oneFichierResults) {
|
||||
const link = linksByUrl.get(url);
|
||||
if (!link) continue;
|
||||
|
||||
+120
-7
@@ -3797,11 +3797,125 @@ class OneFichierClient {
|
||||
}
|
||||
}
|
||||
|
||||
const DDOWNLOAD_URL_RE = /^https?:\/\/(?:www\.)?(?:ddownload\.com|ddl\.to)\/([a-z0-9]+)/i;
|
||||
const DDOWNLOAD_WEB_BASE = "https://ddownload.com";
|
||||
const DDOWNLOAD_WEB_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36";
|
||||
|
||||
class DdownloadClient {
|
||||
const DDOWNLOAD_URL_RE = /^https?:\/\/(?:www\.)?(?:ddownload\.com|ddl\.to)\/([a-z0-9]{8,20})(?:\/|$)/i;
|
||||
const DDOWNLOAD_WEB_BASE = "https://ddownload.com";
|
||||
const DDOWNLOAD_WEB_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36";
|
||||
const DDOWNLOAD_SCAN_MAX_BYTES = 512 * 1024;
|
||||
const DDOWNLOAD_FILE_NOT_FOUND_RE = /(?:File Not Found|No such file|file was removed|file was banned|Unavailable for legal reasons)/i;
|
||||
|
||||
export interface DdownloadCheckResult {
|
||||
online: boolean;
|
||||
fileName: string;
|
||||
fileSizeBytes: number | null;
|
||||
}
|
||||
|
||||
export function isDdownloadLink(link: string): boolean {
|
||||
return DDOWNLOAD_URL_RE.test(String(link || "").trim());
|
||||
}
|
||||
|
||||
export function filenameFromDdownloadUrlPath(link: string): string {
|
||||
if (!isDdownloadLink(link)) return "";
|
||||
try {
|
||||
const parts = new URL(link).pathname.split("/").filter(Boolean);
|
||||
for (let index = parts.length - 1; index >= 1; index -= 1) {
|
||||
const normalized = normalizeResolvedFilename(safeDecode(parts[index]).replace(/\.html?$/i, ""));
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function extractDdownloadFilenameFromHtml(html: string): string {
|
||||
const patterns = [
|
||||
/class=["'][^"']*dk-dl-icon[^"']*["'][^>]*data-fn=["']([^"']+)["']/i,
|
||||
/data-fn=["']([^"']+)["'][^>]*class=["'][^"']*dk-dl-icon[^"']*["']/i,
|
||||
/<h2[^>]*class=["'][^"']*dk-dl-name[^"']*["'][^>]*>([^<]+)<\/h2>/i,
|
||||
/class=["'][^"']*file-info-name[^"']*["'][^>]*>([^<]+)</i
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
const normalized = normalizeResolvedFilename(html.match(pattern)?.[1] || "");
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function parseDdownloadFileSize(value: string | null | undefined): number | null {
|
||||
return parseRapidgatorFileSize(value);
|
||||
}
|
||||
|
||||
export async function checkDdownloadOnline(
|
||||
link: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<DdownloadCheckResult | null> {
|
||||
if (!isDdownloadLink(link)) return null;
|
||||
const headers = {
|
||||
"User-Agent": DDOWNLOAD_WEB_UA,
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9,de;q=0.8"
|
||||
};
|
||||
|
||||
for (let attempt = 1; attempt <= REQUEST_RETRIES + 1; attempt += 1) {
|
||||
try {
|
||||
if (signal?.aborted) throw new Error("aborted:debrid");
|
||||
const response = await fetch(link, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers,
|
||||
signal: withTimeoutSignal(signal, API_TIMEOUT_MS)
|
||||
});
|
||||
if (response.status === 404) {
|
||||
try { await response.body?.cancel(); } catch { }
|
||||
return { online: false, fileName: "", fileSizeBytes: null };
|
||||
}
|
||||
if (!response.ok) {
|
||||
try { await response.body?.cancel(); } catch { }
|
||||
if (shouldRetryStatus(response.status) && attempt <= REQUEST_RETRIES) {
|
||||
await sleepWithSignal(retryDelayForResponse(response, attempt), signal);
|
||||
continue;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const contentType = String(response.headers.get("content-type") || "").toLowerCase();
|
||||
const contentLength = Number(response.headers.get("content-length") || NaN);
|
||||
if (contentType
|
||||
&& !contentType.includes("text/html")
|
||||
&& !contentType.includes("application/xhtml")
|
||||
&& !contentType.includes("text/plain")) {
|
||||
try { await response.body?.cancel(); } catch { }
|
||||
return null;
|
||||
}
|
||||
if (!contentType && Number.isFinite(contentLength) && contentLength > DDOWNLOAD_SCAN_MAX_BYTES) {
|
||||
try { await response.body?.cancel(); } catch { }
|
||||
return null;
|
||||
}
|
||||
const html = await readResponseTextLimited(response, DDOWNLOAD_SCAN_MAX_BYTES, signal);
|
||||
if (DDOWNLOAD_FILE_NOT_FOUND_RE.test(html)) {
|
||||
return { online: false, fileName: "", fileSizeBytes: null };
|
||||
}
|
||||
const pageName = extractDdownloadFilenameFromHtml(html);
|
||||
const hasFilePage = Boolean(pageName)
|
||||
|| /\bdk-dl-(?:icon|name|size)\b/i.test(html)
|
||||
|| /Your file is ready to download|Regular Download via DDownload/i.test(html);
|
||||
if (!hasFilePage) return null;
|
||||
const sizeMatch = html.match(/class=["'][^"']*dk-dl-size[^"']*["'][^>]*>([^<]+)</i)
|
||||
|| html.match(/(?:File\s*size|Dateigröße)\s*[:\-]?\s*<[^>]*>([^<]+)</i);
|
||||
return {
|
||||
online: true,
|
||||
fileName: pageName || filenameFromDdownloadUrlPath(link),
|
||||
fileSizeBytes: parseDdownloadFileSize(sizeMatch?.[1])
|
||||
};
|
||||
} catch (error) {
|
||||
const errorText = compactErrorText(error);
|
||||
if (signal?.aborted || (/aborted/i.test(errorText) && !/timeout/i.test(errorText))) throw error;
|
||||
if (attempt > REQUEST_RETRIES || !isRetryableErrorText(errorText)) return null;
|
||||
}
|
||||
if (attempt <= REQUEST_RETRIES) await sleepWithSignal(retryDelay(attempt), signal);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
class DdownloadClient {
|
||||
private login: string;
|
||||
private password: string;
|
||||
private cookies: string = "";
|
||||
@@ -3899,8 +4013,7 @@ class DdownloadClient {
|
||||
|
||||
const idVal = html.match(/name="id" value="([^"]+)"/)?.[1] || fileCode;
|
||||
const randVal = html.match(/name="rand" value="([^"]+)"/)?.[1] || "";
|
||||
const fileNameMatch = html.match(/class="file-info-name"[^>]*>([^<]+)</);
|
||||
const fileName = fileNameMatch?.[1]?.trim() || filenameFromUrl(link);
|
||||
const fileName = extractDdownloadFilenameFromHtml(html) || filenameFromDdownloadUrlPath(link) || filenameFromUrl(link);
|
||||
|
||||
const dlBody = new URLSearchParams({
|
||||
op: "download2",
|
||||
|
||||
@@ -58,7 +58,7 @@ function releaseTlsSkip(): void {
|
||||
}
|
||||
import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup";
|
||||
import { planDownloadCompletion, reconcileFinalizedSize, validateDownloadedFileCompletion } from "./download-completion";
|
||||
import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkOneFichierLinks, checkRapidgatorOnline, fetchAllDebridHostInfo, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, isOneFichierLink, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown, type OneFichierCheckResult } from "./debrid";
|
||||
import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkDdownloadOnline, checkOneFichierLinks, checkRapidgatorOnline, fetchAllDebridHostInfo, filenameFromDdownloadUrlPath, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, isDdownloadLink, isOneFichierLink, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown, type DdownloadCheckResult, type OneFichierCheckResult } from "./debrid";
|
||||
import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, detectArchiveSignature, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, resetExtractorCachesForPasswordChange, type ExtractArchiveFailureInfo } from "./extractor";
|
||||
import { validateFileAgainstManifest } from "./integrity";
|
||||
import { classifyDiskError } from "./fs-error";
|
||||
@@ -2009,6 +2009,7 @@ export class DownloadManager extends EventEmitter {
|
||||
void this.recoverRetryableItems("startup").catch((err) => logger.warn(`recoverRetryableItems Fehler (startup): ${compactErrorText(err)}`));
|
||||
this.recoverPostProcessingOnStartup();
|
||||
this.checkExistingRapidgatorLinks();
|
||||
this.checkExistingDdownloadLinks();
|
||||
this.checkExistingOneFichierLinks();
|
||||
void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (constructor): ${compactErrorText(err)}`));
|
||||
setRotationEventListener(() => {
|
||||
@@ -3234,6 +3235,7 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
if (newItemIds.length > 0) {
|
||||
void this.checkRapidgatorLinks(newItemIds).catch((err) => logger.warn(`checkRapidgatorLinks Fehler: ${compactErrorText(err)}`));
|
||||
void this.checkDdownloadItems(newItemIds).catch((err) => logger.warn(`checkDdownloadItems Fehler: ${compactErrorText(err)}`));
|
||||
void this.checkOneFichierItems(newItemIds).catch((err) => logger.warn(`checkOneFichierItems Fehler: ${compactErrorText(err)}`));
|
||||
}
|
||||
return { addedPackages, addedLinks };
|
||||
@@ -3597,6 +3599,78 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
private async checkDdownloadItems(itemIds: string[]): Promise<void> {
|
||||
const itemsToCheck: Array<{ itemId: string; url: string }> = [];
|
||||
for (const itemId of itemIds) {
|
||||
const item = this.session.items[itemId];
|
||||
if (!item || !isDdownloadLink(item.url)) continue;
|
||||
if (item.status !== "queued" && item.status !== "reconnect_wait") continue;
|
||||
item.onlineStatus = "checking";
|
||||
itemsToCheck.push({ itemId, url: item.url });
|
||||
}
|
||||
if (itemsToCheck.length === 0) return;
|
||||
this.emitState();
|
||||
const checkedUrls = new Map<string, Promise<DdownloadCheckResult | null>>();
|
||||
await runWithLimitedConcurrency(itemsToCheck, 4, async ({ itemId, url }) => {
|
||||
let pending = checkedUrls.get(url);
|
||||
if (!pending) {
|
||||
pending = checkDdownloadOnline(url);
|
||||
checkedUrls.set(url, pending);
|
||||
}
|
||||
let result: DdownloadCheckResult | null = null;
|
||||
try {
|
||||
result = await pending;
|
||||
} catch (error) {
|
||||
logger.warn(`DDownload-Linkprüfung fehlgeschlagen: ${compactErrorText(error)}`);
|
||||
}
|
||||
const item = this.session.items[itemId];
|
||||
if (item) this.applyDdownloadCheckResult(item, result);
|
||||
this.persistSoon();
|
||||
this.emitState();
|
||||
});
|
||||
this.persistSoon();
|
||||
}
|
||||
|
||||
private applyDdownloadCheckResult(item: DownloadItem, result: DdownloadCheckResult | null): void {
|
||||
if (!result) {
|
||||
if (item.onlineStatus === "checking") item.onlineStatus = undefined;
|
||||
return;
|
||||
}
|
||||
const canUpdateMetadata = item.status === "queued"
|
||||
|| item.status === "reconnect_wait"
|
||||
|| (item.status === "validating" && item.downloadedBytes === 0 && (!item.targetPath || !fs.existsSync(item.targetPath)));
|
||||
if (item.status !== "queued" && item.status !== "reconnect_wait" && item.status !== "validating") {
|
||||
item.onlineStatus = result.online ? "online" : "offline";
|
||||
item.updatedAt = nowMs();
|
||||
return;
|
||||
}
|
||||
if (item.status === "validating" && !result.online) {
|
||||
item.onlineStatus = "offline";
|
||||
item.updatedAt = nowMs();
|
||||
return;
|
||||
}
|
||||
if (!result.online) {
|
||||
item.status = "failed";
|
||||
item.fullStatus = "Offline";
|
||||
item.lastError = "Datei nicht gefunden auf DDownload";
|
||||
item.onlineStatus = "offline";
|
||||
item.updatedAt = nowMs();
|
||||
if (this.runItemIds.has(item.id)) this.recordRunOutcome(item.id, "failed");
|
||||
const pkg = this.session.packages[item.packageId];
|
||||
if (pkg) this.refreshPackageStatus(pkg);
|
||||
return;
|
||||
}
|
||||
const unresolvedFileName = looksLikeOpaqueFilename(item.fileName)
|
||||
|| (!filenameFromDdownloadUrlPath(item.url) && item.fileName === filenameFromUrl(item.url));
|
||||
if (canUpdateMetadata && result.fileName && unresolvedFileName) {
|
||||
item.fileName = sanitizeFilename(result.fileName);
|
||||
this.assignItemTargetPath(item, path.join(this.session.packages[item.packageId]?.outputDir || this.settings.outputDir, item.fileName));
|
||||
}
|
||||
if (canUpdateMetadata && result.fileSizeBytes !== null && result.fileSizeBytes > 0) item.totalBytes = result.fileSizeBytes;
|
||||
item.onlineStatus = "online";
|
||||
item.updatedAt = nowMs();
|
||||
}
|
||||
|
||||
private async checkOneFichierItems(itemIds: string[]): Promise<void> {
|
||||
const itemIdsByUrl = new Map<string, string[]>();
|
||||
for (const itemId of itemIds) {
|
||||
@@ -3701,6 +3775,21 @@ export class DownloadManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
private checkExistingDdownloadLinks(): void {
|
||||
const uncheckedIds: string[] = [];
|
||||
for (const item of Object.values(this.session.items)) {
|
||||
if (item.status !== "queued" && item.status !== "reconnect_wait") continue;
|
||||
if (!isDdownloadLink(item.url) || item.onlineStatus === "offline") continue;
|
||||
const unresolvedFileName = looksLikeOpaqueFilename(item.fileName)
|
||||
|| (!filenameFromDdownloadUrlPath(item.url) && item.fileName === filenameFromUrl(item.url));
|
||||
if (item.onlineStatus === "online" && item.totalBytes !== null && item.totalBytes > 0 && !unresolvedFileName) continue;
|
||||
uncheckedIds.push(item.id);
|
||||
}
|
||||
if (uncheckedIds.length > 0) {
|
||||
void this.checkDdownloadItems(uncheckedIds).catch((err) => logger.warn(`checkDdownloadItems Fehler (startup): ${compactErrorText(err)}`));
|
||||
}
|
||||
}
|
||||
|
||||
private checkExistingOneFichierLinks(): void {
|
||||
const uncheckedIds: string[] = [];
|
||||
for (const item of Object.values(this.session.items)) {
|
||||
|
||||
Reference in New Issue
Block a user