Handle source links going offline during downloads

This commit is contained in:
Sucukdeluxe
2026-09-02 11:32:06 +02:00
parent cd0464d658
commit ad75469854
15 changed files with 573 additions and 84 deletions
+270 -61
View File
@@ -116,7 +116,7 @@ type ActiveTask = {
itemId: string;
packageId: string;
abortController: AbortController;
abortReason: "stop" | "cancel" | "reconnect" | "package_toggle" | "stall" | "shutdown" | "reset" | "none";
abortReason: "stop" | "cancel" | "reconnect" | "package_toggle" | "stall" | "shutdown" | "reset" | "offline_skip" | "none";
resumable: boolean;
nonResumableCounted: boolean;
freshRetryUsed?: boolean;
@@ -722,14 +722,18 @@ export function getAuthoritativeRealDebridTotal(
|| evaluateCandidate(contentLength, "content-length");
}
function isPermanentLinkError(errorText: string): boolean {
const text = String(errorText || "").toLowerCase();
return text.includes("permanent ungültig")
|| /file.?not.?found/.test(text)
|| /file.?unavailable/.test(text)
|| /link.?is.?dead/.test(text)
|| text.includes("file has been removed")
|| text.includes("file has been deleted")
function isPermanentLinkError(errorText: string): boolean {
const text = String(errorText || "").toLowerCase();
return text.includes("source_link_offline:")
|| text.includes("permanent ungültig")
|| /(?:^|[^a-z0-9_])(?:file_unavailable|invalid_link|bad_link)(?:$|[^a-z0-9_])/.test(text)
|| /(?:^|[^a-z0-9_])file[ -]+unavailable(?:$|[^a-z0-9_])/.test(text)
|| /file.?not.?found/.test(text)
|| /link.?is.?dead/.test(text)
|| text.includes("datei nicht gefunden")
|| text.includes("datei nicht mehr verfügbar")
|| text.includes("file has been removed")
|| text.includes("file has been deleted")
|| text.includes("file is no longer available")
|| text.includes("file was removed")
|| text.includes("file was deleted");
@@ -834,9 +838,12 @@ function isProviderBusyUnrestrictError(errorText: string): boolean {
|| text.includes("zu viele downloads");
}
function isHosterUnavailableError(errorText: string): boolean {
return String(errorText || "").toLowerCase().includes("hosternotavailable");
}
function isHosterUnavailableError(errorText: string): boolean {
const text = String(errorText || "").toLowerCase();
return text.includes("hosternotavailable")
|| /hoster[ _-]?(?:temporarily[ _-]?)?unavailable/.test(text)
|| /hoster[ _-]?(?:maintenance|not[ _-]?supported)/.test(text);
}
function isTemporaryUnrestrictError(errorText: string): boolean {
const text = String(errorText || "").toLowerCase();
@@ -858,6 +865,20 @@ function isTemporaryUnrestrictError(errorText: string): boolean {
|| text.includes("worker error");
}
function isRapidgatorSourceLink(link: string): boolean {
try {
const hostname = new URL(link).hostname.toLowerCase();
return hostname === "rapidgator.net"
|| hostname.endsWith(".rapidgator.net")
|| hostname === "rg.to"
|| hostname.endsWith(".rg.to")
|| hostname === "rapidgator.asia"
|| hostname.endsWith(".rapidgator.asia");
} catch {
return false;
}
}
export function classifyProviderUnrestrictBackoff(errorText: string): "busy" | "temporary" | null {
const deepbridClassification = String(errorText || "").match(/deepbrid-anfrage fehlgeschlagen \((auth|rate_limit|temporary|link|malformed),/i)?.[1]?.toLowerCase();
if (deepbridClassification === "rate_limit") {
@@ -1635,15 +1656,15 @@ export function decideAutoRenameBaseName(
return { kind: "rename", baseName: targetBaseName, note };
}
const ARCHIVE_MULTIPART_RAR_RE = /^(.*)\.part0*1\.rar$/;
const ARCHIVE_RAR_RE = /^(.*)\.rar$/;
const ARCHIVE_ZIP_SPLIT_RE = /^(.*)\.zip\.001$/;
const ARCHIVE_7Z_SPLIT_RE = /^(.*)\.7z\.001$/;
const ARCHIVE_GENERIC_001_RE = /^(.*)\.001$/;
const ARCHIVE_KNOWN_001_RE = /\.(zip|7z)\.001$/;
const REGEX_ESCAPE_RE = /[.*+?^${}()|[\]\\]/g;
const ARCHIVE_MULTIPART_RAR_RE = /^(.*)\.part0*1\.rar$/;
const ARCHIVE_RAR_RE = /^(.*)\.rar$/;
const ARCHIVE_ZIP_SPLIT_RE = /^(.*)\.zip\.001$/;
const ARCHIVE_7Z_SPLIT_RE = /^(.*)\.7z\.001$/;
const ARCHIVE_GENERIC_001_RE = /^(.*)\.001$/;
const ARCHIVE_KNOWN_001_RE = /\.(zip|7z)\.001$/;
const REGEX_ESCAPE_RE = /[.*+?^${}()|[\]\\]/g;
export function resolveArchiveItemsFromList(archiveName: string, items: DownloadItem[]): DownloadItem[] {
export function resolveArchiveItemsFromList(archiveName: string, items: DownloadItem[]): DownloadItem[] {
const normalizeArchiveMatchName = (value: string): string =>
stripDuplicateSuffixBeforeExtension(path.basename(String(value || "")));
const entryLower = normalizeArchiveMatchName(archiveName).toLowerCase();
@@ -1716,8 +1737,58 @@ export function resolveArchiveItemsFromList(archiveName: string, items: Download
}
}
return [];
}
return [];
}
export function resolveOfflineArchiveItemsFromList(archiveName: string, items: DownloadItem[]): DownloadItem[] {
const normalizeArchiveMatchName = (value: string): string =>
stripDuplicateSuffixBeforeExtension(path.basename(String(value || "")));
const entryLower = normalizeArchiveMatchName(archiveName).toLowerCase();
const itemBaseName = (item: DownloadItem): string =>
normalizeArchiveMatchName(item.targetPath || item.fileName || "");
let pattern: RegExp | null = null;
const multipartMatch = entryLower.match(/^(.*)\.part0*\d+\.rar$/);
if (multipartMatch) {
const prefix = multipartMatch[1].replace(REGEX_ESCAPE_RE, "\\$&");
pattern = new RegExp(`^${prefix}\\.part\\d+\\.rar$`, "i");
}
if (!pattern) {
const rarMatch = entryLower.match(/^(.*)\.r(?:ar|\d{2,3})$/);
if (rarMatch) {
const stem = rarMatch[1].replace(REGEX_ESCAPE_RE, "\\$&");
pattern = new RegExp(`^${stem}\\.r(ar|\\d{2,3})$`, "i");
}
}
if (!pattern) {
const zipSplitMatch = entryLower.match(/^(.*)\.zip\.\d+$/);
if (zipSplitMatch) {
const stem = zipSplitMatch[1].replace(REGEX_ESCAPE_RE, "\\$&");
pattern = new RegExp(`^${stem}\\.zip(\\.\\d+)?$`, "i");
}
}
if (!pattern) {
const sevenSplitMatch = entryLower.match(/^(.*)\.7z\.\d+$/);
if (sevenSplitMatch) {
const stem = sevenSplitMatch[1].replace(REGEX_ESCAPE_RE, "\\$&");
pattern = new RegExp(`^${stem}\\.7z(\\.\\d+)?$`, "i");
}
}
if (!pattern && /^(.*)\.\d{3}$/.test(entryLower) && !/\.(zip|7z)\.\d{3}$/.test(entryLower)) {
const genericSplitMatch = entryLower.match(/^(.*)\.\d{3}$/);
if (genericSplitMatch) {
const stem = genericSplitMatch[1].replace(REGEX_ESCAPE_RE, "\\$&");
pattern = new RegExp(`^${stem}\\.\\d{3}$`, "i");
}
}
if (pattern) {
const matched = items.filter((item) => pattern!.test(itemBaseName(item)));
if (matched.length > 0) return matched;
}
return items.filter((item) => itemBaseName(item).toLowerCase() === entryLower);
}
function stripDuplicateSuffixBeforeExtension(fileName: string): string {
return String(fileName || "").replace(/ \(\d+\)(?=\.[^.]+$)/, "");
@@ -2032,6 +2103,8 @@ export class DownloadManager extends EventEmitter {
private retryAfterByItem = new Map<string, number>();
private sourceAvailabilityRecheckAt = new Map<string, number>();
private packageDiskRetryAfterByPackage = new Map<string, number>();
private diskWaitEvents: NonNullable<UiSnapshot["diskWaitEvents"]> = [];
@@ -3220,9 +3293,10 @@ export class DownloadManager extends EventEmitter {
if (!removedByPackageCleanup) {
delete this.session.items[itemId];
this.itemCount = Math.max(0, this.itemCount - 1);
}
this.retryAfterByItem.delete(itemId);
this.retryStateByItem.delete(itemId);
}
this.retryAfterByItem.delete(itemId);
this.sourceAvailabilityRecheckAt.delete(itemId);
this.retryStateByItem.delete(itemId);
this.dropItemContribution(itemId);
if (!hasActiveTask) {
this.releaseTargetPath(itemId);
@@ -3394,8 +3468,9 @@ export class DownloadManager extends EventEmitter {
clearTimeout(this.successDigestTimer);
this.successDigestTimer = null;
}
this.retryAfterByItem.clear();
this.providerStartReservations.clear();
this.retryAfterByItem.clear();
this.sourceAvailabilityRecheckAt.clear();
this.providerStartReservations.clear();
this.pacedStartReservationByItem.clear();
this.retryStateByItem.clear();
this.reservedTargetPaths.clear();
@@ -3634,8 +3709,9 @@ export class DownloadManager extends EventEmitter {
this.runItemIds.delete(itemId);
this.runOutcomes.delete(itemId);
this.retryAfterByItem.delete(itemId);
this.retryStateByItem.delete(itemId);
this.retryAfterByItem.delete(itemId);
this.sourceAvailabilityRecheckAt.delete(itemId);
this.retryStateByItem.delete(itemId);
}
this.abortPackagePostProcessing(packageId, "skip");
@@ -3881,6 +3957,9 @@ export class DownloadManager extends EventEmitter {
return;
}
if (item.status !== "queued") {
if (item.status === "failed" && item.onlineStatus === "offline") {
return;
}
item.onlineStatus = result.online ? "online" : "offline";
item.updatedAt = nowMs();
return;
@@ -3910,6 +3989,116 @@ export class DownloadManager extends EventEmitter {
}
}
private async confirmSourceOfflineAfterFailure(
item: DownloadItem,
errorText: string,
signal: AbortSignal
): Promise<"provider" | "hoster" | null> {
if (isPermanentLinkError(errorText)) {
return "provider";
}
if (item.onlineStatus === "offline") {
return "hoster";
}
if (item.status !== "validating" || item.onlineStatus === "checking") {
return null;
}
const supported = isRapidgatorSourceLink(item.url) || isDdownloadLink(item.url) || isOneFichierLink(item.url);
if (!supported) {
return null;
}
const checkedAt = nowMs();
const previousCheckAt = this.sourceAvailabilityRecheckAt.get(item.id) || 0;
if (checkedAt - previousCheckAt < 30_000) {
return null;
}
this.sourceAvailabilityRecheckAt.set(item.id, checkedAt);
try {
const checkSignal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]);
if (isRapidgatorSourceLink(item.url)) {
const result = await checkRapidgatorOnline(item.url, checkSignal);
return result && !result.online ? "hoster" : null;
}
if (isDdownloadLink(item.url)) {
const result = await checkDdownloadOnline(item.url, checkSignal);
return result && !result.online ? "hoster" : null;
}
const results = await checkOneFichierLinks([item.url], checkSignal);
const result = results.get(item.url);
return result && !result.online ? "hoster" : null;
} catch (error) {
if (signal.aborted) {
throw error;
}
logger.warn(`Quelllink-Nachprüfung fehlgeschlagen: item=${item.fileName || item.id}, error=${compactErrorText(error)}`);
return null;
}
}
private markItemOfflineAndSkipRelated(
pkg: PackageEntry,
item: DownloadItem,
errorText: string,
confirmedBy: "provider" | "hoster"
): void {
const cleanError = errorText
.replace(/^Error:\s*/i, "")
.replace(/^source_link_offline:(?:provider|hoster):/i, "")
.trim();
item.status = "failed";
item.onlineStatus = "offline";
item.lastError = cleanError || "Quelllink ist nicht mehr verfügbar";
item.fullStatus = "Offline";
item.speedBps = 0;
item.updatedAt = nowMs();
this.retryAfterByItem.delete(item.id);
this.retryStateByItem.delete(item.id);
this.sourceAvailabilityRecheckAt.delete(item.id);
this.recordRunOutcome(item.id, "failed");
const packageItems = pkg.itemIds
.map((itemId) => this.session.items[itemId])
.filter(Boolean) as DownloadItem[];
const scopeItems = this.settings.offlineSkipScope === "package"
? packageItems
: resolveOfflineArchiveItemsFromList(item.fileName, packageItems);
const skippedItems: DownloadItem[] = [];
for (const related of scopeItems) {
if (related.id === item.id || isFinishedStatus(related.status)) {
continue;
}
const relatedActive = this.activeTasks.get(related.id);
if (relatedActive) {
relatedActive.abortReason = "offline_skip";
relatedActive.abortController.abort("offline_skip");
}
related.status = "cancelled";
related.fullStatus = this.settings.offlineSkipScope === "package"
? "Übersprungen (Paket enthält Offline-Link)"
: "Übersprungen (Archivteil offline)";
related.lastError = `Übersprungen, weil ${item.fileName} offline ist`;
related.speedBps = 0;
related.updatedAt = nowMs();
this.retryAfterByItem.delete(related.id);
this.retryStateByItem.delete(related.id);
this.sourceAvailabilityRecheckAt.delete(related.id);
if (!relatedActive) {
this.releaseTargetPath(related.id);
}
this.recordRunOutcome(related.id, "cancelled");
skippedItems.push(related);
}
this.logPackageForItem(item, "WARN", "Quelllink während des Downloads offline geworden", {
confirmedBy,
offlineSkipScope: this.settings.offlineSkipScope,
skippedItems: skippedItems.length,
skippedNames: skippedItems.map((entry) => entry.fileName).join(" | ")
});
this.refreshPackageStatus(pkg);
}
private async checkDdownloadItems(itemIds: string[]): Promise<void> {
const itemsToCheck: Array<{ itemId: string; url: string }> = [];
for (const itemId of itemIds) {
@@ -3948,6 +4137,9 @@ export class DownloadManager extends EventEmitter {
if (item.onlineStatus === "checking") item.onlineStatus = undefined;
return;
}
if (item.status === "failed" && item.onlineStatus === "offline") {
return;
}
if (!result.online) {
item.onlineStatus = "offline";
item.updatedAt = nowMs();
@@ -4009,6 +4201,9 @@ export class DownloadManager extends EventEmitter {
if (item.onlineStatus === "checking") item.onlineStatus = undefined;
return;
}
if (item.status === "failed" && item.onlineStatus === "offline") {
return;
}
if (!result.online) {
item.onlineStatus = "offline";
item.updatedAt = nowMs();
@@ -6059,8 +6254,9 @@ export class DownloadManager extends EventEmitter {
this.dropItemContribution(itemId);
this.runOutcomes.delete(itemId);
this.runItemIds.delete(itemId);
this.retryAfterByItem.delete(itemId);
this.retryStateByItem.delete(itemId);
this.retryAfterByItem.delete(itemId);
this.sourceAvailabilityRecheckAt.delete(itemId);
this.retryStateByItem.delete(itemId);
item.status = "queued";
item.downloadedBytes = 0;
@@ -6142,8 +6338,9 @@ export class DownloadManager extends EventEmitter {
this.dropItemContribution(itemId);
this.runOutcomes.delete(itemId);
this.retryAfterByItem.delete(itemId);
this.retryStateByItem.delete(itemId);
this.retryAfterByItem.delete(itemId);
this.sourceAvailabilityRecheckAt.delete(itemId);
this.retryStateByItem.delete(itemId);
item.status = "queued";
item.downloadedBytes = 0;
@@ -6240,9 +6437,10 @@ export class DownloadManager extends EventEmitter {
item.status = "cancelled";
item.fullStatus = "Übersprungen";
item.speedBps = 0;
item.updatedAt = nowMs();
this.retryAfterByItem.delete(itemId);
this.retryStateByItem.delete(itemId);
item.updatedAt = nowMs();
this.retryAfterByItem.delete(itemId);
this.sourceAvailabilityRecheckAt.delete(itemId);
this.retryStateByItem.delete(itemId);
this.releaseTargetPath(itemId);
this.recordRunOutcome(itemId, "cancelled");
affectedPackageIds.add(item.packageId);
@@ -8842,9 +9040,10 @@ export class DownloadManager extends EventEmitter {
}
this.historyRecordedPackages.delete(packageId);
this.abortPackagePostProcessing(packageId, "package_removed");
for (const itemId of itemIds) {
this.retryAfterByItem.delete(itemId);
this.retryStateByItem.delete(itemId);
for (const itemId of itemIds) {
this.retryAfterByItem.delete(itemId);
this.sourceAvailabilityRecheckAt.delete(itemId);
this.retryStateByItem.delete(itemId);
this.releaseTargetPath(itemId);
this.dropItemContribution(itemId);
delete this.session.items[itemId];
@@ -10145,12 +10344,16 @@ export class DownloadManager extends EventEmitter {
}
}
);
} catch (unrestrictError) {
if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) {
this.recordProviderFailure(cooldownProvider);
} catch (unrestrictError) {
if (!active.abortController.signal.aborted && unrestrictTimeoutSignal.aborted) {
this.recordProviderFailure(cooldownProvider);
throw new Error(`Unrestrict Timeout nach ${Math.ceil(unrestrictTimeoutMs / 1000)}s`);
}
const errText = compactErrorText(unrestrictError);
}
const errText = compactErrorText(unrestrictError);
const offlineConfirmation = await this.confirmSourceOfflineAfterFailure(item, errText, active.abortController.signal);
if (offlineConfirmation) {
throw new Error(`source_link_offline:${offlineConfirmation}:${errText}`);
}
if (isProviderUnrestrictFailure(errText) && !isHosterUnavailableError(errText)) {
this.recordProviderFailure(cooldownProvider);
const backoffKind = classifyProviderUnrestrictBackoff(errText);
@@ -10481,9 +10684,22 @@ export class DownloadManager extends EventEmitter {
genericErrorRetries: Number(active.genericErrorRetries || 0),
unrestrictRetries: Number(active.unrestrictRetries || 0)
});
} else if (reason === "reset") {
this.retryStateByItem.delete(item.id);
} else if (reason === "package_toggle") {
} else if (reason === "reset") {
this.retryStateByItem.delete(item.id);
} else if (reason === "offline_skip") {
this.logPackageForItem(item, "WARN", "Download wegen Offline-Link im Zusammenhang übersprungen", {
reason
});
item.status = "cancelled";
item.speedBps = 0;
if (!/Übersprungen/.test(item.fullStatus || "")) {
item.fullStatus = "Übersprungen (Offline-Link im Zusammenhang)";
}
item.updatedAt = nowMs();
this.retryAfterByItem.delete(item.id);
this.retryStateByItem.delete(item.id);
this.recordRunOutcome(item.id, "cancelled");
} else if (reason === "package_toggle") {
this.logPackageForItem(item, "WARN", "Download wegen Paket-Toggle pausiert", {
reason
});
@@ -10693,20 +10909,13 @@ export class DownloadManager extends EventEmitter {
return;
}
if (isPermanentLinkError(errorText)) {
logger.error(`Link permanent ungültig: item=${item.fileName || item.id}, error=${errorText}, link=${item.url.slice(0, 80)}`);
item.status = "failed";
this.recordRunOutcome(item.id, "failed");
item.lastError = errorText;
item.fullStatus = `Link ungültig: ${errorText}`;
item.speedBps = 0;
item.updatedAt = nowMs();
this.retryStateByItem.delete(item.id);
const failPkgDead = this.session.packages[item.packageId];
if (failPkgDead) this.refreshPackageStatus(failPkgDead);
this.persistSoon();
this.emitState();
return;
if (isPermanentLinkError(errorText)) {
logger.error(`Link permanent ungültig: item=${item.fileName || item.id}, error=${errorText}, link=${item.url.slice(0, 80)}`);
const confirmedBy = /source_link_offline:hoster:/i.test(errorText) ? "hoster" : "provider";
this.markItemOfflineAndSkipRelated(pkg, item, errorText, confirmedBy);
this.persistSoon();
this.emitState();
return;
}
const totalNonStallFailures = (active.stallRetries || 0) + (active.unrestrictRetries || 0) + (active.genericErrorRetries || 0);