fix(onefichier): resolve metadata during import

Batch 1Fichier link checks before downloads start, populate original filenames, exact sizes, and availability, preserve resolved names across generic debrid responses, and normalize all supported mirror domains under one hoster identity. Pace batches to the hoster's documented safe interval and cover offline, private, delayed, malformed, alias, and 100-link boundary cases.
2026-08-22 01:27:47 +02:00
parent a6c6464761
commit 5f391a35d6
8 changed files with 586 additions and 31 deletions
@@ -25,8 +25,11 @@ const ALL_DEBRID_API_BASE_V41 = "https://api.alldebrid.com/v4.1";

 const MEGA_DEBRID_API_BASE = "https://www.mega-debrid.eu/api.php";

-const ONEFICHIER_API_BASE = "https://api.1fichier.com/v1";
-const ONEFICHIER_URL_RE = /^https?:\/\/(?:www\.)?(?:1fichier\.com|alterupload\.com|cjoint\.net|desfichiers\.com|dfichiers\.com|megadl\.fr|mesfichiers\.org|piecejointe\.net|pjointe\.com|tenvoi\.com|dl4free\.com)\/\?([a-z0-9]{5,20})$/i;
+const ONEFICHIER_API_BASE = "https://api.1fichier.com/v1";
+const ONEFICHIER_CHECK_URL = "https://1fichier.com/check_links.pl";
+const ONEFICHIER_CHECK_BATCH_SIZE = 100;
+const ONEFICHIER_CHECK_BATCH_DELAY_MS = 1000;
+const ONEFICHIER_URL_RE = /^https?:\/\/(?:www\.)?(?:1fichier\.com|alterupload\.com|cjoint\.net|desfichiers\.(?:com|net)|dfichiers\.com|megadl\.fr|mesfichiers\.org|piecejointe\.net|pjointe\.com|tenvoi\.com|dl4free\.com)\/\?([a-z0-9]{5,20})$/i;

 const DEBRID_LINK_API_BASE = "https://debrid-link.com/api/v2";
 const DEBRID_LINK_KEY_QUOTA_ERRORS = new Set(["maxLink", "maxData"]);
@@ -1901,7 +1904,7 @@ export function parseRapidgatorFileSize(value: string | null | undefined): numbe
  return Number.isSafeInteger(bytes) ? bytes : null;
 }

-export async function checkRapidgatorOnline(
+export async function checkRapidgatorOnline(
  link: string,
  signal?: AbortSignal
 ): Promise<RapidgatorCheckResult | null> {
@@ -1975,10 +1978,132 @@ export async function checkRapidgatorOnline(
    }
  }

-  return null;
-}
-
-function buildBestDebridRequests(link: string, token: string): BestDebridRequest[] {
+  return null;
+}
+
+export interface OneFichierCheckResult {
+  online: boolean;
+  fileName: string;
+  fileSizeBytes: number | null;
+  accessRestricted: boolean;
+}
+
+export function isOneFichierLink(link: string): boolean {
+  return ONEFICHIER_URL_RE.test(String(link || "").trim());
+}
+
+function getOneFichierLinkId(link: string): string {
+  return String(link || "").trim().match(ONEFICHIER_URL_RE)?.[1]?.toLowerCase() || "";
+}
+
+function parseOneFichierCheckResponse(
+  responseText: string,
+  linksById: Map<string, string[]>
+): Map<string, OneFichierCheckResult> {
+  const results = new Map<string, OneFichierCheckResult>();
+  for (const rawLine of String(responseText || "").split(/\r?\n/)) {
+    const line = rawLine.trim();
+    if (!line) {
+      continue;
+    }
+    const id = line.match(/\?([a-z0-9]{5,20})(?:;|$)/i)?.[1]?.toLowerCase() || "";
+    const requestedLinks = linksById.get(id);
+    if (!requestedLinks || requestedLinks.length === 0) {
+      continue;
+    }
+
+    let result: OneFichierCheckResult | null = null;
+    if (/;;;(?:NOT FOUND|BAD LINK)\s*$/i.test(line)) {
+      result = { online: false, fileName: "", fileSizeBytes: null, accessRestricted: false };
+    } else if (/;;;PRIVATE\s*$/i.test(line)) {
+      result = { online: true, fileName: "", fileSizeBytes: null, accessRestricted: true };
+    } else {
+      const firstSeparator = line.indexOf(";");
+      const lastSeparator = line.lastIndexOf(";");
+      const fileName = decodeHtmlEntities(firstSeparator >= 0 && lastSeparator > firstSeparator
+        ? line.slice(firstSeparator + 1, lastSeparator)
+        : "").trim();
+      const fileSizeBytes = Number(lastSeparator >= 0 ? line.slice(lastSeparator + 1).trim() : NaN);
+      if (fileName && Number.isSafeInteger(fileSizeBytes) && fileSizeBytes >= 0) {
+        result = { online: true, fileName, fileSizeBytes, accessRestricted: false };
+      }
+    }
+
+    if (result) {
+      for (const link of requestedLinks) {
+        results.set(link, result);
+      }
+    }
+  }
+  return results;
+}
+
+export async function checkOneFichierLinks(
+  links: string[],
+  signal?: AbortSignal
+): Promise<Map<string, OneFichierCheckResult>> {
+  const supportedLinks = Array.from(new Set(links.map((link) => String(link || "").trim()).filter(isOneFichierLink)));
+  const results = new Map<string, OneFichierCheckResult>();
+
+  for (let offset = 0; offset < supportedLinks.length; offset += ONEFICHIER_CHECK_BATCH_SIZE) {
+    if (offset > 0) {
+      await sleepWithSignal(ONEFICHIER_CHECK_BATCH_DELAY_MS, signal);
+    }
+    const batch = supportedLinks.slice(offset, offset + ONEFICHIER_CHECK_BATCH_SIZE);
+    const linksById = new Map<string, string[]>();
+    const body = new URLSearchParams();
+    for (const link of batch) {
+      body.append("links[]", link);
+      const id = getOneFichierLinkId(link);
+      const existing = linksById.get(id) ?? [];
+      existing.push(link);
+      linksById.set(id, existing);
+    }
+
+    for (let attempt = 1; attempt <= REQUEST_RETRIES + 1; attempt += 1) {
+      try {
+        if (signal?.aborted) {
+          throw new Error("aborted:debrid");
+        }
+        const response = await fetch(ONEFICHIER_CHECK_URL, {
+          method: "POST",
+          headers: { "Content-Type": "application/x-www-form-urlencoded" },
+          body,
+          signal: withTimeoutSignal(signal, API_TIMEOUT_MS)
+        });
+        if (!response.ok) {
+          try { await response.body?.cancel(); } catch {  }
+          if (shouldRetryStatus(response.status) && attempt <= REQUEST_RETRIES) {
+            await sleepWithSignal(retryDelayForResponse(response, attempt), signal);
+            continue;
+          }
+          break;
+        }
+        const parsed = parseOneFichierCheckResponse(
+          await readResponseTextLimited(response, RAPIDGATOR_SCAN_MAX_BYTES, signal),
+          linksById
+        );
+        for (const [link, result] of parsed) {
+          results.set(link, result);
+        }
+        break;
+      } catch (error) {
+        const errorText = compactErrorText(error);
+        if (signal?.aborted || (/aborted/i.test(errorText) && !/timeout/i.test(errorText))) {
+          throw error;
+        }
+        if (attempt > REQUEST_RETRIES || !isRetryableErrorText(errorText)) {
+          break;
+        }
+        await sleepWithSignal(retryDelay(attempt), signal);
+      }
+    }
+  }
+
+  return results;
+}
+
+function buildBestDebridRequests(link: string, token: string): BestDebridRequest[] {
  const linkParam = encodeURIComponent(link);
  const safeToken = String(token || "").trim();
  const useAuthHeader = Boolean(safeToken);
@@ -3622,7 +3747,7 @@ class OneFichierClient {
  }

  public async unrestrictLink(link: string, signal?: AbortSignal): Promise<UnrestrictedLink> {
-    if (!ONEFICHIER_URL_RE.test(link)) {
+    if (!isOneFichierLink(link)) {
      throw new Error("Kein 1Fichier-Link");
    }

@@ -4110,7 +4235,7 @@ export class DebridService {
      }
    }

-    if (ONEFICHIER_URL_RE.test(link) && this.isProviderSelectableFor(settings, "onefichier")) {
+    if (isOneFichierLink(link) && this.isProviderSelectableFor(settings, "onefichier")) {
      try {
        const result = await this.unrestrictViaProvider(settings, "onefichier", link, signal);
        return {
@@ -58,7 +58,7 @@ function releaseTlsSkip(): void {
 }
 import { cleanupCancelledPackageArtifactsAsync, removeDownloadLinkArtifacts, removeSampleArtifacts } from "./cleanup";
 import { planDownloadCompletion, reconcileFinalizedSize, validateDownloadedFileCompletion } from "./download-completion";
-import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkRapidgatorOnline, fetchAllDebridHostInfo, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown } from "./debrid";
+import { AllDebridWebUnrestrictor, BestDebridWebUnrestrictor, DebridService, MegaWebUnrestrictor, RealDebridWebUnrestrictor, checkOneFichierLinks, checkRapidgatorOnline, fetchAllDebridHostInfo, getAvailableDebridLinkApiKeys, getAvailableMegaDebridAccounts, getAvailableRealDebridAccounts, getMegaDebridAccountCooldownState, getMegaDebridInFlightCountForMode, getRealDebridAccountAttemptTimeoutMs, isOneFichierLink, pruneExpiredDebridLinkRuntimeState, pruneExpiredMegaDebridRuntimeState, pruneExpiredRealDebridRuntimeState, releaseRealDebridAccountCooldown, type OneFichierCheckResult } from "./debrid";
 import { cleanupArchives, clearExtractResumeState, collectArchiveCleanupTargets, detectArchiveSignature, extractPackageArchives, findArchiveCandidates, hasAnyFilesRecursive, removeEmptyDirectoryTree, resetExtractorCachesForPasswordChange, type ExtractArchiveFailureInfo } from "./extractor";
 import { validateFileAgainstManifest } from "./integrity";
 import { classifyDiskError } from "./fs-error";
@@ -2007,9 +2007,10 @@ export class DownloadManager extends EventEmitter {
    this.resolveExistingQueuedOpaqueFilenames();
    this.revalidateCompletedItems();
    void this.recoverRetryableItems("startup").catch((err) => logger.warn(`recoverRetryableItems Fehler (startup): ${compactErrorText(err)}`));
-    this.recoverPostProcessingOnStartup();
-    this.checkExistingRapidgatorLinks();
-    void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (constructor): ${compactErrorText(err)}`));
+    this.recoverPostProcessingOnStartup();
+    this.checkExistingRapidgatorLinks();
+    this.checkExistingOneFichierLinks();
+    void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (constructor): ${compactErrorText(err)}`));
    setRotationEventListener(() => {
      if (this.rotationListenerActive === false) {
        return;
@@ -3231,9 +3232,10 @@ export class DownloadManager extends EventEmitter {
    if (unresolvedByLink.size > 0) {
      void this.resolveQueuedFilenames(unresolvedByLink).catch((err) => logger.warn(`resolveQueuedFilenames Fehler (addPackages): ${compactErrorText(err)}`));
    }
-    if (newItemIds.length > 0) {
-      void this.checkRapidgatorLinks(newItemIds).catch((err) => logger.warn(`checkRapidgatorLinks Fehler: ${compactErrorText(err)}`));
-    }
+    if (newItemIds.length > 0) {
+      void this.checkRapidgatorLinks(newItemIds).catch((err) => logger.warn(`checkRapidgatorLinks Fehler: ${compactErrorText(err)}`));
+      void this.checkOneFichierItems(newItemIds).catch((err) => logger.warn(`checkOneFichierItems Fehler: ${compactErrorText(err)}`));
+    }
    return { addedPackages, addedLinks };
  }

@@ -3558,7 +3560,7 @@ export class DownloadManager extends EventEmitter {
    }
  }

-  private applyRapidgatorCheckResult(item: DownloadItem, result: Awaited<ReturnType<typeof checkRapidgatorOnline>>): void {
+  private applyRapidgatorCheckResult(item: DownloadItem, result: Awaited<ReturnType<typeof checkRapidgatorOnline>>): void {
    if (!result) {
      if (item.onlineStatus === "checking") {
        item.onlineStatus = undefined;
@@ -3591,11 +3593,98 @@ export class DownloadManager extends EventEmitter {
        item.totalBytes = result.fileSizeBytes;
      }
      item.onlineStatus = "online";
-      item.updatedAt = nowMs();
-    }
-  }
-
-  private checkExistingRapidgatorLinks(): void {
+      item.updatedAt = nowMs();
+    }
+  }
+
+  private async checkOneFichierItems(itemIds: string[]): Promise<void> {
+    const itemIdsByUrl = new Map<string, string[]>();
+    for (const itemId of itemIds) {
+      const item = this.session.items[itemId];
+      if (!item || !isOneFichierLink(item.url)) {
+        continue;
+      }
+      if (item.status !== "queued" && item.status !== "reconnect_wait") {
+        continue;
+      }
+      item.onlineStatus = "checking";
+      const existing = itemIdsByUrl.get(item.url) ?? [];
+      existing.push(itemId);
+      itemIdsByUrl.set(item.url, existing);
+    }
+    if (itemIdsByUrl.size === 0) {
+      return;
+    }
+
+    this.emitState();
+    let results: Map<string, OneFichierCheckResult>;
+    try {
+      results = await checkOneFichierLinks(Array.from(itemIdsByUrl.keys()));
+    } catch (error) {
+      logger.warn(`1Fichier-Linkprüfung fehlgeschlagen: ${compactErrorText(error)}`);
+      results = new Map();
+    }
+
+    for (const [url, ids] of itemIdsByUrl) {
+      const result = results.get(url) ?? null;
+      for (const itemId of ids) {
+        const item = this.session.items[itemId];
+        if (item) {
+          this.applyOneFichierCheckResult(item, result);
+        }
+      }
+    }
+    this.persistSoon();
+    this.emitState();
+  }
+
+  private applyOneFichierCheckResult(item: DownloadItem, result: OneFichierCheckResult | null): void {
+    if (!result) {
+      if (item.onlineStatus === "checking") {
+        item.onlineStatus = undefined;
+      }
+      return;
+    }
+    const canUpdateMetadata = item.status === "queued"
+      || item.status === "reconnect_wait"
+      || (item.status === "validating" && item.downloadedBytes === 0 && (!item.targetPath || !fs.existsSync(item.targetPath)));
+    if (item.status !== "queued" && item.status !== "reconnect_wait" && item.status !== "validating") {
+      item.onlineStatus = result.online ? "online" : "offline";
+      item.updatedAt = nowMs();
+      return;
+    }
+    if (item.status === "validating" && !result.online) {
+      item.onlineStatus = "offline";
+      item.updatedAt = nowMs();
+      return;
+    }
+    if (!result.online) {
+      item.status = "failed";
+      item.fullStatus = "Offline";
+      item.lastError = "Datei nicht gefunden auf 1Fichier";
+      item.onlineStatus = "offline";
+      item.updatedAt = nowMs();
+      if (this.runItemIds.has(item.id)) {
+        this.recordRunOutcome(item.id, "failed");
+      }
+      const pkg = this.session.packages[item.packageId];
+      if (pkg) {
+        this.refreshPackageStatus(pkg);
+      }
+      return;
+    }
+    if (canUpdateMetadata && result.fileName && looksLikeOpaqueFilename(item.fileName)) {
+      item.fileName = sanitizeFilename(result.fileName);
+      this.assignItemTargetPath(item, path.join(this.session.packages[item.packageId]?.outputDir || this.settings.outputDir, item.fileName));
+    }
+    if (canUpdateMetadata && result.fileSizeBytes !== null && result.fileSizeBytes > 0) {
+      item.totalBytes = result.fileSizeBytes;
+    }
+    item.onlineStatus = "online";
+    item.updatedAt = nowMs();
+  }
+
+  private checkExistingRapidgatorLinks(): void {
    const uncheckedIds: string[] = [];
    for (const item of Object.values(this.session.items)) {
      if (item.status !== "queued") continue;
@@ -3607,10 +3696,29 @@ export class DownloadManager extends EventEmitter {
      } catch { continue; }
      uncheckedIds.push(item.id);
    }
-    if (uncheckedIds.length > 0) {
-      void this.checkRapidgatorLinks(uncheckedIds).catch((err) => logger.warn(`checkRapidgatorLinks Fehler (startup): ${compactErrorText(err)}`));
-    }
-  }
+    if (uncheckedIds.length > 0) {
+      void this.checkRapidgatorLinks(uncheckedIds).catch((err) => logger.warn(`checkRapidgatorLinks Fehler (startup): ${compactErrorText(err)}`));
+    }
+  }
+
+  private checkExistingOneFichierLinks(): void {
+    const uncheckedIds: string[] = [];
+    for (const item of Object.values(this.session.items)) {
+      if (item.status !== "queued" && item.status !== "reconnect_wait") {
+        continue;
+      }
+      if (!isOneFichierLink(item.url) || item.onlineStatus === "offline") {
+        continue;
+      }
+      if (item.onlineStatus === "online" && !looksLikeOpaqueFilename(item.fileName) && item.totalBytes !== null && item.totalBytes > 0) {
+        continue;
+      }
+      uncheckedIds.push(item.id);
+    }
+    if (uncheckedIds.length > 0) {
+      void this.checkOneFichierItems(uncheckedIds).catch((err) => logger.warn(`checkOneFichierItems Fehler (startup): ${compactErrorText(err)}`));
+    }
+  }

  private async cleanupExistingExtractedArchives(): Promise<void> {
    if (this.settings.cleanupMode === "none") {
@@ -9412,7 +9520,10 @@ export class DownloadManager extends EventEmitter {
        item.providerAccountId = unrestricted.sourceAccountId;
        item.providerAccountLabel = unrestricted.sourceAccountLabel;
        item.retries += unrestricted.retriesUsed;
-        item.fileName = sanitizeFilename(unrestricted.fileName || filenameFromUrl(item.url));
+        const unrestrictedFileName = sanitizeFilename(unrestricted.fileName || filenameFromUrl(item.url));
+        if (!looksLikeOpaqueFilename(unrestrictedFileName) || looksLikeOpaqueFilename(item.fileName)) {
+          item.fileName = unrestrictedFileName;
+        }
        let directHost = "";
        try {
          directHost = new URL(unrestricted.directUrl).host;