From b64cdd0ff35d0f2eab7efcf065f4644d7b5cbdf1 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe <259325684+Sucukdeluxe@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:43:48 +0200 Subject: [PATCH] fix: harden hoster confirmation and recovery Require every successful upload to expose a validated HTTPS result and rebuild all Doodstream and DSVPlay output from the confirmed file code. Keep failed baselines distinct from empty accounts, reject stale, foreign, and ambiguous recovery candidates across Doodstream, Byse, VOE, and Vidmoly, and preserve exact filename recovery with normalized extensions. Emit bounded structured transport diagnostics without raw response bodies or tokenized URLs, and remove sensitive values from Doodstream debug traces. Tests: node --test tests/upload-confirmation.test.js tests/hosters.test.js tests/doodstream-api-upload.test.js tests/doodstream-upload.test.js tests/byse-reject-recovery.test.js tests/hoster-recovery-provenance.test.js tests/suspect-reject-alternates.test.js Lint: eslint lib/hoster-transport-error.js lib/hosters.js lib/doodstream-upload.js lib/voe-upload.js lib/vidmoly-upload.js lib/upload-confirmation.js --- lib/doodstream-upload.js | 209 ++++++++--- lib/hoster-transport-error.js | 90 +++++ lib/hosters.js | 424 ++++++++++++++++------- lib/upload-confirmation.js | 41 +-- lib/vidmoly-upload.js | 257 +++++++++----- lib/voe-upload.js | 258 ++++++++++---- tests/byse-reject-recovery.test.js | 88 +++++ tests/doodstream-api-upload.test.js | 94 ++++- tests/doodstream-upload.test.js | 56 +++ tests/hoster-recovery-provenance.test.js | 118 +++++++ tests/hosters.test.js | 7 +- tests/upload-confirmation.test.js | 56 ++- 12 files changed, 1320 insertions(+), 378 deletions(-) create mode 100644 lib/hoster-transport-error.js create mode 100644 tests/hoster-recovery-provenance.test.js diff --git a/lib/doodstream-upload.js b/lib/doodstream-upload.js index 4cc45d1..d6a56a0 100644 --- a/lib/doodstream-upload.js +++ b/lib/doodstream-upload.js @@ -2,6 +2,12 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const { request } = require('undici'); +const { + createTransportError, + safeEndpoint, + sanitizeRemoteText, + summarizeResponse +} = require('./hoster-transport-error'); const BASE_URL = 'https://doodstream.com'; const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; @@ -97,8 +103,15 @@ class DoodstreamUploader { break; } catch (err) { if (opts.signal && opts.signal.aborted) throw err; // caller abort: don't retry - if (attempt >= 3) throw err; - _debugLog(`_fetch transient (${attempt}/3) ${url}: ${err && err.message}; retry`); + if (attempt >= 3) { + throw createTransportError('Doodstream: Webanfrage fehlgeschlagen', { + phase: 'web-request', + endpoint: url, + retryable: true, + transientNetwork: true + }); + } + _debugLog(`_fetch transient (${attempt}/3) ${safeEndpoint(url) || 'unknown endpoint'}: ${err && err.name ? err.name : 'network error'}; retry`); await new Promise(r => setTimeout(r, 400 * attempt)); } } @@ -167,16 +180,35 @@ class DoodstreamUploader { // Explicit success response } else if (json && json.message && /otp/i.test(json.message)) { // OTP required — signal caller to collect OTP from user - const err = new Error(`Doodstream Login: ${json.message}`); + const err = createTransportError(`Doodstream Login: ${sanitizeRemoteText(json.message)}`, { + phase: 'login', + endpoint: BASE_URL, + httpStatus: res.status, + contentType: res.headers && res.headers.get ? res.headers.get('content-type') : null, + body + }); err.otpRequired = true; throw err; } else if (json && json.status === 'fail') { - throw new Error(`Doodstream Login: ${json.message || 'Login fehlgeschlagen'}`); + throw createTransportError(`Doodstream Login: ${sanitizeRemoteText(json.message) || 'Login fehlgeschlagen'}`, { + phase: 'login', + endpoint: BASE_URL, + httpStatus: res.status, + contentType: res.headers && res.headers.get ? res.headers.get('content-type') : null, + body, + accountError: true + }); } else if (body.includes('Dashboard')) { // Got dashboard HTML directly — login worked } else { - const msg = (json && json.message) || 'Login fehlgeschlagen'; - throw new Error(`Doodstream Login: ${msg}`); + const msg = sanitizeRemoteText(json && json.message) || 'Login fehlgeschlagen'; + throw createTransportError(`Doodstream Login: ${msg}`, { + phase: 'login', + endpoint: BASE_URL, + httpStatus: res.status, + contentType: res.headers && res.headers.get ? res.headers.get('content-type') : null, + body + }); } } @@ -220,7 +252,7 @@ class DoodstreamUploader { const res = await this._fetch(BASE_URL + '/?op=upload_server'); const text = await res.text(); const ctype = (res.headers && res.headers.get) ? (res.headers.get('content-type') || '') : ''; - _debugLog(`upload_server: status=${res.status} ctype=${ctype} body(800)=${(text || '').slice(0, 800)}`); + _debugLog(`upload_server: status=${res.status} ctype=${ctype} response=${summarizeResponse(text, ctype)}`); let json; try { json = JSON.parse(text); } catch { json = null; } @@ -254,7 +286,7 @@ class DoodstreamUploader { // Capture the form's real fields so upload() submits exactly what the // browser would (file_title, submit_btn, …) instead of stale hardcoded ones. this._uploadFormFields = this._parseUploadFormFields(html); - _debugLog(`upload_server: using form action node=${url} sess=${this.sessId} fields=${Object.keys(this._uploadFormFields).join(',')}`); + _debugLog(`upload_server: using form action node=${safeEndpoint(url)} sessLength=${this.sessId.length} fields=${Object.keys(this._uploadFormFields).join(',')}`); return url; } @@ -265,15 +297,19 @@ class DoodstreamUploader { // No upload server could be extracted. We MUST NOT silently fall back to a // hardcoded node: that node is stale and accepts the bytes but returns an // empty form (no filecode) — so the user wastes ~90s uploading 95 MB into a - // dead end and gets a cryptic "kein Filecode" 90s later. Fail fast and put - // the raw responses in the error so the real format change is diagnosable. - const urlHints = (html.match(/https?:\/\/[^'">\s]+/g) || []).slice(0, 4).join(' , '); - _debugLog(`upload_server: NO SERVER. upload-page html(2000)=${(html || '').slice(0, 2000)}`); - throw new Error( - `Doodstream: konnte Upload-Server nicht ermitteln (Endpoint geändert?). ` + - `op=upload_server status=${res.status} ctype=${ctype} body=${(text || '').slice(0, 300)} ` + - `| upload-page URL-Treffer: ${urlHints || 'keine'}` - ); + // dead end and gets a cryptic "kein Filecode" 90s later. Fail fast with + // safe structured diagnostics. + _debugLog(`upload_server: no server response=${summarizeResponse(text, ctype)} page=${summarizeResponse(html, pageRes.headers && pageRes.headers.get ? pageRes.headers.get('content-type') : '')}`); + throw createTransportError('Doodstream: konnte Upload-Server nicht ermitteln', { + phase: 'upload-server', + endpoint: BASE_URL + '/?op=upload_server', + httpStatus: res.status, + contentType: ctype, + body: text, + retryable: res.status >= 500, + transientNetwork: res.status >= 500, + hosterTransient: res.status >= 500 + }); } /** @@ -370,15 +406,14 @@ class DoodstreamUploader { bodyTimeout: UPLOAD_TIMEOUT, headersTimeout: 60000 }); - } catch (err) { - // Label which phase failed so a future "fetch failed"/"terminated" is - // attributable to the big upload POST vs the small bookend requests. The - // original message is preserved as a substring so upload-manager's - // transient classification still matches. NOTE: undici may surface - // "terminated"/"other side closed", which are not yet in that transient - // list — revisit if logs show them. + } catch { const mb = Math.round(bytesRead / 1048576); - throw new Error(`Doodstream Upload-POST (${mb} MB an ${uploadUrl}): ${err && err.message ? err.message : err}`); + throw createTransportError(`Doodstream Upload-POST nach ${mb} MB fehlgeschlagen`, { + phase: 'upload-request', + endpoint: uploadUrl, + retryable: true, + transientNetwork: true + }); } const statusCode = uploadRes.statusCode; @@ -395,13 +430,22 @@ class DoodstreamUploader { } const resText = await uploadRes.body.text(); - _debugLog(`Upload response body (first 500): ${resText.slice(0, 500)}`); + const uploadContentType = uploadRes.headers && uploadRes.headers['content-type']; + _debugLog(`Upload response: ${summarizeResponse(resText, uploadContentType)}`); if (statusCode >= 400) { let payload; try { payload = JSON.parse(resText); } catch {} - const msg = payload && payload.msg ? payload.msg : resText.slice(0, 200); - throw new Error(`Doodstream Upload HTTP ${statusCode}: ${msg}`); + const msg = payload && payload.msg ? sanitizeRemoteText(payload.msg) : ''; + throw createTransportError(`Doodstream Upload fehlgeschlagen${msg ? `: ${msg}` : ''}`, { + phase: 'upload-response', + endpoint: uploadUrl, + httpStatus: statusCode, + contentType: uploadContentType, + body: resText, + retryable: statusCode === 429 || statusCode >= 500, + transientNetwork: statusCode >= 500 + }); } return this._parseUploadResponse(resText); @@ -411,10 +455,11 @@ class DoodstreamUploader { * Follow a redirect URL from upload server and extract filecode */ async _handleUploadResult(url) { - _debugLog(`Following upload result URL: ${url}`); + _debugLog(`Following upload result URL: ${safeEndpoint(url) || 'unknown endpoint'}`); const res = await this._fetch(url); const html = await res.text(); - _debugLog(`Result page (first 500): ${html.slice(0, 500)}`); + const contentType = res.headers && typeof res.headers.get === 'function' ? res.headers.get('content-type') : ''; + _debugLog(`Result page: ${summarizeResponse(html, contentType)}`); return this._parseUploadResponse(html); } @@ -458,12 +503,12 @@ class DoodstreamUploader { // 3. Parse HTML form (XFileSharing two-step upload) const hiddenFields = this._extractHiddenFields(resText); - _debugLog(`Hidden fields: ${JSON.stringify(hiddenFields)}`); + _debugLog(`Hidden fields: ${Object.keys(hiddenFields).join(',')}`); // Check if filecode is already in hidden fields const fnCode = hiddenFields.fn || hiddenFields.filecode || hiddenFields.file_code; if (fnCode && fnCode.length >= 8) { - _debugLog(`Filecode from hidden field 'fn': ${fnCode}`); + _debugLog(`Filecode from hidden field 'fn': length ${fnCode.length}`); // We still need to submit the form so doodstream registers the file // But the filecode is the 'fn' value } @@ -474,7 +519,7 @@ class DoodstreamUploader { // Ensure op=upload_result is set if (!hiddenFields.op) hiddenFields.op = 'upload_result'; - _debugLog(`Submitting upload_result to ${BASE_URL}/ with fields: ${JSON.stringify(hiddenFields)}`); + _debugLog(`Submitting upload_result fields: ${Object.keys(hiddenFields).join(',')}`); const formData = new URLSearchParams(hiddenFields); let followText = ''; try { @@ -487,18 +532,23 @@ class DoodstreamUploader { body: formData.toString() }); followText = await followRes.text(); - } catch (err) { + } catch { // The file already uploaded to the CDN; this POST only registers it on // doodstream's side. If it fails transiently (even after _fetch's own // retries) but we already hold the filecode, the upload succeeded from // the user's view — return it rather than discarding a done upload. if (fnCode && fnCode.length >= 8) { - _debugLog(`upload_result submit failed (${err && err.message}); using fn ${fnCode}`); + _debugLog(`upload_result submit failed; using existing filecode length ${fnCode.length}`); return this._buildResult(fnCode); } - throw err; + throw createTransportError('Doodstream Upload: Ergebnis konnte nicht registriert werden', { + phase: 'upload-result-submit', + endpoint: BASE_URL, + retryable: true, + transientNetwork: true + }); } - _debugLog(`upload_result response (first 500): ${followText.slice(0, 500)}`); + _debugLog(`upload_result response: ${summarizeResponse(followText, '')}`); // Try to find filecode in result page const resultCode = this._findFilecodeInHtml(followText); @@ -523,11 +573,17 @@ class DoodstreamUploader { // download link being empty while the page structure is unchanged points // at doodstream's backend, not at a parsing bug on our side. const st = hiddenFields.st || ''; - const fnInfo = fnCode ? `"${fnCode}"(len ${fnCode.length})` : 'fehlt/leer'; - const node = this._lastUploadUrl || '?'; - _debugLog(`No filecode. st=${st} fn=${fnInfo} node=${node} CDN-body=${(resText || '').slice(0, 400)}`); + const safeStatus = sanitizeRemoteText(st, 100); + const fnInfo = fnCode ? `vorhanden(len ${fnCode.length})` : 'fehlt/leer'; + const node = safeEndpoint(this._lastUploadUrl) || 'unbekannt'; + _debugLog(`No filecode. st=${safeStatus || '?'} fn=${fnInfo} node=${node} response=${summarizeResponse(resText, 'text/html')}`); if (st && st !== 'OK') { - throw new Error(`Doodstream lehnt Datei ab (Server-Status: ${st}). CDN=${node}`); + throw createTransportError(`Doodstream lehnt Datei ab (Server-Status: ${safeStatus || 'unbekannt'})`, { + phase: 'upload-result', + endpoint: this._lastUploadUrl || BASE_URL, + contentType: 'text/html', + body: resText + }); } // Empty form (no fn, no st) is a doodstream-side processing flake — same // account + same file works on a later attempt. Tag it explicitly so the @@ -536,15 +592,20 @@ class DoodstreamUploader { // session and later batches hit `pre-job-swap-blocked` for no fault of // the account). The flag is the primary signal; the message text is a // belt-and-suspenders regex fallback in the classifier. - const emptyLinkErr = new Error(`Doodstream Upload: kein Filecode — Server gab leeren Link zurück (st=${st || '?'}, fn=${fnInfo}, CDN=${node}). CDN-Antwort: ${(resText || '').slice(0, 200)}`); - emptyLinkErr.hosterTransient = true; - throw emptyLinkErr; + throw createTransportError(`Doodstream Upload: kein Filecode (st=${safeStatus || '?'}, fn=${fnInfo}, CDN=${node})`, { + phase: 'upload-result', + endpoint: this._lastUploadUrl || BASE_URL, + contentType: 'text/html', + body: resText, + retryable: true, + hosterTransient: true + }); } // 4. Fallback: follow form action as-is (for non-XFS forms) const formAction = resText.match(/]*action=['"]([^'"]+)['"]/i); if (formAction) { - _debugLog(`Fallback: following form action ${formAction[1]}`); + _debugLog(`Fallback: following form action ${safeEndpoint(formAction[1]) || 'unknown endpoint'}`); const formData = new URLSearchParams(hiddenFields); const followRes = await this._fetch(formAction[1], { method: 'POST', @@ -555,7 +616,7 @@ class DoodstreamUploader { body: formData.toString() }); const followText = await followRes.text(); - _debugLog(`Fallback response (first 500): ${followText.slice(0, 500)}`); + _debugLog(`Fallback response: ${summarizeResponse(followText, '')}`); const fallbackCode = this._findFilecodeInHtml(followText); if (fallbackCode) return this._buildResult(fallbackCode); @@ -563,10 +624,23 @@ class DoodstreamUploader { // Check if fn was in original hidden fields if (fnCode && fnCode.length >= 8) return this._buildResult(fnCode); - throw new Error(`Doodstream Upload: Redirect-Antwort ungültig (${followText.slice(0, 150)})`); + throw createTransportError('Doodstream Upload: Redirect-Antwort ungültig', { + phase: 'upload-result', + endpoint: formAction[1], + body: followText, + hosterTransient: true, + retryable: true + }); } - throw new Error(`Doodstream Upload: Keine gültige Antwort (Body: ${resText.slice(0, 150)})`); + throw createTransportError('Doodstream Upload: Keine gültige Antwort', { + phase: 'upload-result', + endpoint: this._lastUploadUrl || BASE_URL, + contentType: /<\s*(?:!doctype|html|body|form|input)\b/i.test(resText) ? 'text/html' : 'text/plain', + body: resText, + hosterTransient: true, + retryable: true + }); } /** @@ -590,7 +664,15 @@ class DoodstreamUploader { */ _extractFromJson(payload) { if (payload.status && Number(payload.status) !== 200 && payload.msg) { - throw new Error(`Doodstream Upload: ${payload.msg}`); + throw createTransportError(`Doodstream Upload: ${sanitizeRemoteText(payload.msg) || 'Antwort wurde abgelehnt'}`, { + phase: 'upload-result', + endpoint: this._lastUploadUrl || BASE_URL, + httpStatus: Number(payload.status), + contentType: 'application/json', + body: JSON.stringify(payload), + retryable: Number(payload.status) === 429 || Number(payload.status) >= 500, + transientNetwork: Number(payload.status) >= 500 + }); } let item = null; @@ -602,15 +684,28 @@ class DoodstreamUploader { } if (!item) { - throw new Error(`Doodstream Upload fehlgeschlagen: ${payload.msg || JSON.stringify(payload).slice(0, 150)}`); + throw createTransportError('Doodstream Upload: Antwort enthielt kein Ergebnis', { + phase: 'upload-result', + endpoint: this._lastUploadUrl || BASE_URL, + contentType: 'application/json', + body: JSON.stringify(payload), + hosterTransient: true, + retryable: true + }); } - const fileCode = item.filecode || item.file_code || ''; - return { - download_url: item.download_url || item.protected_dl || (fileCode ? `https://doodstream.com/d/${fileCode}` : null), - embed_url: item.protected_embed || (fileCode ? `https://doodstream.com/e/${fileCode}` : null), - file_code: fileCode - }; + const fileCode = String(item.filecode || item.file_code || '').trim(); + if (!fileCode) { + throw createTransportError('Doodstream Upload: Antwort enthielt keinen Filecode', { + phase: 'upload-result', + endpoint: this._lastUploadUrl || BASE_URL, + contentType: 'application/json', + body: JSON.stringify(payload), + hosterTransient: true, + retryable: true + }); + } + return this._buildResult(fileCode); } _buildResult(fileCode) { @@ -696,7 +791,7 @@ class DoodstreamUploader { return key; } } - _debugLog(`api-key derive: ${candidates.length} candidate(s), none validated. settings html(2500)=${(html || '').slice(0, 2500)}`); + _debugLog(`api-key derive: ${candidates.length} candidate(s), none validated. response=${summarizeResponse(html, 'text/html')}`); return null; } } diff --git a/lib/hoster-transport-error.js b/lib/hoster-transport-error.js new file mode 100644 index 0000000..59261b6 --- /dev/null +++ b/lib/hoster-transport-error.js @@ -0,0 +1,90 @@ +function normalizeContentType(value) { + const contentType = String(value || '').trim().slice(0, 120); + const parts = contentType.split(';').map(part => part.trim()); + if (parts.length < 1 || parts.length > 2) return null; + const slashIndex = parts[0].indexOf('/'); + if (slashIndex <= 0 || slashIndex === parts[0].length - 1) return null; + const tokenCharacters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.+-_'; + const validToken = token => Array.from(token).every(char => tokenCharacters.includes(char)); + if (!validToken(parts[0].slice(0, slashIndex)) || !validToken(parts[0].slice(slashIndex + 1))) return null; + if (parts.length === 2) { + const charsetPrefix = 'charset='; + if (!parts[1].toLowerCase().startsWith(charsetPrefix)) return null; + const charset = parts[1].slice(charsetPrefix.length); + if (!charset || !validToken(charset)) return null; + } + return contentType; +} + +function safeEndpoint(value) { + try { + const url = new URL(String(value || '')); + return `${url.hostname.toLowerCase()}${url.pathname}`; + } catch { + return null; + } +} + +function safeEndpointHost(value) { + try { + return new URL(String(value || '')).hostname.toLowerCase(); + } catch { + return null; + } +} + +function responseKind(body, contentType) { + const text = String(body || '').trim(); + if (!text) return 'empty'; + const type = String(contentType || '').toLowerCase(); + if (type.includes('json') || /^[\[{]/.test(text)) return 'json'; + if (type.includes('html') || /<\s*(?:!doctype|html|body|form|input)\b/i.test(text)) return 'html'; + return 'text'; +} + +function summarizeResponse(body, contentType) { + const text = String(body || ''); + const kind = responseKind(text, contentType); + return `${kind} response (${Buffer.byteLength(text, 'utf8')} bytes)`; +} + +function sanitizeRemoteText(value, limit = 180) { + let text = String(value || '').replace(/[\r\n\t]+/g, ' ').replace(/\s+/g, ' ').trim(); + text = text.replace(/https?:\/\/[^\s"'<>]+/gi, (raw) => safeEndpoint(raw) || '[URL]'); + text = text.replace(/\b(?:authorization|proxy-authorization|cookie|set-cookie)\s*[:=]\s*[^,]+/gi, '[redacted]'); + text = text.replace(/((?:api[_-]?key|token|password|secret|session|sess[_-]?id|csrf)["']?\s*[:=]\s*["']?)[^\s,;"'<>]+/gi, '$1[redacted]'); + text = text.replace(/(<(?:input|textarea)[^>]*(?:name|id)=["'][^"']*(?:key|token|password|secret|session|sess|csrf)[^"']*["'][^>]*(?:value=["']))[^"']*(["'])/gi, '$1[redacted]$2'); + text = text.replace(/\b[A-Za-z0-9_-]{20,}\b/g, '[redacted]'); + return text.slice(0, limit); +} + +function createTransportError(message, options = {}) { + const httpStatus = Number(options.httpStatus); + const hasHttpStatus = Number.isInteger(httpStatus) && httpStatus >= 100 && httpStatus <= 599; + const contentType = normalizeContentType(options.contentType); + const endpointHost = safeEndpointHost(options.endpoint); + const kind = responseKind(options.body, contentType); + const suffix = hasHttpStatus ? ` (HTTP ${httpStatus})` : ''; + const error = new Error(`${sanitizeRemoteText(message, 220)}${suffix}`); + error.diagnostic = { + phase: String(options.phase || 'transport').slice(0, 80), + http: hasHttpStatus ? httpStatus : null, + contentType, + safeEndpointHost: endpointHost, + responseKind: kind, + retryable: options.retryable === true, + payloadSnippet: summarizeResponse(options.body, contentType) + }; + if (options.transientNetwork === true) error.transientNetwork = true; + if (options.hosterTransient === true) error.hosterTransient = true; + if (options.accountError === true) error.accountError = true; + if (options.fileRejected === true) error.fileRejected = true; + return error; +} + +module.exports = { + createTransportError, + safeEndpoint, + sanitizeRemoteText, + summarizeResponse +}; diff --git a/lib/hosters.js b/lib/hosters.js index 2221642..5dd1c2e 100644 --- a/lib/hosters.js +++ b/lib/hosters.js @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const { request } = require('undici'); +const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error'); const UPLOAD_TIMEOUT = 1800000; // 30 minutes const API_TIMEOUT = 45000; // 45 seconds @@ -172,10 +173,11 @@ function parseDoodstreamResult(payload) { item = result; } + const fileCode = item.filecode || item.file_code || null; return { - download_url: item.download_url || item.protected_dl || null, - embed_url: item.protected_embed || null, - file_code: item.filecode || item.file_code || null + download_url: fileCode ? `https://doodstream.com/d/${fileCode}` : null, + embed_url: fileCode ? `https://doodstream.com/e/${fileCode}` : null, + file_code: fileCode }; } @@ -234,7 +236,7 @@ function parseByseResult(payload) { // wall, so we must rotate. File-specific rejections (Duplicate, wrong // format, too small/large) ARE per-file and rotation is pointless. const accountLevel = /(not enough (disk )?(space|storage)|insufficient (disk )?space|disk (space )?full|storage (exhausted|full|voll|limit)|quota (exceeded|voll|überschritten)|account (full|voll|suspended|banned))/i.test(perFileError); - const err = new Error(`Byse lehnte Datei ab: ${perFileError}`); + const err = new Error(`Byse lehnte Datei ab: ${sanitizeRemoteText(perFileError)}`); if (accountLevel) { err.accountError = true; } else { @@ -308,32 +310,64 @@ function createUploadBody(filePath, formFields, onProgress, throttle, signal) { // --- API helper using built-in fetch (follows redirects automatically) --- -async function apiGet(url, signal) { +async function apiGet(url, signal, hosterName) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), API_TIMEOUT); const onAbort = () => controller.abort(); if (signal) signal.addEventListener('abort', onAbort); try { - const res = await fetch(url, { - method: 'GET', - signal: controller.signal, - redirect: 'follow' - }); + let res; + try { + res = await fetch(url, { + method: 'GET', + signal: controller.signal, + redirect: 'follow' + }); + } catch (err) { + if (signal && signal.aborted) throw err; + throw createTransportError(`${hosterName}: Upload-Server-Abfrage fehlgeschlagen`, { + phase: 'upload-server', + endpoint: url, + retryable: true, + transientNetwork: true + }); + } const text = await res.text(); + const contentType = res.headers && typeof res.headers.get === 'function' + ? res.headers.get('content-type') + : null; let data; try { data = JSON.parse(text); } catch { - const err = new Error(`API-Antwort war kein JSON (HTTP ${res.status}): ${(text || '').slice(0, 200)}`); - if (res.status >= 500) err.transientNetwork = true; - throw err; + throw createTransportError(`${hosterName}: Upload-Server-Antwort war kein JSON`, { + phase: 'upload-server', + endpoint: url, + httpStatus: res.status, + contentType, + body: text, + retryable: res.status >= 500, + transientNetwork: res.status >= 500 + }); } - if (data.status && [401, 403, 429, 500].includes(data.status)) { - const err = new Error(data.msg || data.message || JSON.stringify(data)); - if (data.status === 500) err.transientNetwork = true; - throw err; + const apiStatus = Number(data && data.status); + const effectiveStatus = res.status < 200 || res.status >= 300 + ? res.status + : (apiStatus >= 400 ? apiStatus : null); + if (effectiveStatus) { + const retryable = effectiveStatus === 429 || effectiveStatus >= 500; + throw createTransportError(`${hosterName}: Upload-Server-Abfrage wurde abgelehnt`, { + phase: 'upload-server', + endpoint: url, + httpStatus: effectiveStatus, + contentType, + body: text, + retryable, + transientNetwork: effectiveStatus >= 500, + accountError: effectiveStatus === 401 || effectiveStatus === 403 + }); } return data; } finally { @@ -347,12 +381,14 @@ async function apiGet(url, signal) { async function getUploadServer(hosterName, hosterConfig, apiKey, signal) { let lastMessage = ''; let lastTransient = false; + let lastError = null; for (let attempt = 1; attempt <= SERVER_RETRY_ATTEMPTS; attempt++) { for (const endpoint of hosterConfig.serverEndpoints) { const url = `${hosterConfig.apiBase}${endpoint}?key=${encodeURIComponent(apiKey)}`; try { - const data = await apiGet(url, signal); + const data = await apiGet(url, signal, hosterName); + lastError = null; const uploadUrl = extractUploadServerUrl(data, hosterConfig.apiBase); if (uploadUrl) { LAST_UPLOAD_SERVERS.set(hosterName, uploadUrl); @@ -365,12 +401,16 @@ async function getUploadServer(hosterName, hosterConfig, apiKey, signal) { if (apiMessage) lastMessage = apiMessage; } catch (err) { if (err.name === 'AbortError') throw err; + lastError = err; if (err.message) lastMessage = err.message; if (err.transientNetwork === true) lastTransient = true; } } - if (attempt < SERVER_RETRY_ATTEMPTS && shouldRetryServerLookup(lastMessage)) { + const retryable = lastError && lastError.diagnostic + ? lastError.diagnostic.retryable === true + : shouldRetryServerLookup(lastMessage); + if (attempt < SERVER_RETRY_ATTEMPTS && retryable) { await sleep(SERVER_RETRY_DELAY_MS, signal); continue; } @@ -379,11 +419,14 @@ async function getUploadServer(hosterName, hosterConfig, apiKey, signal) { } const cachedServer = LAST_UPLOAD_SERVERS.get(hosterName); - if (cachedServer && shouldRetryServerLookup(lastMessage)) { + const retryable = lastError && lastError.diagnostic + ? lastError.diagnostic.retryable === true + : shouldRetryServerLookup(lastMessage); + if (cachedServer && retryable) { return cachedServer; } - if (shouldRetryServerLookup(lastMessage) && Array.isArray(hosterConfig.fallbackUploadServers)) { + if (retryable && Array.isArray(hosterConfig.fallbackUploadServers)) { for (const fallback of hosterConfig.fallbackUploadServers) { const normalized = normalizeAbsoluteUrl(fallback, hosterConfig.apiBase); if (normalized) { @@ -394,43 +437,122 @@ async function getUploadServer(hosterName, hosterConfig, apiKey, signal) { } if (lastMessage) { - const e = new Error(`Kein Upload-Server erhalten: ${lastMessage}`); - // "no servers available" / busy / try-again is a transient hoster-side - // condition, not an account fault — tag it so the account isn't blacklisted. - // Genuine auth failures (invalid key / unauthorized / forbidden) make - // shouldRetryServerLookup return false and stay classified as account errors. - if (shouldRetryServerLookup(lastMessage)) e.hosterTransient = true; + const e = lastError || createTransportError(`Kein Upload-Server für ${hosterName} erhalten`, { + phase: 'upload-server', + endpoint: hosterConfig.apiBase, + retryable + }); + if (retryable) e.hosterTransient = true; if (lastTransient) e.transientNetwork = true; throw e; } - throw new Error('Kein Upload-Server erhalten. API-Key prüfen.'); + throw createTransportError(`Kein Upload-Server für ${hosterName} erhalten`, { + phase: 'upload-server', + endpoint: hosterConfig.apiBase + }); } -async function _fetchByseFileList(apiKey, signal) { +async function _requestFileList(url, signal, phase, hosterName) { + let response; + try { + response = await request(url, { + method: 'GET', signal, + headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' }, + headersTimeout: 30_000, bodyTimeout: 30_000 + }); + } catch (err) { + if (signal && signal.aborted) throw err; + throw createTransportError(`${hosterName}: Dateiliste konnte nicht geladen werden`, { + phase, + endpoint: url, + retryable: true, + transientNetwork: true + }); + } + + const contentType = response.headers && response.headers['content-type']; + let text; + try { + text = await response.body.text(); + } catch (err) { + if (signal && signal.aborted) throw err; + throw createTransportError(`${hosterName}: Dateiliste konnte nicht gelesen werden`, { + phase, + endpoint: url, + httpStatus: response.statusCode, + contentType, + retryable: true, + transientNetwork: true + }); + } + + if (response.statusCode < 200 || response.statusCode >= 300) { + const retryable = response.statusCode === 429 || response.statusCode >= 500; + throw createTransportError(`${hosterName}: Dateiliste konnte nicht geladen werden`, { + phase, + endpoint: url, + httpStatus: response.statusCode, + contentType, + body: text, + retryable, + transientNetwork: response.statusCode >= 500 + }); + } + + let data; + try { + data = JSON.parse(text); + } catch { + throw createTransportError(`${hosterName}: Dateiliste war kein JSON`, { + phase, + endpoint: url, + httpStatus: response.statusCode, + contentType, + body: text + }); + } + + if (!data || typeof data !== 'object') { + throw createTransportError(`${hosterName}: Dateiliste hatte ein ungültiges Format`, { + phase, + endpoint: url, + httpStatus: response.statusCode, + contentType, + body: text + }); + } + + const apiStatus = Number(data && data.status); + if (apiStatus >= 400) { + const retryable = apiStatus === 429 || apiStatus >= 500; + throw createTransportError(`${hosterName}: Dateiliste wurde abgelehnt`, { + phase, + endpoint: url, + httpStatus: apiStatus, + contentType, + body: text, + retryable, + transientNetwork: apiStatus >= 500 + }); + } + + return data; +} + +async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') { // Byse's file-list endpoint. Returns up to 100 most-recent files — enough // to match the upload we just did against what the server has. The API // shape is typical XFS: { status, msg, result: { files: [...] } } or // { status, msg, files: [...] }. const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`; - try { - const { body, statusCode } = await request(url, { - method: 'GET', signal, - headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' }, - headersTimeout: 30_000, bodyTimeout: 30_000 - }); - const text = await body.text(); - if (statusCode < 200 || statusCode >= 300) return []; - const data = JSON.parse(text); - const src = Array.isArray(data.files) ? data.files - : (data.result && Array.isArray(data.result.files) ? data.result.files - : (Array.isArray(data.result) ? data.result : [])); - return src.map(f => ({ - file_code: String(f.file_code || f.filecode || '').trim(), - file_name: String(f.title || f.name || f.file_name || '').trim() - })).filter(f => f.file_code); - } catch { - return []; - } + const data = await _requestFileList(url, signal, phase, 'Byse'); + const src = Array.isArray(data.files) ? data.files + : (data.result && Array.isArray(data.result.files) ? data.result.files + : (Array.isArray(data.result) ? data.result : [])); + return src.map(f => ({ + file_code: String(f.file_code || f.filecode || '').trim(), + file_name: String(f.title || f.name || f.file_name || '').trim() + })).filter(f => f.file_code); } function _normalizeFileTitle(s) { @@ -438,6 +560,7 @@ function _normalizeFileTitle(s) { } async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) { + if (!(baselineCodes instanceof Set)) return null; const expected = _normalizeFileTitle(fileName); const POLL_ATTEMPTS = 15; const POLL_DELAY_MS = 2000; @@ -450,8 +573,10 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) // poller could claim job B's newly appeared file and return the wrong // URL. At the cost of a few false-negatives when byse mangles the // filename beyond our normalizer, correctness for parallel uploads wins. - const match = newFiles.find(f => _normalizeFileTitle(f.file_name) === expected); - if (match) { + const matches = newFiles.filter(f => _normalizeFileTitle(f.file_name) === expected); + if (matches.length > 1) return null; + if (matches.length === 1) { + const match = matches[0]; return { download_url: `https://byse.sx/d/${match.file_code}`, embed_url: `https://byse.sx/e/${match.file_code}`, @@ -469,34 +594,24 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) return null; } -async function _fetchDoodstreamFileList(apiKey, signal) { +async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll') { // doodapi.co file list: { msg, status:200, result: { files: [{ file_code, title, uploaded, ... }] } } // sort=created&order=desc forces newest-first — VERIFIED against a real 90k-file // account, where a single page without it could miss a just-uploaded file. The // recovery only needs the most recent uploads, so page 1 newest-first suffices. const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`; - try { - const { body, statusCode } = await request(url, { - method: 'GET', signal, - headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' }, - headersTimeout: 30_000, bodyTimeout: 30_000 - }); - const text = await body.text(); - if (statusCode < 200 || statusCode >= 300) return []; - const data = JSON.parse(text); - const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : []; - return files.map(f => ({ - file_code: String(f.file_code || f.filecode || '').trim(), - file_name: String(f.title || f.file_name || f.name || '').trim() - })).filter(f => f.file_code); - } catch { - return []; - } + const data = await _requestFileList(url, signal, phase, 'Doodstream'); + const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : []; + return files.map(f => ({ + file_code: String(f.file_code || f.filecode || '').trim(), + file_name: String(f.title || f.file_name || f.name || '').trim() + })).filter(f => f.file_code); } const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal) { + if (!(baselineCodes instanceof Set)) return null; // Same recovery byse uses: the upload POST returned no filecode, but the file // may register in the account a little later. Poll the list for a NEW file // whose normalized title matches what we uploaded. Exact-name match only @@ -509,8 +624,10 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s if (signal && signal.aborted) return null; const list = await _fetchDoodstreamFileList(apiKey, signal); const fresh = list.filter(f => !baselineCodes.has(f.file_code)); - const match = fresh.find(f => _normalizeFileTitle(f.file_name) === expected); - if (match) { + const matches = fresh.filter(f => _normalizeFileTitle(f.file_name) === expected); + if (matches.length > 1) return null; + if (matches.length === 1) { + const match = matches[0]; return { download_url: `https://doodstream.com/d/${match.file_code}`, embed_url: `https://doodstream.com/e/${match.file_code}`, @@ -533,21 +650,33 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro if (!config) throw new Error(`Unbekannter Hoster: ${hosterName}`); let byseBaseline = null; + let byseBaselineError = null; if (hosterName === 'byse.sx') { if (opts && opts.byseBaseline instanceof Set) { byseBaseline = opts.byseBaseline; } else { - const baseline = await _fetchByseFileList(apiKey, signal); - byseBaseline = new Set(baseline.map(f => f.file_code)); + try { + const baseline = await _fetchByseFileList(apiKey, signal, 'recovery-baseline'); + byseBaseline = new Set(baseline.map(f => f.file_code)); + } catch (err) { + if (signal && signal.aborted) throw err; + byseBaselineError = err; + } } } let doodBaseline = null; + let doodBaselineError = null; if (hosterName === 'doodstream.com') { if (opts && opts.doodBaseline instanceof Set) { doodBaseline = opts.doodBaseline; } else { - const baseline = await _fetchDoodstreamFileList(apiKey, signal); - doodBaseline = new Set(baseline.map(f => f.file_code)); + try { + const baseline = await _fetchDoodstreamFileList(apiKey, signal, 'recovery-baseline'); + doodBaseline = new Set(baseline.map(f => f.file_code)); + } catch (err) { + if (signal && signal.aborted) throw err; + doodBaselineError = err; + } } } @@ -560,31 +689,47 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro const { iterable, boundary, totalSize } = createUploadBody(filePath, formFields, onProgress, throttle, signal); - const { body, statusCode, headers } = await request(targetUrl, { - method: 'POST', - body: iterable, - signal, - headers: { - 'Content-Type': `multipart/form-data; boundary=${boundary}`, - 'Content-Length': String(totalSize), - 'Accept': 'application/json, text/plain;q=0.9, */*;q=0.8', - 'User-Agent': 'multi-hoster-uploader/1.1' - }, - headersTimeout: UPLOAD_TIMEOUT, - bodyTimeout: UPLOAD_TIMEOUT - }); + let uploadResponse; + try { + uploadResponse = await request(targetUrl, { + method: 'POST', + body: iterable, + signal, + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize), + 'Accept': 'application/json, text/plain;q=0.9, */*;q=0.8', + 'User-Agent': 'multi-hoster-uploader/1.1' + }, + headersTimeout: UPLOAD_TIMEOUT, + bodyTimeout: UPLOAD_TIMEOUT + }); + } catch (err) { + if (signal && signal.aborted) throw err; + throw createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, { + phase: 'upload-request', + endpoint: targetUrl, + retryable: true, + transientNetwork: true + }); + } + + const { body, statusCode, headers } = uploadResponse; const rawBody = await body.text(); let payload = null; try { payload = rawBody ? JSON.parse(rawBody) : {}; } catch { - const snippet = rawBody ? rawBody.slice(0, 240).replace(/\s+/g, ' ').trim() : ''; - const err = new Error( - `Upload-Antwort von ${hosterName} war kein JSON (HTTP ${statusCode}${snippet ? `): ${snippet}` : ')'}` - ); - if (statusCode >= 500) err.transientNetwork = true; - throw err; + throw createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, { + phase: 'upload-response', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody, + retryable: statusCode >= 500, + transientNetwork: statusCode >= 500 + }); } // Normalize valid-but-not-object JSON (JSON.parse('null') → null; // JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this @@ -598,19 +743,27 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro } if (statusCode < 200 || statusCode >= 300) { - const err = new Error( - payload.msg - || payload.message - || `Upload fehlgeschlagen (HTTP ${statusCode}${headers?.['content-type'] ? `, ${headers['content-type']}` : ''})` - ); - if (statusCode >= 500) err.transientNetwork = true; - throw err; + throw createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, { + phase: 'upload-response', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody, + retryable: statusCode === 429 || statusCode >= 500, + transientNetwork: statusCode >= 500 + }); } if (payload.status && [401, 403, 429, 500].includes(payload.status)) { - const err = new Error(payload.msg || payload.message || JSON.stringify(payload)); - if (payload.status === 500) err.transientNetwork = true; - throw err; + throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, { + phase: 'upload-response', + endpoint: targetUrl, + httpStatus: Number(payload.status), + contentType: headers && headers['content-type'], + body: rawBody, + retryable: Number(payload.status) === 429 || Number(payload.status) >= 500, + transientNetwork: Number(payload.status) >= 500 + }); } let result = null; @@ -619,15 +772,13 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro result = config.parseResult(payload); } catch (err) { if (err && typeof err === 'object' && !err.diagnostic) { - try { - err.diagnostic = { - hoster: hosterName, - http: statusCode, - contentType: (headers && headers['content-type']) || null, - payloadSnippet: JSON.stringify(payload).slice(0, 1000), - uploadUrl: targetUrl - }; - } catch { /* JSON cycle — skip diagnostic */ } + err.diagnostic = createTransportError(`Upload zu ${hosterName} konnte nicht ausgewertet werden`, { + phase: 'upload-result', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody + }).diagnostic; } parseErr = err; } @@ -670,20 +821,35 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro if (polled) return polled; } + if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) { + byseBaselineError.hosterTransient = true; + throw byseBaselineError; + } + + if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) { + doodBaselineError.hosterTransient = true; + throw doodBaselineError; + } + if (parseErr) throw parseErr; if (payload.success === false) { - throw new Error(payload.msg || payload.message || `Upload zu ${hosterName} wurde vom Server abgelehnt.`); + throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, { + phase: 'upload-result', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody + }); } // Avoid throwing a bare "OK" / "SUCCESS" as the error message — that happens // when the server says "msg: OK" but ships no file_code anywhere we know - // about, typically an API change. Surface the full (trimmed) payload so - // future logs actually show what the server returned. + // about, typically an API change. Surface safe structured response metadata + // so future logs show what kind of response the server returned. const msg = String(payload.msg || payload.message || '').trim(); const isOkishNoPayload = /^(ok|success|done|accepted)$/i.test(msg); if (isOkishNoPayload || !msg) { - const snippet = JSON.stringify(payload).slice(0, 400); // 2xx with no filecode: the hoster accepted the upload (bytes sent, status // OK) but returned no usable link. For doodstream this is the API-path // analog of the web empty-form — the backend file-registration timing out @@ -691,23 +857,33 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro // so tag it hosterTransient: the upload-manager then fails this file WITHOUT // blacklisting the account (same protection the web path got in 3.3.29) and // the account stays usable for the next retry/batch. - const err = new Error( - `Upload zu ${hosterName} lieferte keine file_code-Antwort (Payload: ${snippet})` - ); - err.hosterTransient = true; - throw err; + throw createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, { + phase: 'upload-result', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody, + retryable: true, + hosterTransient: true + }); } - throw new Error(msg); + throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, { + phase: 'upload-result', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody + }); } async function prefetchBaseline(hosterName, apiKey, signal) { try { if (hosterName === 'byse.sx') { - const baseline = await _fetchByseFileList(apiKey, signal); + const baseline = await _fetchByseFileList(apiKey, signal, 'recovery-baseline'); return new Set(baseline.map(f => f.file_code)); } if (hosterName === 'doodstream.com') { - const baseline = await _fetchDoodstreamFileList(apiKey, signal); + const baseline = await _fetchDoodstreamFileList(apiKey, signal, 'recovery-baseline'); return new Set(baseline.map(f => f.file_code)); } } catch { /* leave caller to fall back to per-job fetch */ } diff --git a/lib/upload-confirmation.js b/lib/upload-confirmation.js index 754a304..4de4655 100644 --- a/lib/upload-confirmation.js +++ b/lib/upload-confirmation.js @@ -11,13 +11,13 @@ const HOSTER_RESULT_DOMAINS = { 'doodstream.com': ['doodstream.com', 'dood.to', 'dood.la', 'dood.so', 'dsvplay.com'] }; -function isExpectedHostUrl(value, expectedHost) { +function isExpectedHostUrl(value, expectedHost, allowHttp = false) { if (typeof value !== 'string' || value.trim() === '') return false; try { const url = new URL(value); const hostname = url.hostname.toLowerCase(); const acceptedDomains = HOSTER_RESULT_DOMAINS[expectedHost] || [expectedHost]; - return (url.protocol === 'http:' || url.protocol === 'https:') + return (url.protocol === 'https:' || (allowHttp && url.protocol === 'http:')) && acceptedDomains.some(domain => hostname === domain || hostname.endsWith(`.${domain}`)); } catch { return false; @@ -32,27 +32,6 @@ function getUrlHost(value) { } } -function normalizeDoodstreamUrl(value) { - if (typeof value !== 'string' || value.trim() === '') return value; - try { - const url = new URL(value); - const hostname = url.hostname.toLowerCase(); - if (hostname !== 'doodstream.com' && HOSTER_RESULT_DOMAINS['doodstream.com'].includes(hostname)) { - url.hostname = 'doodstream.com'; - return url.toString(); - } - } catch {} - return value; -} - -function normalizeConfirmedResult(result, hoster) { - if (hoster !== 'doodstream.com') return result; - const downloadUrl = normalizeDoodstreamUrl(result.download_url); - const embedUrl = normalizeDoodstreamUrl(result.embed_url); - if (downloadUrl === result.download_url && embedUrl === result.embed_url) return result; - return { ...result, download_url: downloadUrl, embed_url: embedUrl }; -} - function assertUploadConfirmation(result, hoster) { const expectedHost = typeof hoster === 'string' ? hoster.trim().toLowerCase() : ''; const fileCode = typeof result?.file_code === 'string' ? result.file_code.trim() : ''; @@ -61,10 +40,18 @@ function assertUploadConfirmation(result, hoster) { && value !== undefined && !(typeof value === 'string' && value.trim() === '') )); - if (SUPPORTED_HOSTERS.has(expectedHost) - && FILE_CODE_PATTERN.test(fileCode) - && urls.every(value => isExpectedHostUrl(value, expectedHost))) { - return normalizeConfirmedResult(result, expectedHost); + if (SUPPORTED_HOSTERS.has(expectedHost) && FILE_CODE_PATTERN.test(fileCode)) { + if (expectedHost === 'doodstream.com' && urls.every(value => isExpectedHostUrl(value, expectedHost, true))) { + return { + ...result, + file_code: fileCode, + download_url: `https://doodstream.com/d/${fileCode}`, + embed_url: `https://doodstream.com/e/${fileCode}` + }; + } + if (urls.length > 0 && urls.every(value => isExpectedHostUrl(value, expectedHost))) { + return fileCode === result.file_code ? result : { ...result, file_code: fileCode }; + } } const error = new Error(`Upload zu ${hoster || 'unbekanntem Hoster'} wurde nicht bestätigt`); error.diagnostic = { diff --git a/lib/vidmoly-upload.js b/lib/vidmoly-upload.js index cd9b145..5a6db92 100644 --- a/lib/vidmoly-upload.js +++ b/lib/vidmoly-upload.js @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const { request } = require('undici'); +const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error'); const BASE_URL = 'https://vidmoly.me'; const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; @@ -130,14 +131,51 @@ class VidmolyUploader { * removed. Returns an XFS-style session token + a transit-server URL. */ async getUploadParams() { - const res = await this._fetch(`${BASE_URL}/api/upload/config`); + const endpoint = `${BASE_URL}/api/upload/config`; + let res; + try { + res = await this._fetch(endpoint); + } catch { + throw createTransportError('Vidmoly: Upload-Konfiguration konnte nicht geladen werden', { + phase: 'upload-config', + endpoint, + retryable: true, + transientNetwork: true + }); + } const body = await res.text(); + const contentType = res.headers && typeof res.headers.get === 'function' + ? res.headers.get('content-type') + : null; + if (res.status < 200 || res.status >= 300) { + throw createTransportError('Vidmoly: Upload-Konfiguration konnte nicht geladen werden', { + phase: 'upload-config', + endpoint, + httpStatus: res.status, + contentType, + body, + retryable: res.status === 429 || res.status >= 500, + transientNetwork: res.status >= 500 + }); + } let payload = null; try { payload = JSON.parse(body); } catch { - throw new Error('Vidmoly: /api/upload/config lieferte kein JSON — evtl. nicht eingeloggt?'); + throw createTransportError('Vidmoly: Upload-Konfiguration war kein JSON', { + phase: 'upload-config', + endpoint, + httpStatus: res.status, + contentType, + body + }); } if (!payload || !payload.sess_id || !payload.upload_url) { - throw new Error('Vidmoly: /api/upload/config unvollständig (sess_id/upload_url fehlt)'); + throw createTransportError('Vidmoly: Upload-Konfiguration war unvollständig', { + phase: 'upload-config', + endpoint, + httpStatus: res.status, + contentType, + body + }); } return { uploadUrl: payload.upload_url, @@ -154,7 +192,14 @@ class VidmolyUploader { async upload(filePath, onProgress, signal, throttle) { const fileName = path.basename(filePath); const fileSize = fs.statSync(filePath).size; - const baselineCodes = await this._captureVmFileCodes(); + let baselineCodes = null; + let baselineError = null; + try { + baselineCodes = await this._captureVmFileCodes(); + } catch (err) { + if (signal && signal.aborted) throw err; + baselineError = err; + } const { uploadUrl, params, fileFieldName } = await this.getUploadParams(); @@ -211,21 +256,34 @@ class VidmolyUploader { const targetUrl = uploadUrl + (uploadUrl.includes('?') ? '&' : '?') + 'X-Progress-ID=' + progressId; // Browsers don't send vidmoly.me cookies across origins, so we don't either. - const { body, statusCode, headers } = await request(targetUrl, { - method: 'POST', - body: generate(), - signal, - headers: { - 'User-Agent': USER_AGENT, - 'Accept': '*/*', - 'Origin': BASE_URL, - 'Referer': `${BASE_URL}/`, - 'Content-Type': `multipart/form-data; boundary=${boundary}`, - 'Content-Length': String(totalSize) - }, - headersTimeout: UPLOAD_TIMEOUT, - bodyTimeout: UPLOAD_TIMEOUT - }); + let uploadResponse; + try { + uploadResponse = await request(targetUrl, { + method: 'POST', + body: generate(), + signal, + headers: { + 'User-Agent': USER_AGENT, + 'Accept': '*/*', + 'Origin': BASE_URL, + 'Referer': `${BASE_URL}/`, + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize) + }, + headersTimeout: UPLOAD_TIMEOUT, + bodyTimeout: UPLOAD_TIMEOUT + }); + } catch (err) { + if (signal && signal.aborted) throw err; + throw createTransportError('Vidmoly Upload konnte nicht übertragen werden', { + phase: 'upload-request', + endpoint: targetUrl, + retryable: true, + transientNetwork: true + }); + } + + const { body, statusCode, headers } = uploadResponse; this._parseCookiesFromHeaders(headers || {}); @@ -245,6 +303,18 @@ class VidmolyUploader { resultHtml = await body.text(); } + if (statusCode >= 400) { + throw createTransportError('Vidmoly Upload fehlgeschlagen', { + phase: 'upload-response', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: resultHtml, + retryable: statusCode === 429 || statusCode >= 500, + transientNetwork: statusCode >= 500 + }); + } + // Try JSON first. The current transit server returns // { status: "OK", file_code: "...", msg: "Upload Completed" }. // Legacy XFS shapes (json.files / json.result) are kept as fallback. @@ -267,17 +337,29 @@ class VidmolyUploader { if (urls) return urls; } if (json.status && !/ok/i.test(json.status) && json.msg) { - throw new Error(`Vidmoly Upload abgelehnt: ${json.msg}`); + throw createTransportError(`Vidmoly Upload abgelehnt: ${sanitizeRemoteText(json.msg)}`, { + phase: 'upload-result', + endpoint: targetUrl, + httpStatus: statusCode, + contentType: 'application/json', + body: resultHtml + }); } } catch (err) { - if (err && /Vidmoly Upload abgelehnt/.test(err.message)) throw err; + if (err && err.diagnostic) throw err; } try { return this._parseUploadResult(resultHtml); } catch (primaryErr) { - const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal); - if (fallback) return fallback; + if (baselineCodes) { + const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal); + if (fallback) return fallback; + } + if (baselineError) { + baselineError.hosterTransient = true; + throw baselineError; + } throw primaryErr; } } @@ -286,21 +368,10 @@ class VidmolyUploader { return String(value || '') .toLowerCase() .normalize('NFKD') + .replace(/\.[a-z0-9]+$/i, '') .replace(/[^a-z0-9]+/g, ''); } - _scoreVmCandidate(file, expectedTitle) { - if (!file || !file.file_code) return -1; - if (!expectedTitle) return 0; - - const title = this._normalizeTitle(file.full_title || file.title_txt || ''); - if (!title) return -1; - if (title === expectedTitle) return 120; - if (title.startsWith(expectedTitle) || expectedTitle.startsWith(title)) return 90; - if (title.includes(expectedTitle) || expectedTitle.includes(title)) return 70; - return 0; - } - _buildUrlsFromCode(fileCode) { const code = String(fileCode || '').trim(); if (!code) return null; @@ -313,19 +384,15 @@ class VidmolyUploader { } async _captureVmFileCodes() { - try { - const files = await this._fetchVmList(); - return new Set( - files - .map((f) => String(f.file_code || '').trim()) - .filter(Boolean) - ); - } catch { - return new Set(); - } + const files = await this._fetchVmList('recovery-baseline'); + return new Set( + files + .map((f) => String(f.file_code || '').trim()) + .filter(Boolean) + ); } - async _fetchVmList() { + async _fetchVmList(phase = 'recovery-poll') { const params = new URLSearchParams({ op: 'vm', api: 'list', @@ -336,14 +403,46 @@ class VidmolyUploader { fld_id: '0' }); - const res = await this._fetch(`${BASE_URL}/?${params.toString()}`); + const endpoint = `${BASE_URL}/?${params.toString()}`; + let res; + try { + res = await this._fetch(endpoint); + } catch { + throw createTransportError('Vidmoly: Dateiliste konnte nicht geladen werden', { + phase, + endpoint, + retryable: true, + transientNetwork: true + }); + } const body = await res.text(); + const contentType = res.headers && typeof res.headers.get === 'function' + ? res.headers.get('content-type') + : null; + + if (res.status < 200 || res.status >= 300) { + throw createTransportError('Vidmoly: Dateiliste konnte nicht geladen werden', { + phase, + endpoint, + httpStatus: res.status, + contentType, + body, + retryable: res.status === 429 || res.status >= 500, + transientNetwork: res.status >= 500 + }); + } let payload; try { payload = JSON.parse(body); } catch { - throw new Error('Vidmoly VM API lieferte kein JSON'); + throw createTransportError('Vidmoly: Dateiliste war kein JSON', { + phase, + endpoint, + httpStatus: res.status, + contentType, + body + }); } if (!payload || !Array.isArray(payload.files)) return []; @@ -351,7 +450,10 @@ class VidmolyUploader { } async _resolveUploadedFileFromVmApi(fileName, baselineCodes, signal) { + if (!(baselineCodes instanceof Set)) return null; const expectedTitle = this._normalizeTitle(path.parse(fileName).name); + let lastPollError = null; + let successfulPoll = false; for (let attempt = 0; attempt < RESULT_POLL_ATTEMPTS; attempt++) { if (signal && signal.aborted) { @@ -362,46 +464,23 @@ class VidmolyUploader { let files = []; try { - files = await this._fetchVmList(); - } catch { - files = []; + files = await this._fetchVmList('recovery-poll'); + successfulPoll = true; + } catch (err) { + if (err && err.name === 'AbortError') throw err; + lastPollError = err; } const withCode = files.filter((f) => f && typeof f.file_code === 'string' && f.file_code.trim()); - const newFiles = withCode.filter((f) => !baselineCodes.has(f.file_code)); + const newFiles = withCode.filter((f) => !baselineCodes.has(f.file_code.trim())); + const matches = newFiles.filter((file) => { + const title = this._normalizeTitle(file.full_title || file.title_txt || ''); + return expectedTitle && title === expectedTitle; + }); - if (newFiles.length > 0) { - let best = null; - let bestScore = -1; - - for (const file of newFiles) { - const score = this._scoreVmCandidate(file, expectedTitle); - if (score > bestScore) { - bestScore = score; - best = file; - } - } - - if (best && bestScore > 0) { - return this._buildUrlsFromCode(best.file_code); - } - } - - if (expectedTitle) { - let bestMatch = null; - let bestScore = -1; - - for (const file of withCode) { - const score = this._scoreVmCandidate(file, expectedTitle); - if (score > bestScore) { - bestScore = score; - bestMatch = file; - } - } - - if (bestMatch && bestScore >= 90) { - return this._buildUrlsFromCode(bestMatch.file_code); - } + if (matches.length > 1) return null; + if (matches.length === 1) { + return this._buildUrlsFromCode(matches[0].file_code); } if (attempt < RESULT_POLL_ATTEMPTS - 1) { @@ -409,6 +488,7 @@ class VidmolyUploader { } } + if (!successfulPoll && lastPollError) throw lastPollError; return null; } @@ -508,7 +588,14 @@ class VidmolyUploader { if (!download_url && !file_code) { const errMatch = html.match(/class=["']err["'][^>]*>([^<]+)/i); const errMsg = errMatch ? errMatch[1].trim() : 'Kein Download-Link gefunden'; - throw new Error(`Vidmoly Upload-Ergebnis: ${errMsg}`); + throw createTransportError(`Vidmoly Upload-Ergebnis: ${sanitizeRemoteText(errMsg)}`, { + phase: 'upload-result', + endpoint: BASE_URL, + contentType: 'text/html', + body: html, + hosterTransient: true, + retryable: true + }); } return { download_url, embed_url, file_code }; diff --git a/lib/voe-upload.js b/lib/voe-upload.js index 4c105c3..f03f5c0 100644 --- a/lib/voe-upload.js +++ b/lib/voe-upload.js @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const { request } = require('undici'); +const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error'); const BASE_URL = 'https://voe.sx'; const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; @@ -160,21 +161,58 @@ class VoeUploader { * Returns { server: "https://cdn-xxx.edgeon-bandwidth.com/node/u/01", session_id: "..." } */ async _getDeliveryNode(csrfToken) { - const res = await this._fetch(`${BASE_URL}/engine/delivery-node`, { - headers: { - 'X-CSRF-TOKEN': csrfToken, - 'X-Requested-With': 'XMLHttpRequest', - 'Accept': 'application/json' - } - }); + const endpoint = `${BASE_URL}/engine/delivery-node`; + let res; + try { + res = await this._fetch(endpoint, { + headers: { + 'X-CSRF-TOKEN': csrfToken, + 'X-Requested-With': 'XMLHttpRequest', + 'Accept': 'application/json' + } + }); + } catch { + throw createTransportError('VOE: Upload-Server konnte nicht geladen werden', { + phase: 'upload-server', + endpoint, + retryable: true, + transientNetwork: true + }); + } const body = await res.text(); + const contentType = res.headers && typeof res.headers.get === 'function' + ? res.headers.get('content-type') + : null; + if (res.status < 200 || res.status >= 300) { + throw createTransportError('VOE: Upload-Server konnte nicht geladen werden', { + phase: 'upload-server', + endpoint, + httpStatus: res.status, + contentType, + body, + retryable: res.status === 429 || res.status >= 500, + transientNetwork: res.status >= 500 + }); + } let data; try { data = JSON.parse(body); } catch { - throw new Error(`VOE: Upload-Server Antwort war kein JSON: ${body.slice(0, 200)}`); + throw createTransportError('VOE: Upload-Server Antwort war kein JSON', { + phase: 'upload-server', + endpoint, + httpStatus: res.status, + contentType, + body + }); } if (!data || !data.success || !data.server) { - throw new Error('VOE: Kein Upload-Server erhalten von delivery-node'); + throw createTransportError('VOE: Kein Upload-Server erhalten von delivery-node', { + phase: 'upload-server', + endpoint, + httpStatus: res.status, + contentType, + body + }); } return { uploadServer: data.server, sessionId: data.session_id || '' }; @@ -183,26 +221,54 @@ class VoeUploader { /** * List current files via VOE API (for result polling fallback) */ - async _fetchFileList() { + async _fetchFileList(phase = 'recovery-poll') { + const endpoint = `${BASE_URL}/api2/my-files?sort=date&order=dsc&page=1&per_page=50`; + let res; try { - const res = await this._fetch(`${BASE_URL}/api2/my-files?sort=date&order=dsc&page=1&per_page=50`); - const body = await res.text(); - const data = JSON.parse(body); - if (data && Array.isArray(data.data)) return data.data; - if (data && Array.isArray(data.files)) return data.files; - return []; + res = await this._fetch(endpoint); } catch { - return []; + throw createTransportError('VOE: Dateiliste konnte nicht geladen werden', { + phase, + endpoint, + retryable: true, + transientNetwork: true + }); } + const body = await res.text(); + const contentType = res.headers && typeof res.headers.get === 'function' + ? res.headers.get('content-type') + : null; + if (res.status < 200 || res.status >= 300) { + throw createTransportError('VOE: Dateiliste konnte nicht geladen werden', { + phase, + endpoint, + httpStatus: res.status, + contentType, + body, + retryable: res.status === 429 || res.status >= 500, + transientNetwork: res.status >= 500 + }); + } + let data; + try { + data = JSON.parse(body); + } catch { + throw createTransportError('VOE: Dateiliste war kein JSON', { + phase, + endpoint, + httpStatus: res.status, + contentType, + body + }); + } + if (data && Array.isArray(data.data)) return data.data; + if (data && Array.isArray(data.files)) return data.files; + return []; } async _captureFileCodes() { - try { - const files = await this._fetchFileList(); - return new Set(files.map(f => String(f.file_code || f.slug || '').trim()).filter(Boolean)); - } catch { - return new Set(); - } + const files = await this._fetchFileList('recovery-baseline'); + return new Set(files.map(f => String(f.file_code || f.slug || '').trim()).filter(Boolean)); } /** @@ -212,7 +278,14 @@ class VoeUploader { async upload(filePath, onProgress, signal, throttle) { const fileName = path.basename(filePath); const fileSize = fs.statSync(filePath).size; - const baselineCodes = await this._captureFileCodes(); + let baselineCodes = null; + let baselineError = null; + try { + baselineCodes = await this._captureFileCodes(); + } catch (err) { + if (signal && signal.aborted) throw err; + baselineError = err; + } // Step 1: Get CSRF token from upload page const { csrfToken } = await this._getUploadParams(); @@ -258,27 +331,51 @@ class VoeUploader { } // Step 3: POST file to CDN upload server - const { body, headers } = await request(uploadServer, { - method: 'POST', - body: generate(), - signal, - headers: { - 'User-Agent': USER_AGENT, - 'Cookie': this._cookieHeader(), - 'Content-Type': `multipart/form-data; boundary=${boundary}`, - 'Content-Length': String(totalSize), - 'X-CSRF-TOKEN': csrfToken, - 'X-Requested-With': 'XMLHttpRequest', - 'Referer': `${BASE_URL}/file-upload`, - 'Origin': BASE_URL - }, - headersTimeout: UPLOAD_TIMEOUT, - bodyTimeout: UPLOAD_TIMEOUT - }); + let uploadResponse; + try { + uploadResponse = await request(uploadServer, { + method: 'POST', + body: generate(), + signal, + headers: { + 'User-Agent': USER_AGENT, + 'Cookie': this._cookieHeader(), + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize), + 'X-CSRF-TOKEN': csrfToken, + 'X-Requested-With': 'XMLHttpRequest', + 'Referer': `${BASE_URL}/file-upload`, + 'Origin': BASE_URL + }, + headersTimeout: UPLOAD_TIMEOUT, + bodyTimeout: UPLOAD_TIMEOUT + }); + } catch (err) { + if (signal && signal.aborted) throw err; + throw createTransportError('VOE Upload konnte nicht übertragen werden', { + phase: 'upload-request', + endpoint: uploadServer, + retryable: true, + transientNetwork: true + }); + } + + const { body, headers, statusCode } = uploadResponse; this._parseCookiesFromHeaders(headers || {}); const rawBody = await body.text(); + if (statusCode < 200 || statusCode >= 300) { + throw createTransportError('VOE Upload fehlgeschlagen', { + phase: 'upload-response', + endpoint: uploadServer, + httpStatus: statusCode, + contentType: headers && headers['content-type'], + body: rawBody, + retryable: statusCode === 429 || statusCode >= 500, + transientNetwork: statusCode >= 500 + }); + } // Try JSON response try { @@ -295,22 +392,44 @@ class VoeUploader { // Check for error if (json.error || json.message) { - throw new Error(`VOE Upload-Fehler: ${json.error || json.message}`); + throw createTransportError(`VOE Upload-Fehler: ${sanitizeRemoteText(json.error || json.message)}`, { + phase: 'upload-result', + endpoint: uploadServer, + contentType: 'application/json', + body: rawBody + }); } } catch (parseErr) { - if (parseErr.message.startsWith('VOE Upload-Fehler')) throw parseErr; + if (parseErr && parseErr.diagnostic) throw parseErr; // Not JSON - might be a redirect or HTML response } // Fallback: poll the file list to find the newly uploaded file - const result = await this._resolveUploadedFile(fileName, baselineCodes, signal); - if (result) return result; + if (baselineCodes) { + const result = await this._resolveUploadedFile(fileName, baselineCodes, signal); + if (result) return result; + } - throw new Error('VOE Upload: Kein file_code in der Antwort gefunden'); + if (baselineError) { + baselineError.hosterTransient = true; + throw baselineError; + } + + throw createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', { + phase: 'upload-result', + endpoint: uploadServer, + contentType: headers && headers['content-type'], + body: rawBody, + hosterTransient: true, + retryable: true + }); } async _resolveUploadedFile(fileName, baselineCodes, signal) { + if (!(baselineCodes instanceof Set)) return null; const expectedTitle = this._normalizeTitle(path.parse(fileName).name); + let lastPollError = null; + let successfulPoll = false; for (let attempt = 0; attempt < RESULT_POLL_ATTEMPTS; attempt++) { if (signal && signal.aborted) { @@ -321,29 +440,24 @@ class VoeUploader { let files = []; try { - files = await this._fetchFileList(); - } catch { files = []; } + files = await this._fetchFileList('recovery-poll'); + successfulPoll = true; + } catch (err) { + if (err && err.name === 'AbortError') throw err; + lastPollError = err; + } const withCode = files.filter(f => f && (f.file_code || f.slug)); const newFiles = withCode.filter(f => !baselineCodes.has(String(f.file_code || f.slug || '').trim())); + const matches = newFiles.filter(file => { + const title = this._normalizeTitle(file.title || file.name || ''); + return expectedTitle && title === expectedTitle; + }); - if (newFiles.length > 0) { - // Try to match by title - let best = null; - let bestScore = -1; - - for (const file of newFiles) { - const score = this._scoreCandidate(file, expectedTitle); - if (score > bestScore) { - bestScore = score; - best = file; - } - } - - if (best && (bestScore > 0 || newFiles.length === 1)) { - const code = best.file_code || best.slug; - return this._buildUrls(code); - } + if (matches.length > 1) return null; + if (matches.length === 1) { + const code = matches[0].file_code || matches[0].slug; + return this._buildUrls(code); } if (attempt < RESULT_POLL_ATTEMPTS - 1) { @@ -351,6 +465,7 @@ class VoeUploader { } } + if (!successfulPoll && lastPollError) throw lastPollError; return null; } @@ -358,21 +473,10 @@ class VoeUploader { return String(value || '') .toLowerCase() .normalize('NFKD') + .replace(/\.[a-z0-9]+$/i, '') .replace(/[^a-z0-9]+/g, ''); } - _scoreCandidate(file, expectedTitle) { - if (!file || !(file.file_code || file.slug)) return -1; - if (!expectedTitle) return 0; - - const title = this._normalizeTitle(file.title || file.name || ''); - if (!title) return -1; - if (title === expectedTitle) return 120; - if (title.startsWith(expectedTitle) || expectedTitle.startsWith(title)) return 90; - if (title.includes(expectedTitle) || expectedTitle.includes(title)) return 70; - return 0; - } - _buildUrls(fileCode) { const code = String(fileCode || '').trim(); if (!code) return null; diff --git a/tests/byse-reject-recovery.test.js b/tests/byse-reject-recovery.test.js index 9740478..8907f62 100644 --- a/tests/byse-reject-recovery.test.js +++ b/tests/byse-reject-recovery.test.js @@ -171,6 +171,94 @@ test('byse empty filecode WITHOUT explicit rejection still polls recovery', asyn assert.ok(listCalls >= 2, 'recovery polling must run when there is no explicit rejection'); }); +test('byse never recovers an old file after a failed baseline', async () => { + stubByseUploadServer(); + const abort = new AbortController(); + const fileName = path.basename(tmpFile); + let listCalls = 0; + requestRouter = async (url, opts) => { + if (/\/file\/list/.test(String(url))) { + listCalls++; + if (listCalls === 1) { + return { + statusCode: 503, + headers: { 'content-type': 'text/html' }, + body: { text: async () => 'baseline-token=SYNTHETIC_BYSE_BASELINE' } + }; + } + abort.abort(); + return { + statusCode: 200, + headers: { 'content-type': 'application/json' }, + body: { text: async () => JSON.stringify({ status: 200, result: { files: [{ file_code: 'OLD_BYSE_123', title: fileName }] } }) } + }; + } + if (opts && opts.body && typeof opts.body[Symbol.asyncIterator] === 'function') { + for await (const chunk of opts.body) { if (chunk && chunk.length === -1) break; } + } + return { + statusCode: 200, + headers: { 'content-type': 'application/json' }, + body: { text: async () => JSON.stringify({ status: 200, msg: 'OK' }) } + }; + }; + + await assert.rejects( + () => uploadFile('byse.sx', tmpFile, 'VALIDKEY', null, abort.signal, null), + (err) => { + assert.doesNotMatch(err.message, /SYNTHETIC_BYSE_BASELINE/); + assert.equal(err.diagnostic.phase, 'recovery-baseline'); + assert.equal(err.diagnostic.http, 503); + return true; + } + ); + assert.equal(listCalls, 1); +}); + +test('byse recovery rejects ambiguous same-title candidates', async () => { + stubByseUploadServer(); + const abort = new AbortController(); + const fileName = path.basename(tmpFile); + let listCalls = 0; + requestRouter = async (url, opts) => { + if (/\/file\/list/.test(String(url))) { + listCalls++; + if (listCalls === 1) { + return { statusCode: 200, headers: {}, body: { text: async () => '{"status":200,"result":{"files":[]}}' } }; + } + abort.abort(); + return { + statusCode: 200, + headers: {}, + body: { + text: async () => JSON.stringify({ + status: 200, + result: { + files: [ + { file_code: 'PARALLEL_A', title: fileName }, + { file_code: 'PARALLEL_B', title: fileName } + ] + } + }) + } + }; + } + if (opts && opts.body && typeof opts.body[Symbol.asyncIterator] === 'function') { + for await (const chunk of opts.body) { if (chunk && chunk.length === -1) break; } + } + return { + statusCode: 200, + headers: { 'content-type': 'application/json' }, + body: { text: async () => JSON.stringify({ status: 200, msg: 'OK' }) } + }; + }; + + await assert.rejects( + () => uploadFile('byse.sx', tmpFile, 'VALIDKEY', null, abort.signal, null), + (err) => err.hosterTransient === true + ); +}); + function stubBysePost(response) { requestRouter = async (url, opts) => { const u = String(url); diff --git a/tests/doodstream-api-upload.test.js b/tests/doodstream-api-upload.test.js index 1913a53..de1911d 100644 --- a/tests/doodstream-api-upload.test.js +++ b/tests/doodstream-api-upload.test.js @@ -59,7 +59,7 @@ function routeWith(uploadBody, listBodies = []) { if (opts && opts.body && typeof opts.body[Symbol.asyncIterator] === 'function') { for await (const chunk of opts.body) { if (chunk && chunk.length === -1) break; } } - return { statusCode: uploadBody.status, headers: { 'content-type': 'application/json' }, body: { text: async () => uploadBody.body } }; + return { statusCode: uploadBody.status, headers: { 'content-type': uploadBody.contentType || 'application/json' }, body: { text: async () => uploadBody.body } }; }; } @@ -103,3 +103,95 @@ test('doodstream API upload: codeless + file never appears → throws hosterTran } ); }); + +test('doodstream API upload never recovers an old file after a failed baseline', async () => { + stubUploadServer(); + const abort = new AbortController(); + const fileName = path.basename(tmpFile); + let listCalls = 0; + requestRouter = async (url, opts) => { + if (/\/api\/file\/list/.test(String(url))) { + listCalls++; + if (listCalls === 1) { + return { + statusCode: 503, + headers: { 'content-type': 'text/html' }, + body: { text: async () => 'baseline-token=SYNTHETIC_BASELINE_SECRET' } + }; + } + abort.abort(); + return { + statusCode: 200, + headers: { 'content-type': 'application/json' }, + body: { text: async () => JSON.stringify({ status: 200, result: { files: [{ file_code: 'OLD_DOOD_123', title: fileName }] } }) } + }; + } + if (opts && opts.body && typeof opts.body[Symbol.asyncIterator] === 'function') { + for await (const chunk of opts.body) { if (chunk && chunk.length === -1) break; } + } + return { + statusCode: 200, + headers: { 'content-type': 'application/json' }, + body: { text: async () => JSON.stringify({ status: 200, msg: 'OK' }) } + }; + }; + + await assert.rejects( + () => uploadFile('doodstream.com', tmpFile, 'VALIDKEY', null, abort.signal, null), + (err) => { + assert.doesNotMatch(err.message, /SYNTHETIC_BASELINE_SECRET/); + assert.equal(err.diagnostic.phase, 'recovery-baseline'); + assert.equal(err.diagnostic.http, 503); + return true; + } + ); + assert.equal(listCalls, 1); +}); + +test('doodstream API recovery rejects ambiguous same-title candidates', async () => { + stubUploadServer(); + const fileName = path.basename(tmpFile); + requestRouter = routeWith( + { status: 200, body: JSON.stringify({ status: 200, msg: 'OK' }) }, + [ + '{"status":200,"result":{"files":[]}}', + JSON.stringify({ + status: 200, + result: { + files: [ + { file_code: 'PARALLEL_A', title: fileName }, + { file_code: 'PARALLEL_B', title: fileName } + ] + } + }) + ] + ); + + await assert.rejects( + () => uploadFile('doodstream.com', tmpFile, 'VALIDKEY', null, null, null), + (err) => err.hosterTransient === true + ); +}); + +test('doodstream API upload errors expose safe structured diagnostics', async () => { + stubUploadServer(); + requestRouter = routeWith({ + status: 502, + contentType: 'text/html; charset=utf-8', + body: 'upstream-token=SYNTHETIC_UPLOAD_SECRET https://node.invalid/upload?session=SYNTHETIC_SESSION' + }); + + await assert.rejects( + () => uploadFile('doodstream.com', tmpFile, 'VALIDKEY', null, null, null), + (err) => { + assert.equal(err.transientNetwork, true); + assert.doesNotMatch(err.message, /SYNTHETIC_UPLOAD_SECRET|SYNTHETIC_SESSION|/); + assert.equal(err.diagnostic.phase, 'upload-response'); + assert.equal(err.diagnostic.http, 502); + assert.equal(err.diagnostic.contentType, 'text/html; charset=utf-8'); + assert.equal(err.diagnostic.responseKind, 'html'); + assert.doesNotMatch(err.diagnostic.payloadSnippet, /SYNTHETIC_UPLOAD_SECRET|SYNTHETIC_SESSION/); + return true; + } + ); +}); diff --git a/tests/doodstream-upload.test.js b/tests/doodstream-upload.test.js index f7cfcb5..492cc0b 100644 --- a/tests/doodstream-upload.test.js +++ b/tests/doodstream-upload.test.js @@ -64,6 +64,39 @@ test('happy path: link in result page wins', async () => { assert.equal(res.file_code, 'jjsuhr931ds9'); }); +test('JSON results rebuild canonical Doodstream URLs from the file code', () => { + const up = new DoodstreamUploader(); + assert.deepEqual( + up._extractFromJson({ + status: 200, + result: { + filecode: 'CANONICAL123', + download_url: 'http://edge.dsvplay.com/result/CANONICAL123?token=SYNTHETIC_SECRET', + protected_embed: 'https://dood.to/arbitrary/CANONICAL123' + } + }), + { + file_code: 'CANONICAL123', + download_url: 'https://doodstream.com/d/CANONICAL123', + embed_url: 'https://doodstream.com/e/CANONICAL123' + } + ); +}); + +test('invalid web upload results expose safe structured diagnostics', async () => { + const up = new DoodstreamUploader(); + await assert.rejects( + () => up._parseUploadResponse(' https://doodstream.com/?session=SYNTHETIC_WEB_SESSION'), + (err) => { + assert.doesNotMatch(err.message, /SYNTHETIC_WEB_SECRET|SYNTHETIC_WEB_SESSION|/); + assert.equal(err.diagnostic.phase, 'upload-result'); + assert.equal(err.diagnostic.responseKind, 'html'); + assert.doesNotMatch(err.diagnostic.payloadSnippet, /SYNTHETIC_WEB_SECRET|SYNTHETIC_WEB_SESSION/); + return true; + } + ); +}); + // --- _parseUploadFormFields: replicate the current upload form faithfully --- test('_parseUploadFormFields extracts the real form fields and excludes the file input', () => { const up = new DoodstreamUploader(); @@ -212,3 +245,26 @@ test('getUploadServer: throws (no silent dead fallback) when discovery fails', a } ); }); + +test('getUploadServer: failures expose safe structured diagnostics without response secrets', async () => { + const up = new DoodstreamUploader(); + up._fetch = async (url) => { + if (/op=upload_server/.test(url)) { + return fakeRes('upstream-token=SYNTHETIC_DISCOVERY_SECRET', { status: 503, ctype: 'text/html; charset=utf-8' }); + } + return fakeRes('x'); + }; + + await assert.rejects( + () => up._getUploadServer(), + (err) => { + assert.doesNotMatch(err.message, /SYNTHETIC_DISCOVERY_SECRET|SYNTHETIC_DISCOVERY_SESSION|SYNTHETIC_QUERY|/); + assert.equal(err.diagnostic.phase, 'upload-server'); + assert.equal(err.diagnostic.http, 503); + assert.equal(err.diagnostic.contentType, 'text/html; charset=utf-8'); + assert.equal(err.diagnostic.safeEndpointHost, 'doodstream.com'); + assert.equal(err.diagnostic.responseKind, 'html'); + return true; + } + ); +}); diff --git a/tests/hoster-recovery-provenance.test.js b/tests/hoster-recovery-provenance.test.js new file mode 100644 index 0000000..7830d88 --- /dev/null +++ b/tests/hoster-recovery-provenance.test.js @@ -0,0 +1,118 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const VoeUploader = require('../lib/voe-upload'); +const VidmolyUploader = require('../lib/vidmoly-upload'); + +function response(body, status = 200, contentType = 'application/json') { + return { + status, + headers: { get: (name) => name.toLowerCase() === 'content-type' ? contentType : null }, + text: async () => body + }; +} + +test('VOE recovery rejects an unrelated singleton candidate', async () => { + const uploader = new VoeUploader(); + uploader._fetchFileList = async () => [{ file_code: 'OTHER999', title: 'foreign-upload' }]; + uploader._sleep = async () => {}; + + assert.equal(await uploader._resolveUploadedFile('wanted-video.mkv', new Set(), null), null); +}); + +test('VOE recovery rejects ambiguous exact-title candidates', async () => { + const uploader = new VoeUploader(); + uploader._fetchFileList = async () => [ + { file_code: 'VOE_FIRST', title: 'wanted-video' }, + { file_code: 'VOE_SECOND', title: 'wanted-video' } + ]; + uploader._sleep = async () => {}; + + assert.equal(await uploader._resolveUploadedFile('wanted-video.mkv', new Set(), null), null); +}); + +test('VOE recovery accepts one new exact-title candidate with a file extension', async () => { + const uploader = new VoeUploader(); + uploader._fetchFileList = async () => [{ file_code: 'VOE_EXACT', title: 'wanted-video.mkv' }]; + uploader._sleep = async () => {}; + + assert.deepEqual(await uploader._resolveUploadedFile('wanted-video.mkv', new Set(), null), { + file_code: 'VOE_EXACT', + download_url: 'https://voe.sx/VOE_EXACT', + embed_url: 'https://voe.sx/e/VOE_EXACT' + }); +}); + +test('VOE preserves a failed recovery baseline as a safe structured error', async () => { + const uploader = new VoeUploader(); + uploader._fetch = async () => response( + 'api_key=SYNTHETIC_VOE_SECRET https://voe.sx/list?session=SYNTHETIC_VOE_SESSION', + 503, + 'text/html' + ); + + await assert.rejects( + () => uploader._captureFileCodes(), + (err) => { + assert.doesNotMatch(err.message, /SYNTHETIC_VOE_SECRET|SYNTHETIC_VOE_SESSION|/); + assert.equal(err.diagnostic.phase, 'recovery-baseline'); + assert.equal(err.diagnostic.http, 503); + assert.equal(err.diagnostic.responseKind, 'html'); + return true; + } + ); +}); + +test('Vidmoly recovery rejects a matching code already present in the baseline', async () => { + const uploader = new VidmolyUploader(); + uploader._fetchVmList = async () => [{ file_code: ' OLDVID123456 ', full_title: 'wanted-video' }]; + uploader._sleep = async () => {}; + + assert.equal( + await uploader._resolveUploadedFileFromVmApi('wanted-video.mkv', new Set(['OLDVID123456']), null), + null + ); +}); + +test('Vidmoly recovery rejects ambiguous exact-title candidates', async () => { + const uploader = new VidmolyUploader(); + uploader._fetchVmList = async () => [ + { file_code: 'NEWVID123456', full_title: 'wanted-video' }, + { file_code: 'NEWVID654321', full_title: 'wanted-video' } + ]; + uploader._sleep = async () => {}; + + assert.equal(await uploader._resolveUploadedFileFromVmApi('wanted-video.mkv', new Set(), null), null); +}); + +test('Vidmoly recovery accepts one new exact-title candidate with a file extension', async () => { + const uploader = new VidmolyUploader(); + uploader._fetchVmList = async () => [{ file_code: 'NEWVID123456', full_title: 'wanted-video.mkv' }]; + uploader._sleep = async () => {}; + + assert.deepEqual(await uploader._resolveUploadedFileFromVmApi('wanted-video.mkv', new Set(), null), { + file_code: 'NEWVID123456', + download_url: 'https://vidmoly.me/w/NEWVID123456', + embed_url: 'https://vidmoly.me/embed-NEWVID123456.html' + }); +}); + +test('Vidmoly preserves a failed recovery baseline as a safe structured error', async () => { + const uploader = new VidmolyUploader(); + uploader._fetch = async () => response( + 'sess_id=SYNTHETIC_VIDMOLY_SECRET https://vidmoly.me/?token=SYNTHETIC_VIDMOLY_SESSION', + 503, + 'text/html' + ); + + await assert.rejects( + () => uploader._captureVmFileCodes(), + (err) => { + assert.doesNotMatch(err.message, /SYNTHETIC_VIDMOLY_SECRET|SYNTHETIC_VIDMOLY_SESSION|/); + assert.equal(err.diagnostic.phase, 'recovery-baseline'); + assert.equal(err.diagnostic.http, 503); + assert.equal(err.diagnostic.responseKind, 'html'); + return true; + } + ); +}); diff --git a/tests/hosters.test.js b/tests/hosters.test.js index 9f41eef..4a2cd34 100644 --- a/tests/hosters.test.js +++ b/tests/hosters.test.js @@ -46,12 +46,13 @@ describe('hosters helpers', () => { it('parseDoodstreamResult handles result-as-array and result-as-object', () => { const arr = __test.parseDoodstreamResult({ result: [{ filecode: 'AB1', protected_dl: 'https://x/1', protected_embed: 'https://x/e/1' }] }); assert.equal(arr.file_code, 'AB1'); - assert.equal(arr.download_url, 'https://x/1'); - assert.equal(arr.embed_url, 'https://x/e/1'); + assert.equal(arr.download_url, 'https://doodstream.com/d/AB1'); + assert.equal(arr.embed_url, 'https://doodstream.com/e/AB1'); const obj = __test.parseDoodstreamResult({ result: { filecode: 'OBJ1', download_url: 'https://x/2' } }); assert.equal(obj.file_code, 'OBJ1'); - assert.equal(obj.download_url, 'https://x/2'); + assert.equal(obj.download_url, 'https://doodstream.com/d/OBJ1'); + assert.equal(obj.embed_url, 'https://doodstream.com/e/OBJ1'); }); it('parseByseResult tolerates null/non-object payload without throwing', () => { diff --git a/tests/upload-confirmation.test.js b/tests/upload-confirmation.test.js index 3baced2..9bcbeba 100644 --- a/tests/upload-confirmation.test.js +++ b/tests/upload-confirmation.test.js @@ -3,9 +3,15 @@ const assert = require('node:assert/strict'); const { assertUploadConfirmation } = require('../lib/upload-confirmation'); -test('accepts a host-confirmed file code without a public URL', () => { - const result = { file_code: 'AB1', download_url: null, embed_url: null }; - assert.equal(assertUploadConfirmation(result, 'doodstream.com'), result); +test('materializes canonical Doodstream URLs from a confirmed file code', () => { + assert.deepEqual( + assertUploadConfirmation({ file_code: 'AB1', download_url: null, embed_url: null }, 'doodstream.com'), + { + file_code: 'AB1', + download_url: 'https://doodstream.com/d/AB1', + embed_url: 'https://doodstream.com/e/AB1' + } + ); }); test('accepts upload URLs for every supported hoster and its subdomains', () => { @@ -18,7 +24,15 @@ test('accepts upload URLs for every supported hoster and its subdomains', () => ]; for (const [hoster, downloadUrl] of cases) { const result = { file_code: 'abc123', download_url: downloadUrl }; - assert.equal(assertUploadConfirmation(result, hoster), result); + const confirmed = assertUploadConfirmation(result, hoster); + if (hoster === 'doodstream.com') { + assert.deepEqual(confirmed, { + ...result, + embed_url: 'https://doodstream.com/e/abc123' + }); + } else { + assert.equal(confirmed, result); + } } }); @@ -48,6 +62,40 @@ test('accepts the Doodstream result domain returned by the current upload servic }); }); +test('rebuilds every accepted Doodstream transport URL from the file code', () => { + const variants = [ + 'http://dsvplay.com/d/DOODCODE1234?token=SYNTHETIC_SECRET#fragment', + 'https://edge.dsvplay.com/result/DOODCODE1234?session=SYNTHETIC_SESSION', + 'https://dood.to/e/DOODCODE1234', + 'https://dood.la/arbitrary/DOODCODE1234' + ]; + + for (const downloadUrl of variants) { + assert.deepEqual( + assertUploadConfirmation({ file_code: 'DOODCODE1234', download_url: downloadUrl }, 'doodstream.com'), + { + file_code: 'DOODCODE1234', + download_url: 'https://doodstream.com/d/DOODCODE1234', + embed_url: 'https://doodstream.com/e/DOODCODE1234' + } + ); + } +}); + +test('rejects code-only confirmations for hosters without canonical materialization', () => { + assert.throws( + () => assertUploadConfirmation({ file_code: 'BYSE123' }, 'byse.sx'), + /Upload zu byse\.sx wurde nicht bestätigt/ + ); +}); + +test('rejects non-HTTPS public URLs outside Doodstream transport normalization', () => { + assert.throws( + () => assertUploadConfirmation({ file_code: 'VOE123', download_url: 'http://voe.sx/VOE123' }, 'voe.sx'), + /Upload zu voe\.sx wurde nicht bestätigt/ + ); +}); + test('rejects an upload URL from a different domain', () => { assert.throws( () => assertUploadConfirmation({ file_code: 'abc123', download_url: 'https://attacker.invalid/file/abc123' }, 'voe.sx'),