Restore the v2.1.19 application baseline and retain only the focused import preflight summary with duplicate, unavailable, destination, job, and size-limit visibility.
This commit is contained in:
+133
-516
@@ -2,7 +2,6 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { request } = require('undici');
|
||||
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
||||
|
||||
const UPLOAD_TIMEOUT = 1800000; // 30 minutes
|
||||
const API_TIMEOUT = 45000; // 45 seconds
|
||||
@@ -173,11 +172,10 @@ function parseDoodstreamResult(payload) {
|
||||
item = result;
|
||||
}
|
||||
|
||||
const fileCode = item.filecode || item.file_code || null;
|
||||
return {
|
||||
download_url: fileCode ? `https://doodstream.com/d/${fileCode}` : null,
|
||||
embed_url: fileCode ? `https://doodstream.com/e/${fileCode}` : null,
|
||||
file_code: fileCode
|
||||
download_url: item.download_url || item.protected_dl || null,
|
||||
embed_url: item.protected_embed || null,
|
||||
file_code: item.filecode || item.file_code || null
|
||||
};
|
||||
}
|
||||
|
||||
@@ -236,7 +234,7 @@ function parseByseResult(payload) {
|
||||
// wall, so we must rotate. File-specific rejections (Duplicate, wrong
|
||||
// format, too small/large) ARE per-file and rotation is pointless.
|
||||
const accountLevel = /(not enough (disk )?(space|storage)|insufficient (disk )?space|disk (space )?full|storage (exhausted|full|voll|limit)|quota (exceeded|voll|überschritten)|account (full|voll|suspended|banned))/i.test(perFileError);
|
||||
const err = new Error(`Byse lehnte Datei ab: ${sanitizeRemoteText(perFileError)}`);
|
||||
const err = new Error(`Byse lehnte Datei ab: ${perFileError}`);
|
||||
if (accountLevel) {
|
||||
err.accountError = true;
|
||||
} else {
|
||||
@@ -310,64 +308,32 @@ function createUploadBody(filePath, formFields, onProgress, throttle, signal) {
|
||||
|
||||
// --- API helper using built-in fetch (follows redirects automatically) ---
|
||||
|
||||
async function apiGet(url, signal, hosterName) {
|
||||
async function apiGet(url, signal) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), API_TIMEOUT);
|
||||
const onAbort = () => controller.abort();
|
||||
if (signal) signal.addEventListener('abort', onAbort);
|
||||
|
||||
try {
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: 'GET',
|
||||
signal: controller.signal,
|
||||
redirect: 'follow'
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
throw createTransportError(`${hosterName}: Upload-Server-Abfrage fehlgeschlagen`, {
|
||||
phase: 'upload-server',
|
||||
endpoint: url,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
}
|
||||
const res = await fetch(url, {
|
||||
method: 'GET',
|
||||
signal: controller.signal,
|
||||
redirect: 'follow'
|
||||
});
|
||||
const text = await res.text();
|
||||
const contentType = res.headers && typeof res.headers.get === 'function'
|
||||
? res.headers.get('content-type')
|
||||
: null;
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(text);
|
||||
} catch {
|
||||
throw createTransportError(`${hosterName}: Upload-Server-Antwort war kein JSON`, {
|
||||
phase: 'upload-server',
|
||||
endpoint: url,
|
||||
httpStatus: res.status,
|
||||
contentType,
|
||||
body: text,
|
||||
retryable: res.status >= 500,
|
||||
transientNetwork: res.status >= 500
|
||||
});
|
||||
const err = new Error(`API-Antwort war kein JSON (HTTP ${res.status}): ${(text || '').slice(0, 200)}`);
|
||||
if (res.status >= 500) err.transientNetwork = true;
|
||||
throw err;
|
||||
}
|
||||
|
||||
const apiStatus = Number(data && data.status);
|
||||
const effectiveStatus = res.status < 200 || res.status >= 300
|
||||
? res.status
|
||||
: (apiStatus >= 400 ? apiStatus : null);
|
||||
if (effectiveStatus) {
|
||||
const retryable = effectiveStatus === 429 || effectiveStatus >= 500;
|
||||
throw createTransportError(`${hosterName}: Upload-Server-Abfrage wurde abgelehnt`, {
|
||||
phase: 'upload-server',
|
||||
endpoint: url,
|
||||
httpStatus: effectiveStatus,
|
||||
contentType,
|
||||
body: text,
|
||||
retryable,
|
||||
transientNetwork: effectiveStatus >= 500,
|
||||
accountError: effectiveStatus === 401 || effectiveStatus === 403
|
||||
});
|
||||
if (data.status && [401, 403, 429, 500].includes(data.status)) {
|
||||
const err = new Error(data.msg || data.message || JSON.stringify(data));
|
||||
if (data.status === 500) err.transientNetwork = true;
|
||||
throw err;
|
||||
}
|
||||
return data;
|
||||
} finally {
|
||||
@@ -381,14 +347,12 @@ async function apiGet(url, signal, hosterName) {
|
||||
async function getUploadServer(hosterName, hosterConfig, apiKey, signal) {
|
||||
let lastMessage = '';
|
||||
let lastTransient = false;
|
||||
let lastError = null;
|
||||
|
||||
for (let attempt = 1; attempt <= SERVER_RETRY_ATTEMPTS; attempt++) {
|
||||
for (const endpoint of hosterConfig.serverEndpoints) {
|
||||
const url = `${hosterConfig.apiBase}${endpoint}?key=${encodeURIComponent(apiKey)}`;
|
||||
try {
|
||||
const data = await apiGet(url, signal, hosterName);
|
||||
lastError = null;
|
||||
const data = await apiGet(url, signal);
|
||||
const uploadUrl = extractUploadServerUrl(data, hosterConfig.apiBase);
|
||||
if (uploadUrl) {
|
||||
LAST_UPLOAD_SERVERS.set(hosterName, uploadUrl);
|
||||
@@ -401,16 +365,12 @@ async function getUploadServer(hosterName, hosterConfig, apiKey, signal) {
|
||||
if (apiMessage) lastMessage = apiMessage;
|
||||
} catch (err) {
|
||||
if (err.name === 'AbortError') throw err;
|
||||
lastError = err;
|
||||
if (err.message) lastMessage = err.message;
|
||||
if (err.transientNetwork === true) lastTransient = true;
|
||||
}
|
||||
}
|
||||
|
||||
const retryable = lastError && lastError.diagnostic
|
||||
? lastError.diagnostic.retryable === true
|
||||
: shouldRetryServerLookup(lastMessage);
|
||||
if (attempt < SERVER_RETRY_ATTEMPTS && retryable) {
|
||||
if (attempt < SERVER_RETRY_ATTEMPTS && shouldRetryServerLookup(lastMessage)) {
|
||||
await sleep(SERVER_RETRY_DELAY_MS, signal);
|
||||
continue;
|
||||
}
|
||||
@@ -419,14 +379,11 @@ async function getUploadServer(hosterName, hosterConfig, apiKey, signal) {
|
||||
}
|
||||
|
||||
const cachedServer = LAST_UPLOAD_SERVERS.get(hosterName);
|
||||
const retryable = lastError && lastError.diagnostic
|
||||
? lastError.diagnostic.retryable === true
|
||||
: shouldRetryServerLookup(lastMessage);
|
||||
if (cachedServer && retryable) {
|
||||
if (cachedServer && shouldRetryServerLookup(lastMessage)) {
|
||||
return cachedServer;
|
||||
}
|
||||
|
||||
if (retryable && Array.isArray(hosterConfig.fallbackUploadServers)) {
|
||||
if (shouldRetryServerLookup(lastMessage) && Array.isArray(hosterConfig.fallbackUploadServers)) {
|
||||
for (const fallback of hosterConfig.fallbackUploadServers) {
|
||||
const normalized = normalizeAbsoluteUrl(fallback, hosterConfig.apiBase);
|
||||
if (normalized) {
|
||||
@@ -437,291 +394,50 @@ async function getUploadServer(hosterName, hosterConfig, apiKey, signal) {
|
||||
}
|
||||
|
||||
if (lastMessage) {
|
||||
const e = lastError || createTransportError(`Kein Upload-Server für ${hosterName} erhalten`, {
|
||||
phase: 'upload-server',
|
||||
endpoint: hosterConfig.apiBase,
|
||||
retryable
|
||||
});
|
||||
if (retryable) e.hosterTransient = true;
|
||||
const e = new Error(`Kein Upload-Server erhalten: ${lastMessage}`);
|
||||
// "no servers available" / busy / try-again is a transient hoster-side
|
||||
// condition, not an account fault — tag it so the account isn't blacklisted.
|
||||
// Genuine auth failures (invalid key / unauthorized / forbidden) make
|
||||
// shouldRetryServerLookup return false and stay classified as account errors.
|
||||
if (shouldRetryServerLookup(lastMessage)) e.hosterTransient = true;
|
||||
if (lastTransient) e.transientNetwork = true;
|
||||
throw e;
|
||||
}
|
||||
throw createTransportError(`Kein Upload-Server für ${hosterName} erhalten`, {
|
||||
phase: 'upload-server',
|
||||
endpoint: hosterConfig.apiBase
|
||||
});
|
||||
throw new Error('Kein Upload-Server erhalten. API-Key prüfen.');
|
||||
}
|
||||
|
||||
async function _requestFileList(url, signal, phase, hosterName) {
|
||||
let response;
|
||||
try {
|
||||
response = await request(url, {
|
||||
method: 'GET', signal,
|
||||
headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' },
|
||||
headersTimeout: 30_000, bodyTimeout: 30_000
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
throw createTransportError(`${hosterName}: Dateiliste konnte nicht geladen werden`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
}
|
||||
|
||||
const contentType = response.headers && response.headers['content-type'];
|
||||
let text;
|
||||
try {
|
||||
text = await response.body.text();
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
throw createTransportError(`${hosterName}: Dateiliste konnte nicht gelesen werden`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: response.statusCode,
|
||||
contentType,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
}
|
||||
|
||||
if (response.statusCode < 200 || response.statusCode >= 300) {
|
||||
const retryable = response.statusCode === 429 || response.statusCode >= 500;
|
||||
throw createTransportError(`${hosterName}: Dateiliste konnte nicht geladen werden`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: response.statusCode,
|
||||
contentType,
|
||||
body: text,
|
||||
retryable,
|
||||
transientNetwork: response.statusCode >= 500
|
||||
});
|
||||
}
|
||||
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(text);
|
||||
} catch {
|
||||
throw createTransportError(`${hosterName}: Dateiliste war kein JSON`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: response.statusCode,
|
||||
contentType,
|
||||
body: text
|
||||
});
|
||||
}
|
||||
|
||||
if (!data || typeof data !== 'object') {
|
||||
throw createTransportError(`${hosterName}: Dateiliste hatte ein ungültiges Format`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: response.statusCode,
|
||||
contentType,
|
||||
body: text
|
||||
});
|
||||
}
|
||||
|
||||
const apiStatus = Number(data && data.status);
|
||||
const statusText = typeof data.status === 'string' ? data.status.trim().toLowerCase() : '';
|
||||
const semanticFailure = data.success === false
|
||||
|| data.ok === false
|
||||
|| data.status === false
|
||||
|| /^(?:error|failed|failure|denied|invalid|rejected)$/.test(statusText);
|
||||
if (apiStatus >= 400 || semanticFailure) {
|
||||
const retryable = apiStatus === 429 || apiStatus >= 500;
|
||||
throw createTransportError(`${hosterName}: Dateiliste wurde abgelehnt`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: apiStatus >= 100 ? apiStatus : response.statusCode,
|
||||
contentType,
|
||||
body: text,
|
||||
retryable,
|
||||
transientNetwork: apiStatus >= 500
|
||||
});
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _requireFileList(data, candidates, phase, hosterName, url) {
|
||||
for (const candidate of candidates) {
|
||||
if (Array.isArray(candidate)) return candidate;
|
||||
}
|
||||
throw createTransportError(`${hosterName}: Dateiliste hatte ein ungültiges Format`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: 200,
|
||||
contentType: 'application/json'
|
||||
});
|
||||
}
|
||||
|
||||
async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
||||
async function _fetchByseFileList(apiKey, signal) {
|
||||
// Byse's file-list endpoint. Returns up to 100 most-recent files — enough
|
||||
// to match the upload we just did against what the server has. The API
|
||||
// shape is typical XFS: { status, msg, result: { files: [...] } } or
|
||||
// { status, msg, files: [...] }.
|
||||
const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`;
|
||||
const data = await _requestFileList(url, signal, phase, 'Byse');
|
||||
const src = _requireFileList(data, [
|
||||
data.files,
|
||||
data.result && data.result.files,
|
||||
data.result
|
||||
], phase, 'Byse', url);
|
||||
return src.map(f => ({
|
||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||
file_name: String(f.title || f.name || f.file_name || '').trim()
|
||||
})).filter(f => f.file_code);
|
||||
try {
|
||||
const { body, statusCode } = await request(url, {
|
||||
method: 'GET', signal,
|
||||
headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' },
|
||||
headersTimeout: 30_000, bodyTimeout: 30_000
|
||||
});
|
||||
const text = await body.text();
|
||||
if (statusCode < 200 || statusCode >= 300) return [];
|
||||
const data = JSON.parse(text);
|
||||
const src = Array.isArray(data.files) ? data.files
|
||||
: (data.result && Array.isArray(data.result.files) ? data.result.files
|
||||
: (Array.isArray(data.result) ? data.result : []));
|
||||
return src.map(f => ({
|
||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||
file_name: String(f.title || f.name || f.file_name || '').trim()
|
||||
})).filter(f => f.file_code);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function _normalizeFileTitle(s) {
|
||||
const normalized = String(s || '')
|
||||
.normalize('NFKD')
|
||||
.toLowerCase()
|
||||
.replace(/\.[\p{Letter}\p{Number}]+$/u, '')
|
||||
.replace(/\p{Variation_Selector}+/gu, '');
|
||||
const alphanumeric = normalized
|
||||
.replace(/\p{Mark}+/gu, '')
|
||||
.replace(/[^\p{Letter}\p{Number}]+/gu, '');
|
||||
if (alphanumeric) return alphanumeric;
|
||||
const codePoints = Array.from(normalized, value => value.codePointAt(0).toString(16)).join('-');
|
||||
return `symbols:${codePoints}`;
|
||||
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
||||
}
|
||||
|
||||
function _normalizeRecoveryHoster(value) {
|
||||
return String(value || '').normalize('NFKC').trim().toLowerCase();
|
||||
}
|
||||
|
||||
function _normalizeRecoveryAccount(value) {
|
||||
return String(value || '').normalize('NFKC').trim();
|
||||
}
|
||||
|
||||
function _createRecoveryUncertainError() {
|
||||
const error = new Error('Upload-Ergebnis für diesen Titel ist wegen eines möglichen Remote-Commits unsicher');
|
||||
error.remoteCommitUncertain = true;
|
||||
error.hosterTransient = true;
|
||||
return error;
|
||||
}
|
||||
|
||||
function _markRecoveryUncertain(recoveryClaim, error) {
|
||||
if (!recoveryClaim) return error;
|
||||
if (typeof recoveryClaim.markUncertain === 'function') {
|
||||
return recoveryClaim.markUncertain(error);
|
||||
}
|
||||
const uncertainError = error && typeof error === 'object'
|
||||
? error
|
||||
: _createRecoveryUncertainError();
|
||||
uncertainError.remoteCommitUncertain = true;
|
||||
uncertainError.hosterTransient = true;
|
||||
return uncertainError;
|
||||
}
|
||||
|
||||
function _createAbortError() {
|
||||
const error = new Error('Operation aborted');
|
||||
error.name = 'AbortError';
|
||||
return error;
|
||||
}
|
||||
|
||||
function _waitForRecoveryTurn(predecessor, signal) {
|
||||
if (!signal) return predecessor;
|
||||
if (signal.aborted) return Promise.reject(_createAbortError());
|
||||
return new Promise((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
reject(_createAbortError());
|
||||
};
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
predecessor.then(
|
||||
() => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
resolve();
|
||||
},
|
||||
error => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
reject(error);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function createRecoveryClaimRegistry() {
|
||||
const accounts = new Map();
|
||||
let nextClaimId = 1;
|
||||
return {
|
||||
forUpload(hosterName, apiKey, fileName) {
|
||||
const accountIdentity = crypto.createHash('sha256')
|
||||
.update(`${_normalizeRecoveryHoster(hosterName)}\0${_normalizeRecoveryAccount(apiKey)}`)
|
||||
.digest('hex');
|
||||
let account = accounts.get(accountIdentity);
|
||||
if (!account) {
|
||||
account = {
|
||||
codes: new Map(),
|
||||
titles: new Map()
|
||||
};
|
||||
accounts.set(accountIdentity, account);
|
||||
}
|
||||
const titleIdentity = _normalizeFileTitle(fileName);
|
||||
let title = account.titles.get(titleIdentity);
|
||||
if (!title) {
|
||||
title = {
|
||||
tail: Promise.resolve(),
|
||||
uncertain: false
|
||||
};
|
||||
account.titles.set(titleIdentity, title);
|
||||
}
|
||||
const claimId = nextClaimId++;
|
||||
return {
|
||||
has(code) {
|
||||
return account.codes.has(String(code || '').trim());
|
||||
},
|
||||
reserve(code) {
|
||||
const normalized = String(code || '').trim();
|
||||
if (!normalized) return false;
|
||||
if (account.codes.has(normalized)) {
|
||||
return account.codes.get(normalized) === claimId;
|
||||
}
|
||||
account.codes.set(normalized, claimId);
|
||||
return true;
|
||||
},
|
||||
markUncertain(error) {
|
||||
title.uncertain = true;
|
||||
const uncertainError = error && typeof error === 'object'
|
||||
? error
|
||||
: _createRecoveryUncertainError();
|
||||
uncertainError.remoteCommitUncertain = true;
|
||||
uncertainError.hosterTransient = true;
|
||||
return uncertainError;
|
||||
},
|
||||
isUncertain() {
|
||||
return title.uncertain;
|
||||
},
|
||||
async runExclusive(operation, signal) {
|
||||
const predecessor = title.tail;
|
||||
let release;
|
||||
const current = new Promise(resolve => {
|
||||
release = resolve;
|
||||
});
|
||||
title.tail = predecessor.then(() => current, () => current);
|
||||
try {
|
||||
await _waitForRecoveryTurn(predecessor, signal);
|
||||
if (title.uncertain) throw _createRecoveryUncertainError();
|
||||
const result = await operation();
|
||||
if (title.uncertain) throw _createRecoveryUncertainError();
|
||||
return result;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
};
|
||||
},
|
||||
clear() {
|
||||
accounts.clear();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
|
||||
if (!(baselineCodes instanceof Set)) return null;
|
||||
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) {
|
||||
const expected = _normalizeFileTitle(fileName);
|
||||
const POLL_ATTEMPTS = 15;
|
||||
const POLL_DELAY_MS = 2000;
|
||||
@@ -734,13 +450,8 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal,
|
||||
// poller could claim job B's newly appeared file and return the wrong
|
||||
// URL. At the cost of a few false-negatives when byse mangles the
|
||||
// filename beyond our normalizer, correctness for parallel uploads wins.
|
||||
const matches = newFiles
|
||||
.filter(f => _normalizeFileTitle(f.file_name) === expected)
|
||||
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
|
||||
if (matches.length > 1) return null;
|
||||
if (matches.length === 1) {
|
||||
const match = matches[0];
|
||||
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
|
||||
const match = newFiles.find(f => _normalizeFileTitle(f.file_name) === expected);
|
||||
if (match) {
|
||||
return {
|
||||
download_url: `https://byse.sx/d/${match.file_code}`,
|
||||
embed_url: `https://byse.sx/e/${match.file_code}`,
|
||||
@@ -758,24 +469,34 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal,
|
||||
return null;
|
||||
}
|
||||
|
||||
async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll') {
|
||||
async function _fetchDoodstreamFileList(apiKey, signal) {
|
||||
// doodapi.co file list: { msg, status:200, result: { files: [{ file_code, title, uploaded, ... }] } }
|
||||
// sort=created&order=desc forces newest-first — VERIFIED against a real 90k-file
|
||||
// account, where a single page without it could miss a just-uploaded file. The
|
||||
// recovery only needs the most recent uploads, so page 1 newest-first suffices.
|
||||
const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`;
|
||||
const data = await _requestFileList(url, signal, phase, 'Doodstream');
|
||||
const files = _requireFileList(data, [data && data.result && data.result.files], phase, 'Doodstream', url);
|
||||
return files.map(f => ({
|
||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||
file_name: String(f.title || f.file_name || f.name || '').trim()
|
||||
})).filter(f => f.file_code);
|
||||
try {
|
||||
const { body, statusCode } = await request(url, {
|
||||
method: 'GET', signal,
|
||||
headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' },
|
||||
headersTimeout: 30_000, bodyTimeout: 30_000
|
||||
});
|
||||
const text = await body.text();
|
||||
if (statusCode < 200 || statusCode >= 300) return [];
|
||||
const data = JSON.parse(text);
|
||||
const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : [];
|
||||
return files.map(f => ({
|
||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||
file_name: String(f.title || f.file_name || f.name || '').trim()
|
||||
})).filter(f => f.file_code);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test
|
||||
|
||||
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
|
||||
if (!(baselineCodes instanceof Set)) return null;
|
||||
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal) {
|
||||
// Same recovery byse uses: the upload POST returned no filecode, but the file
|
||||
// may register in the account a little later. Poll the list for a NEW file
|
||||
// whose normalized title matches what we uploaded. Exact-name match only
|
||||
@@ -788,13 +509,8 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s
|
||||
if (signal && signal.aborted) return null;
|
||||
const list = await _fetchDoodstreamFileList(apiKey, signal);
|
||||
const fresh = list.filter(f => !baselineCodes.has(f.file_code));
|
||||
const matches = fresh
|
||||
.filter(f => _normalizeFileTitle(f.file_name) === expected)
|
||||
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
|
||||
if (matches.length > 1) return null;
|
||||
if (matches.length === 1) {
|
||||
const match = matches[0];
|
||||
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
|
||||
const match = fresh.find(f => _normalizeFileTitle(f.file_name) === expected);
|
||||
if (match) {
|
||||
return {
|
||||
download_url: `https://doodstream.com/d/${match.file_code}`,
|
||||
embed_url: `https://doodstream.com/e/${match.file_code}`,
|
||||
@@ -817,33 +533,21 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
if (!config) throw new Error(`Unbekannter Hoster: ${hosterName}`);
|
||||
|
||||
let byseBaseline = null;
|
||||
let byseBaselineError = null;
|
||||
if (hosterName === 'byse.sx') {
|
||||
if (opts && opts.byseBaseline instanceof Set) {
|
||||
byseBaseline = opts.byseBaseline;
|
||||
} else {
|
||||
try {
|
||||
const baseline = await _fetchByseFileList(apiKey, signal, 'recovery-baseline');
|
||||
byseBaseline = new Set(baseline.map(f => f.file_code));
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
byseBaselineError = err;
|
||||
}
|
||||
const baseline = await _fetchByseFileList(apiKey, signal);
|
||||
byseBaseline = new Set(baseline.map(f => f.file_code));
|
||||
}
|
||||
}
|
||||
let doodBaseline = null;
|
||||
let doodBaselineError = null;
|
||||
if (hosterName === 'doodstream.com') {
|
||||
if (opts && opts.doodBaseline instanceof Set) {
|
||||
doodBaseline = opts.doodBaseline;
|
||||
} else {
|
||||
try {
|
||||
const baseline = await _fetchDoodstreamFileList(apiKey, signal, 'recovery-baseline');
|
||||
doodBaseline = new Set(baseline.map(f => f.file_code));
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
doodBaselineError = err;
|
||||
}
|
||||
const baseline = await _fetchDoodstreamFileList(apiKey, signal);
|
||||
doodBaseline = new Set(baseline.map(f => f.file_code));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -856,52 +560,31 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
|
||||
const { iterable, boundary, totalSize } = createUploadBody(filePath, formFields, onProgress, throttle, signal);
|
||||
|
||||
let uploadResponse;
|
||||
try {
|
||||
uploadResponse = await request(targetUrl, {
|
||||
method: 'POST',
|
||||
body: iterable,
|
||||
signal,
|
||||
headers: {
|
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||||
'Content-Length': String(totalSize),
|
||||
'Accept': 'application/json, text/plain;q=0.9, */*;q=0.8',
|
||||
'User-Agent': 'multi-hoster-uploader/1.1'
|
||||
},
|
||||
headersTimeout: UPLOAD_TIMEOUT,
|
||||
bodyTimeout: UPLOAD_TIMEOUT
|
||||
});
|
||||
} catch (err) {
|
||||
const error = signal && signal.aborted ? err : createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
|
||||
phase: 'upload-request',
|
||||
endpoint: targetUrl,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, error);
|
||||
}
|
||||
const { body, statusCode, headers } = await request(targetUrl, {
|
||||
method: 'POST',
|
||||
body: iterable,
|
||||
signal,
|
||||
headers: {
|
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||||
'Content-Length': String(totalSize),
|
||||
'Accept': 'application/json, text/plain;q=0.9, */*;q=0.8',
|
||||
'User-Agent': 'multi-hoster-uploader/1.1'
|
||||
},
|
||||
headersTimeout: UPLOAD_TIMEOUT,
|
||||
bodyTimeout: UPLOAD_TIMEOUT
|
||||
});
|
||||
|
||||
const { body, statusCode, headers } = uploadResponse;
|
||||
|
||||
let rawBody;
|
||||
try {
|
||||
rawBody = await body.text();
|
||||
} catch (err) {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||
}
|
||||
const rawBody = await body.text();
|
||||
let payload = null;
|
||||
try {
|
||||
payload = rawBody ? JSON.parse(rawBody) : {};
|
||||
} catch {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
retryable: statusCode >= 500,
|
||||
transientNetwork: statusCode >= 500
|
||||
}));
|
||||
const snippet = rawBody ? rawBody.slice(0, 240).replace(/\s+/g, ' ').trim() : '';
|
||||
const err = new Error(
|
||||
`Upload-Antwort von ${hosterName} war kein JSON (HTTP ${statusCode}${snippet ? `): ${snippet}` : ')'}`
|
||||
);
|
||||
if (statusCode >= 500) err.transientNetwork = true;
|
||||
throw err;
|
||||
}
|
||||
// Normalize valid-but-not-object JSON (JSON.parse('null') → null;
|
||||
// JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this
|
||||
@@ -915,33 +598,19 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
}
|
||||
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
const error = createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
retryable: statusCode === 429 || statusCode >= 500,
|
||||
transientNetwork: statusCode >= 500
|
||||
});
|
||||
throw statusCode >= 500
|
||||
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
|
||||
: error;
|
||||
const err = new Error(
|
||||
payload.msg
|
||||
|| payload.message
|
||||
|| `Upload fehlgeschlagen (HTTP ${statusCode}${headers?.['content-type'] ? `, ${headers['content-type']}` : ''})`
|
||||
);
|
||||
if (statusCode >= 500) err.transientNetwork = true;
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
|
||||
const error = createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: Number(payload.status),
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
retryable: Number(payload.status) === 429 || Number(payload.status) >= 500,
|
||||
transientNetwork: Number(payload.status) >= 500
|
||||
});
|
||||
throw Number(payload.status) >= 500
|
||||
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
|
||||
: error;
|
||||
const err = new Error(payload.msg || payload.message || JSON.stringify(payload));
|
||||
if (payload.status === 500) err.transientNetwork = true;
|
||||
throw err;
|
||||
}
|
||||
|
||||
let result = null;
|
||||
@@ -950,32 +619,19 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
result = config.parseResult(payload);
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && !err.diagnostic) {
|
||||
err.diagnostic = createTransportError(`Upload zu ${hosterName} konnte nicht ausgewertet werden`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody
|
||||
}).diagnostic;
|
||||
try {
|
||||
err.diagnostic = {
|
||||
hoster: hosterName,
|
||||
http: statusCode,
|
||||
contentType: (headers && headers['content-type']) || null,
|
||||
payloadSnippet: JSON.stringify(payload).slice(0, 1000),
|
||||
uploadUrl: targetUrl
|
||||
};
|
||||
} catch { /* JSON cycle — skip diagnostic */ }
|
||||
}
|
||||
parseErr = err;
|
||||
}
|
||||
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
||||
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
|
||||
if (!opts.recoveryClaim.reserve(result.file_code)) {
|
||||
const error = createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
retryable: true,
|
||||
hosterTransient: true
|
||||
});
|
||||
error.remoteIdentityClaimed = true;
|
||||
throw _markRecoveryUncertain(opts.recoveryClaim, error);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1001,12 +657,8 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// even after our uploader gave up.
|
||||
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
||||
const fileName = path.basename(filePath);
|
||||
try {
|
||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
} catch (err) {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||
}
|
||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal);
|
||||
if (polled) return polled;
|
||||
}
|
||||
|
||||
// Doodstream: the doodapi upload POST returned no filecode (the same backend
|
||||
@@ -1014,47 +666,24 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// the file did register, claim its code instead of failing the upload.
|
||||
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
||||
const fileName = path.basename(filePath);
|
||||
try {
|
||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
} catch (err) {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||
}
|
||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal);
|
||||
if (polled) return polled;
|
||||
}
|
||||
|
||||
if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) {
|
||||
byseBaselineError.hosterTransient = true;
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, byseBaselineError);
|
||||
}
|
||||
|
||||
if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) {
|
||||
doodBaselineError.hosterTransient = true;
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, doodBaselineError);
|
||||
}
|
||||
|
||||
if (parseErr) {
|
||||
throw explicitlyRejected
|
||||
? parseErr
|
||||
: _markRecoveryUncertain(opts && opts.recoveryClaim, parseErr);
|
||||
}
|
||||
if (parseErr) throw parseErr;
|
||||
|
||||
if (payload.success === false) {
|
||||
throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody
|
||||
});
|
||||
throw new Error(payload.msg || payload.message || `Upload zu ${hosterName} wurde vom Server abgelehnt.`);
|
||||
}
|
||||
|
||||
// Avoid throwing a bare "OK" / "SUCCESS" as the error message — that happens
|
||||
// when the server says "msg: OK" but ships no file_code anywhere we know
|
||||
// about, typically an API change. Surface safe structured response metadata
|
||||
// so future logs show what kind of response the server returned.
|
||||
// about, typically an API change. Surface the full (trimmed) payload so
|
||||
// future logs actually show what the server returned.
|
||||
const msg = String(payload.msg || payload.message || '').trim();
|
||||
const isOkishNoPayload = /^(ok|success|done|accepted)$/i.test(msg);
|
||||
if (isOkishNoPayload || !msg) {
|
||||
const snippet = JSON.stringify(payload).slice(0, 400);
|
||||
// 2xx with no filecode: the hoster accepted the upload (bytes sent, status
|
||||
// OK) but returned no usable link. For doodstream this is the API-path
|
||||
// analog of the web empty-form — the backend file-registration timing out
|
||||
@@ -1062,33 +691,23 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// so tag it hosterTransient: the upload-manager then fails this file WITHOUT
|
||||
// blacklisting the account (same protection the web path got in 3.3.29) and
|
||||
// the account stays usable for the next retry/batch.
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
retryable: true,
|
||||
hosterTransient: true
|
||||
}));
|
||||
const err = new Error(
|
||||
`Upload zu ${hosterName} lieferte keine file_code-Antwort (Payload: ${snippet})`
|
||||
);
|
||||
err.hosterTransient = true;
|
||||
throw err;
|
||||
}
|
||||
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody
|
||||
});
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
async function prefetchBaseline(hosterName, apiKey, signal) {
|
||||
try {
|
||||
if (hosterName === 'byse.sx') {
|
||||
const baseline = await _fetchByseFileList(apiKey, signal, 'recovery-baseline');
|
||||
const baseline = await _fetchByseFileList(apiKey, signal);
|
||||
return new Set(baseline.map(f => f.file_code));
|
||||
}
|
||||
if (hosterName === 'doodstream.com') {
|
||||
const baseline = await _fetchDoodstreamFileList(apiKey, signal, 'recovery-baseline');
|
||||
const baseline = await _fetchDoodstreamFileList(apiKey, signal);
|
||||
return new Set(baseline.map(f => f.file_code));
|
||||
}
|
||||
} catch { /* leave caller to fall back to per-job fetch */ }
|
||||
@@ -1098,8 +717,6 @@ async function prefetchBaseline(hosterName, apiKey, signal) {
|
||||
module.exports = {
|
||||
uploadFile,
|
||||
prefetchBaseline,
|
||||
createRecoveryClaimRegistry,
|
||||
normalizeRecoveryTitle: _normalizeFileTitle,
|
||||
HOSTER_CONFIGS,
|
||||
__test: {
|
||||
extractUploadServerUrl,
|
||||
|
||||
Reference in New Issue
Block a user