1e449e3d67
User reported uploads appearing on the byse dashboard (2+ GB MKV, Server #262, status OK) even though the app marked them failed. The byse API sometimes replies with msg=OK + files:[{filecode:"", status:"Not video file format"}] — a misleading response where the file is actually being accepted and gets its filecode assigned asynchronously. - Before the upload POST, snapshot the current /api/file/list to know what was already there. - If parseByseResult returns an empty filecode (or throws a fileRejected error), poll /api/file/list up to 15 × 2s looking for a new file_code matching the uploaded filename (case/punct normalized, extension stripped). - If matched, return the real download/embed URLs and let the upload complete as successful. Only throw the parser's error if polling also finds nothing.
536 lines
17 KiB
JavaScript
536 lines
17 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
const { request } = require('undici');
|
|
|
|
const UPLOAD_TIMEOUT = 1800000; // 30 minutes
|
|
const API_TIMEOUT = 45000; // 45 seconds
|
|
const SERVER_RETRY_ATTEMPTS = 6;
|
|
const SERVER_RETRY_DELAY_MS = 2500;
|
|
const LAST_UPLOAD_SERVERS = new Map();
|
|
|
|
function appendRawQuery(url, rawQuery) {
|
|
const parsed = new URL(url);
|
|
const cleanQuery = String(rawQuery || '').trim().replace(/^\?+/, '');
|
|
if (!cleanQuery) return parsed.toString();
|
|
|
|
if (parsed.search && parsed.search.length > 1) {
|
|
parsed.search = `${parsed.search.slice(1)}&${cleanQuery}`;
|
|
} else {
|
|
parsed.search = cleanQuery;
|
|
}
|
|
|
|
return parsed.toString();
|
|
}
|
|
|
|
function appendKeyParam(url, key) {
|
|
const parsed = new URL(url);
|
|
parsed.searchParams.set('key', key);
|
|
return parsed.toString();
|
|
}
|
|
|
|
// Hoster definitions - based on official API docs
|
|
const HOSTER_CONFIGS = {
|
|
'doodstream.com': {
|
|
apiBase: 'https://doodapi.co',
|
|
serverEndpoints: ['/api/upload/server'],
|
|
fallbackUploadServers: ['https://tr1128ve.cloudatacdn.com/upload/01'],
|
|
buildUploadUrl: (url, key) => appendRawQuery(url, key),
|
|
formFields: (key) => ({ api_key: key }),
|
|
parseResult: parseDoodstreamResult
|
|
},
|
|
'voe.sx': {
|
|
apiBase: 'https://voe.sx',
|
|
serverEndpoints: ['/api/upload/server', '/api/v1/upload/server'],
|
|
buildUploadUrl: (url, key) => appendKeyParam(url, key),
|
|
formFields: () => ({}),
|
|
parseResult: parseVoeResult
|
|
},
|
|
'byse.sx': {
|
|
apiBase: 'https://api.byse.sx',
|
|
serverEndpoints: ['/upload/server'],
|
|
buildUploadUrl: (url, key) => appendKeyParam(url, key),
|
|
formFields: (key) => ({ key }),
|
|
parseResult: parseByseResult
|
|
}
|
|
};
|
|
|
|
function normalizeAbsoluteUrl(raw, apiBase) {
|
|
if (typeof raw !== 'string') return null;
|
|
const trimmed = raw.trim();
|
|
if (!trimmed || /^\[object\s+Object\]$/i.test(trimmed)) return null;
|
|
|
|
let candidate = trimmed;
|
|
if (candidate.startsWith('//')) {
|
|
candidate = `https:${candidate}`;
|
|
} else if (candidate.startsWith('/')) {
|
|
try {
|
|
candidate = new URL(candidate, apiBase).href;
|
|
} catch {
|
|
return null;
|
|
}
|
|
} else if (!/^[a-z][a-z\d+.-]*:\/\//i.test(candidate)) {
|
|
candidate = `https://${candidate.replace(/^\/+/, '')}`;
|
|
}
|
|
|
|
try {
|
|
const parsed = new URL(candidate);
|
|
if (!['http:', 'https:'].includes(parsed.protocol)) return null;
|
|
return parsed.href;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function collectUploadUrlCandidates(value, out = []) {
|
|
if (typeof value === 'string') {
|
|
out.push(value);
|
|
return out;
|
|
}
|
|
|
|
if (Array.isArray(value)) {
|
|
for (const entry of value) collectUploadUrlCandidates(entry, out);
|
|
return out;
|
|
}
|
|
|
|
if (value && typeof value === 'object') {
|
|
const preferredKeys = ['upload_url', 'uploadUrl', 'url', 'server', 'srv', 'result'];
|
|
for (const key of preferredKeys) {
|
|
if (Object.prototype.hasOwnProperty.call(value, key)) {
|
|
collectUploadUrlCandidates(value[key], out);
|
|
}
|
|
}
|
|
|
|
for (const nested of Object.values(value)) {
|
|
if (typeof nested === 'string') out.push(nested);
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
function extractUploadServerUrl(payload, apiBase) {
|
|
const source = payload && Object.prototype.hasOwnProperty.call(payload, 'result')
|
|
? payload.result
|
|
: payload;
|
|
|
|
const candidates = collectUploadUrlCandidates(source, []);
|
|
for (const candidate of candidates) {
|
|
const normalized = normalizeAbsoluteUrl(candidate, apiBase);
|
|
if (normalized) return normalized;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function shouldRetryServerLookup(message) {
|
|
const msg = String(message || '').toLowerCase();
|
|
if (!msg) return true;
|
|
if (msg.includes('invalid') && msg.includes('key')) return false;
|
|
if (msg.includes('unauthorized') || msg.includes('forbidden')) return false;
|
|
if (msg.includes('no servers available')) return true;
|
|
if (msg.includes('temporar') || msg.includes('busy') || msg.includes('try again')) return true;
|
|
return true;
|
|
}
|
|
|
|
function sleep(ms, signal) {
|
|
return new Promise((resolve, reject) => {
|
|
const timer = setTimeout(() => {
|
|
if (signal) signal.removeEventListener('abort', onAbort);
|
|
resolve();
|
|
}, ms);
|
|
|
|
function onAbort() {
|
|
clearTimeout(timer);
|
|
if (signal) signal.removeEventListener('abort', onAbort);
|
|
const err = new Error('Aborted');
|
|
err.name = 'AbortError';
|
|
reject(err);
|
|
}
|
|
|
|
if (signal) {
|
|
if (signal.aborted) return onAbort();
|
|
signal.addEventListener('abort', onAbort);
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- Result parsers ---
|
|
|
|
// Doodstream: { result: [{ download_url, protected_embed, filecode, protected_dl }] }
|
|
function parseDoodstreamResult(payload) {
|
|
let item = {};
|
|
const result = payload.result;
|
|
if (Array.isArray(result) && result.length > 0) {
|
|
item = result[0];
|
|
} else if (result && typeof result === 'object') {
|
|
item = result;
|
|
}
|
|
|
|
return {
|
|
download_url: item.download_url || item.protected_dl || null,
|
|
embed_url: item.protected_embed || null,
|
|
file_code: item.filecode || item.file_code || null
|
|
};
|
|
}
|
|
|
|
// VOE: { file: { file_code } }
|
|
function parseVoeResult(payload) {
|
|
const source = payload && typeof payload === 'object' && payload.result && typeof payload.result === 'object'
|
|
? payload.result
|
|
: payload;
|
|
const file = source && typeof source.file === 'object' ? source.file : null;
|
|
const file_code = file?.file_code
|
|
|| file?.filecode
|
|
|| source?.file_code
|
|
|| source?.filecode
|
|
|| null;
|
|
|
|
return {
|
|
download_url: file_code ? `https://voe.sx/${file_code}` : null,
|
|
embed_url: file_code ? `https://voe.sx/e/${file_code}` : null,
|
|
file_code
|
|
};
|
|
}
|
|
|
|
// Byse: { files: [{ filecode, filename, status }] }
|
|
function parseByseResult(payload) {
|
|
let file_code = null;
|
|
let perFileError = null;
|
|
|
|
// Primary: files array (per official Byse API docs)
|
|
if (Array.isArray(payload.files) && payload.files.length > 0) {
|
|
const f = payload.files[0];
|
|
file_code = f.filecode || f.file_code || null;
|
|
// Byse returns HTTP 200 + msg=OK even when a specific file was rejected
|
|
// ("Not video file format", "Duplicate", "File too small", ...). When
|
|
// filecode is empty and status carries a non-OK message, that IS the
|
|
// actual per-file error, not a server problem.
|
|
if (!file_code && f.status && !/^(ok|success|done)$/i.test(String(f.status))) {
|
|
perFileError = String(f.status).trim();
|
|
}
|
|
}
|
|
// Fallback: result object
|
|
if (!file_code && payload.result) {
|
|
const result = payload.result;
|
|
if (Array.isArray(result) && result.length > 0) {
|
|
file_code = result[0].filecode || result[0].file_code;
|
|
} else if (typeof result === 'object') {
|
|
file_code = result.filecode || result.file_code;
|
|
}
|
|
}
|
|
|
|
if (!file_code && perFileError) {
|
|
const err = new Error(`Byse lehnte Datei ab: ${perFileError}`);
|
|
err.fileRejected = true;
|
|
throw err;
|
|
}
|
|
|
|
return {
|
|
download_url: file_code ? `https://byse.sx/d/${file_code}` : null,
|
|
embed_url: file_code ? `https://byse.sx/e/${file_code}` : null,
|
|
file_code
|
|
};
|
|
}
|
|
|
|
// --- Multipart upload with progress ---
|
|
|
|
function buildMultipart(filePath, formFields) {
|
|
const boundary = '----FormBoundary' + crypto.randomBytes(16).toString('hex');
|
|
const fileName = path.basename(filePath);
|
|
const fileSize = fs.statSync(filePath).size;
|
|
|
|
let preamble = '';
|
|
for (const [key, value] of Object.entries(formFields)) {
|
|
preamble += `--${boundary}\r\n`;
|
|
preamble += `Content-Disposition: form-data; name="${key}"\r\n\r\n`;
|
|
preamble += `${value}\r\n`;
|
|
}
|
|
preamble += `--${boundary}\r\n`;
|
|
const safeFileName = fileName.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
preamble += `Content-Disposition: form-data; name="file"; filename="${safeFileName}"\r\n`;
|
|
preamble += `Content-Type: application/octet-stream\r\n\r\n`;
|
|
|
|
const epilogue = `\r\n--${boundary}--\r\n`;
|
|
|
|
const preambleBuf = Buffer.from(preamble, 'utf-8');
|
|
const epilogueBuf = Buffer.from(epilogue, 'utf-8');
|
|
const totalSize = preambleBuf.length + fileSize + epilogueBuf.length;
|
|
|
|
return { boundary, preambleBuf, epilogueBuf, totalSize, fileSize };
|
|
}
|
|
|
|
function createUploadBody(filePath, formFields, onProgress, throttle, signal) {
|
|
const { boundary, preambleBuf, epilogueBuf, totalSize, fileSize } = buildMultipart(filePath, formFields);
|
|
|
|
let bytesRead = 0;
|
|
const CHUNK_SIZE = 256 * 1024;
|
|
|
|
async function* generate() {
|
|
yield preambleBuf;
|
|
const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE });
|
|
for await (const chunk of fileStream) {
|
|
if (signal && signal.aborted) throw new Error('Aborted');
|
|
if (throttle) await throttle.consume(chunk.length, signal);
|
|
bytesRead += chunk.length;
|
|
yield chunk;
|
|
if (onProgress) onProgress(bytesRead, fileSize);
|
|
}
|
|
yield epilogueBuf;
|
|
}
|
|
|
|
return { iterable: generate(), boundary, totalSize };
|
|
}
|
|
|
|
// --- API helper using built-in fetch (follows redirects automatically) ---
|
|
|
|
async function apiGet(url, signal) {
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), API_TIMEOUT);
|
|
const onAbort = () => controller.abort();
|
|
if (signal) signal.addEventListener('abort', onAbort);
|
|
|
|
try {
|
|
const res = await fetch(url, {
|
|
method: 'GET',
|
|
signal: controller.signal,
|
|
redirect: 'follow'
|
|
});
|
|
const text = await res.text();
|
|
let data;
|
|
try {
|
|
data = JSON.parse(text);
|
|
} catch {
|
|
throw new Error(`API-Antwort war kein JSON (HTTP ${res.status}): ${(text || '').slice(0, 200)}`);
|
|
}
|
|
|
|
if (data.status && [401, 403, 429, 500].includes(data.status)) {
|
|
throw new Error(data.msg || data.message || JSON.stringify(data));
|
|
}
|
|
return data;
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
if (signal) signal.removeEventListener('abort', onAbort);
|
|
}
|
|
}
|
|
|
|
// --- Main upload function ---
|
|
|
|
async function getUploadServer(hosterName, hosterConfig, apiKey, signal) {
|
|
let lastMessage = '';
|
|
|
|
for (let attempt = 1; attempt <= SERVER_RETRY_ATTEMPTS; attempt++) {
|
|
for (const endpoint of hosterConfig.serverEndpoints) {
|
|
const url = `${hosterConfig.apiBase}${endpoint}?key=${encodeURIComponent(apiKey)}`;
|
|
try {
|
|
const data = await apiGet(url, signal);
|
|
const uploadUrl = extractUploadServerUrl(data, hosterConfig.apiBase);
|
|
if (uploadUrl) {
|
|
LAST_UPLOAD_SERVERS.set(hosterName, uploadUrl);
|
|
return uploadUrl;
|
|
}
|
|
|
|
const apiMessage = data && (data.msg || data.message)
|
|
? String(data.msg || data.message).trim()
|
|
: '';
|
|
if (apiMessage) lastMessage = apiMessage;
|
|
} catch (err) {
|
|
if (err.name === 'AbortError') throw err;
|
|
if (err.message) lastMessage = err.message;
|
|
}
|
|
}
|
|
|
|
if (attempt < SERVER_RETRY_ATTEMPTS && shouldRetryServerLookup(lastMessage)) {
|
|
await sleep(SERVER_RETRY_DELAY_MS, signal);
|
|
continue;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
const cachedServer = LAST_UPLOAD_SERVERS.get(hosterName);
|
|
if (cachedServer && shouldRetryServerLookup(lastMessage)) {
|
|
return cachedServer;
|
|
}
|
|
|
|
if (shouldRetryServerLookup(lastMessage) && Array.isArray(hosterConfig.fallbackUploadServers)) {
|
|
for (const fallback of hosterConfig.fallbackUploadServers) {
|
|
const normalized = normalizeAbsoluteUrl(fallback, hosterConfig.apiBase);
|
|
if (normalized) {
|
|
LAST_UPLOAD_SERVERS.set(hosterName, normalized);
|
|
return normalized;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (lastMessage) {
|
|
throw new Error(`Kein Upload-Server erhalten: ${lastMessage}`);
|
|
}
|
|
throw new Error('Kein Upload-Server erhalten. API-Key pruefen.');
|
|
}
|
|
|
|
async function _fetchByseFileList(apiKey, signal) {
|
|
// Byse's file-list endpoint. Returns up to 100 most-recent files — enough
|
|
// to match the upload we just did against what the server has. The API
|
|
// shape is typical XFS: { status, msg, result: { files: [...] } } or
|
|
// { status, msg, files: [...] }.
|
|
const url = `https://api.byse.sx/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`;
|
|
try {
|
|
const { body, statusCode } = await request(url, {
|
|
method: 'GET', signal,
|
|
headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' },
|
|
headersTimeout: 30_000, bodyTimeout: 30_000
|
|
});
|
|
const text = await body.text();
|
|
if (statusCode < 200 || statusCode >= 300) return [];
|
|
const data = JSON.parse(text);
|
|
const src = Array.isArray(data.files) ? data.files
|
|
: (data.result && Array.isArray(data.result.files) ? data.result.files
|
|
: (Array.isArray(data.result) ? data.result : []));
|
|
return src.map(f => ({
|
|
file_code: String(f.file_code || f.filecode || '').trim(),
|
|
file_name: String(f.title || f.name || f.file_name || '').trim()
|
|
})).filter(f => f.file_code);
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function _normalizeFileTitle(s) {
|
|
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
|
}
|
|
|
|
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) {
|
|
const expected = _normalizeFileTitle(fileName);
|
|
const POLL_ATTEMPTS = 15;
|
|
const POLL_DELAY_MS = 2000;
|
|
for (let i = 0; i < POLL_ATTEMPTS; i++) {
|
|
if (signal && signal.aborted) return null;
|
|
const list = await _fetchByseFileList(apiKey, signal);
|
|
const newFiles = list.filter(f => !baselineCodes.has(f.file_code));
|
|
// Prefer exact filename match (ignoring case/punctuation/extension)
|
|
const match = newFiles.find(f => _normalizeFileTitle(f.file_name) === expected)
|
|
|| (newFiles.length === 1 ? newFiles[0] : null);
|
|
if (match) {
|
|
return {
|
|
download_url: `https://byse.sx/d/${match.file_code}`,
|
|
embed_url: `https://byse.sx/e/${match.file_code}`,
|
|
file_code: match.file_code
|
|
};
|
|
}
|
|
if (i < POLL_ATTEMPTS - 1) await sleep(POLL_DELAY_MS, signal);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, throttle) {
|
|
const config = HOSTER_CONFIGS[hosterName];
|
|
if (!config) throw new Error(`Unbekannter Hoster: ${hosterName}`);
|
|
|
|
// For byse: snapshot the current file-code list so the post-upload poller
|
|
// can identify new arrivals even when the initial POST response has an
|
|
// empty filecode.
|
|
let byseBaseline = null;
|
|
if (hosterName === 'byse.sx') {
|
|
const baseline = await _fetchByseFileList(apiKey, signal);
|
|
byseBaseline = new Set(baseline.map(f => f.file_code));
|
|
}
|
|
|
|
// Step 1: Get upload server
|
|
const uploadUrl = await getUploadServer(hosterName, config, apiKey, signal);
|
|
|
|
// Step 2: Upload file with progress
|
|
const targetUrl = config.buildUploadUrl(uploadUrl, apiKey);
|
|
const formFields = config.formFields(apiKey);
|
|
|
|
const { iterable, boundary, totalSize } = createUploadBody(filePath, formFields, onProgress, throttle, signal);
|
|
|
|
const { body, statusCode, headers } = await request(targetUrl, {
|
|
method: 'POST',
|
|
body: iterable,
|
|
signal,
|
|
headers: {
|
|
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
|
'Content-Length': String(totalSize),
|
|
'Accept': 'application/json, text/plain;q=0.9, */*;q=0.8',
|
|
'User-Agent': 'multi-hoster-uploader/1.1'
|
|
},
|
|
headersTimeout: UPLOAD_TIMEOUT,
|
|
bodyTimeout: UPLOAD_TIMEOUT
|
|
});
|
|
|
|
const rawBody = await body.text();
|
|
let payload = null;
|
|
try {
|
|
payload = rawBody ? JSON.parse(rawBody) : {};
|
|
} catch {
|
|
const snippet = rawBody ? rawBody.slice(0, 240).replace(/\s+/g, ' ').trim() : '';
|
|
throw new Error(
|
|
`Upload-Antwort von ${hosterName} war kein JSON (HTTP ${statusCode}${snippet ? `): ${snippet}` : ')'}`
|
|
);
|
|
}
|
|
|
|
if (statusCode < 200 || statusCode >= 300) {
|
|
throw new Error(
|
|
payload.msg
|
|
|| payload.message
|
|
|| `Upload fehlgeschlagen (HTTP ${statusCode}${headers?.['content-type'] ? `, ${headers['content-type']}` : ''})`
|
|
);
|
|
}
|
|
|
|
if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
|
|
throw new Error(payload.msg || payload.message || JSON.stringify(payload));
|
|
}
|
|
|
|
let result = null;
|
|
let parseErr = null;
|
|
try {
|
|
result = config.parseResult(payload);
|
|
} catch (err) {
|
|
parseErr = err;
|
|
}
|
|
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
|
return result;
|
|
}
|
|
|
|
// Byse-specific async handling: server accepts the file but responds with
|
|
// filecode="" + misleading status ("Not video file format"). The file shows
|
|
// up in the account shortly after — poll the list to claim it. User observed
|
|
// this with 2+ GB MKV uploads that appeared as "OK" on the byse dashboard
|
|
// even after our uploader gave up.
|
|
if (hosterName === 'byse.sx' && byseBaseline) {
|
|
const fileName = path.basename(filePath);
|
|
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal);
|
|
if (polled) return polled;
|
|
}
|
|
|
|
if (parseErr) throw parseErr;
|
|
|
|
if (payload.success === false) {
|
|
throw new Error(payload.msg || payload.message || `Upload zu ${hosterName} wurde vom Server abgelehnt.`);
|
|
}
|
|
|
|
// Avoid throwing a bare "OK" / "SUCCESS" as the error message — that happens
|
|
// when the server says "msg: OK" but ships no file_code anywhere we know
|
|
// about, typically an API change. Surface the full (trimmed) payload so
|
|
// future logs actually show what the server returned.
|
|
const msg = String(payload.msg || payload.message || '').trim();
|
|
const isOkishNoPayload = /^(ok|success|done|accepted)$/i.test(msg);
|
|
if (isOkishNoPayload || !msg) {
|
|
const snippet = JSON.stringify(payload).slice(0, 400);
|
|
throw new Error(
|
|
`Upload zu ${hosterName} lieferte keine file_code-Antwort (Payload: ${snippet})`
|
|
);
|
|
}
|
|
throw new Error(msg);
|
|
}
|
|
|
|
module.exports = {
|
|
uploadFile,
|
|
HOSTER_CONFIGS,
|
|
__test: {
|
|
extractUploadServerUrl,
|
|
parseVoeResult
|
|
}
|
|
};
|