Administrator/Multi-Hoster-Upload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 47 Wiki Activity

Harden same-name hoster recovery

Browse Source 
Serialize recovery windows per hoster account and normalized filename, reserve remote codes across the batch, and keep cancellation responsive while waiting. Reject semantic and malformed file-list baselines instead of treating them as empty accounts, with focused regression coverage and release allowlist updates.
This commit is contained in:
Sucukdeluxe
2026-08-13 21:22:52 +02:00
parent 107dcfb9ae
commit 82e74a18c6
5 changed files with 525 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/hosters.js
+126 -12
View File
@@ -523,12 +523,17 @@ async function _requestFileList(url, signal, phase, hosterName) {
}
const apiStatus = Number(data && data.status);
if (apiStatus >= 400) {
const statusText = typeof data.status === 'string' ? data.status.trim().toLowerCase() : '';
const semanticFailure = data.success === false
|| data.ok === false
|| data.status === false
|| /^(?:error|failed|failure|denied|invalid|rejected)$/.test(statusText);
if (apiStatus >= 400 || semanticFailure) {
const retryable = apiStatus === 429 || apiStatus >= 500;
throw createTransportError(`${hosterName}: Dateiliste wurde abgelehnt`, {
phase,
endpoint: url,
httpStatus: apiStatus,
httpStatus: apiStatus >= 100 ? apiStatus : response.statusCode,
contentType,
body: text,
retryable,
@@ -539,6 +544,18 @@ async function _requestFileList(url, signal, phase, hosterName) {
return data;
}
function _requireFileList(data, candidates, phase, hosterName, url) {
for (const candidate of candidates) {
if (Array.isArray(candidate)) return candidate;
}
throw createTransportError(`${hosterName}: Dateiliste hatte ein ungültiges Format`, {
phase,
endpoint: url,
httpStatus: 200,
contentType: 'application/json'
});
}
async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
// Byse's file-list endpoint. Returns up to 100 most-recent files — enough
// to match the upload we just did against what the server has. The API
@@ -546,9 +563,11 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
// { status, msg, files: [...] }.
const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`;
const data = await _requestFileList(url, signal, phase, 'Byse');
const src = Array.isArray(data.files) ? data.files
: (data.result && Array.isArray(data.result.files) ? data.result.files
: (Array.isArray(data.result) ? data.result : []));
const src = _requireFileList(data, [
data.files,
data.result && data.result.files,
data.result
], phase, 'Byse', url);
return src.map(f => ({
file_code: String(f.file_code || f.filecode || '').trim(),
file_name: String(f.title || f.name || f.file_name || '').trim()
@@ -559,7 +578,82 @@ function _normalizeFileTitle(s) {
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
}
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) {
function _createAbortError() {
const error = new Error('Operation aborted');
error.name = 'AbortError';
return error;
}
function _waitForRecoveryTurn(predecessor, signal) {
if (!signal) return predecessor;
if (signal.aborted) return Promise.reject(_createAbortError());
return new Promise((resolve, reject) => {
const onAbort = () => {
signal.removeEventListener('abort', onAbort);
reject(_createAbortError());
};
signal.addEventListener('abort', onAbort, { once: true });
predecessor.then(
() => {
signal.removeEventListener('abort', onAbort);
resolve();
},
error => {
signal.removeEventListener('abort', onAbort);
reject(error);
}
);
});
}
function createRecoveryClaimRegistry() {
const scopes = new Map();
return {
forUpload(hosterName, apiKey, fileName) {
const identity = crypto.createHash('sha256')
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`)
.digest('hex');
let codes = scopes.get(identity);
if (!codes) {
codes = {
values: new Set(),
tail: Promise.resolve()
};
scopes.set(identity, codes);
}
return {
has(code) {
return codes.values.has(String(code || '').trim());
},
reserve(code) {
const normalized = String(code || '').trim();
if (!normalized || codes.values.has(normalized)) return false;
codes.values.add(normalized);
return true;
},
async runExclusive(operation, signal) {
const predecessor = codes.tail;
let release;
const current = new Promise(resolve => {
release = resolve;
});
codes.tail = predecessor.then(() => current);
try {
await _waitForRecoveryTurn(predecessor, signal);
return await operation();
} finally {
release();
}
}
};
},
clear() {
scopes.clear();
}
};
}
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
if (!(baselineCodes instanceof Set)) return null;
const expected = _normalizeFileTitle(fileName);
const POLL_ATTEMPTS = 15;
@@ -573,10 +667,13 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal)
// poller could claim job B's newly appeared file and return the wrong
// URL. At the cost of a few false-negatives when byse mangles the
// filename beyond our normalizer, correctness for parallel uploads wins.
const matches = newFiles.filter(f => _normalizeFileTitle(f.file_name) === expected);
const matches = newFiles
.filter(f => _normalizeFileTitle(f.file_name) === expected)
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
if (matches.length > 1) return null;
if (matches.length === 1) {
const match = matches[0];
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
return {
download_url: `https://byse.sx/d/${match.file_code}`,
embed_url: `https://byse.sx/e/${match.file_code}`,
@@ -601,7 +698,7 @@ async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll')
// recovery only needs the most recent uploads, so page 1 newest-first suffices.
const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`;
const data = await _requestFileList(url, signal, phase, 'Doodstream');
const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : [];
const files = _requireFileList(data, [data && data.result && data.result.files], phase, 'Doodstream', url);
return files.map(f => ({
file_code: String(f.file_code || f.filecode || '').trim(),
file_name: String(f.title || f.file_name || f.name || '').trim()
@@ -610,7 +707,7 @@ async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll')
const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal) {
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
if (!(baselineCodes instanceof Set)) return null;
// Same recovery byse uses: the upload POST returned no filecode, but the file
// may register in the account a little later. Poll the list for a NEW file
@@ -624,10 +721,13 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s
if (signal && signal.aborted) return null;
const list = await _fetchDoodstreamFileList(apiKey, signal);
const fresh = list.filter(f => !baselineCodes.has(f.file_code));
const matches = fresh.filter(f => _normalizeFileTitle(f.file_name) === expected);
const matches = fresh
.filter(f => _normalizeFileTitle(f.file_name) === expected)
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
if (matches.length > 1) return null;
if (matches.length === 1) {
const match = matches[0];
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
return {
download_url: `https://doodstream.com/d/${match.file_code}`,
embed_url: `https://doodstream.com/e/${match.file_code}`,
@@ -783,6 +883,19 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
parseErr = err;
}
if (result && (result.file_code || result.download_url || result.embed_url)) {
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
if (!opts.recoveryClaim.reserve(result.file_code)) {
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
phase: 'upload-result',
endpoint: targetUrl,
httpStatus: statusCode,
contentType: headers && headers['content-type'],
body: rawBody,
retryable: true,
hosterTransient: true
});
}
}
return result;
}
@@ -808,7 +921,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// even after our uploader gave up.
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
const fileName = path.basename(filePath);
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal);
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
}
@@ -817,7 +930,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// the file did register, claim its code instead of failing the upload.
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
const fileName = path.basename(filePath);
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal);
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
}
@@ -893,6 +1006,7 @@ async function prefetchBaseline(hosterName, apiKey, signal) {
module.exports = {
uploadFile,
prefetchBaseline,
createRecoveryClaimRegistry,
HOSTER_CONFIGS,
__test: {
extractUploadServerUrl,
lib/upload-manager.js
+22 -10
View File
@@ -3,7 +3,7 @@ const path = require('path');
const { assertUploadConfirmation } = require('./upload-confirmation');
const fs = require('fs');
const crypto = require('crypto');
const { uploadFile, prefetchBaseline } = require('./hosters');
const { uploadFile, prefetchBaseline, createRecoveryClaimRegistry } = require('./hosters');
const VidmolyUploader = require('./vidmoly-upload');
const VoeUploader = require('./voe-upload');
const DoodstreamUploader = require('./doodstream-upload');
@@ -52,6 +52,7 @@ class UploadManager extends EventEmitter {
this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file
this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
this._recoveryClaims = createRecoveryClaimRegistry();
}
updateAccountPools(accountPools) {
@@ -68,6 +69,7 @@ class UploadManager extends EventEmitter {
this._suspectGoodAccounts.clear();
this._doodApiKeyCache.clear();
this._baselineCache.clear();
this._recoveryClaims.clear();
}
switchAccount(hoster, fallbackAccount) {
@@ -348,6 +350,7 @@ class UploadManager extends EventEmitter {
for (const jobId of pendingCancelledJobIds) this.cancelledJobIds.add(jobId);
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
this._recoveryClaims.clear();
this.semaphores = {};
this.globalSemaphore = null;
this.globalThrottle = null;
@@ -1258,9 +1261,7 @@ class UploadManager extends EventEmitter {
const apiKey = await this._resolveDoodstreamApiKey(task);
if (apiKey) {
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle, {
doodBaseline: await this._getBaseline('doodstream.com', apiKey, signal)
});
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe);
}
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
const dood = new DoodstreamUploader();
@@ -1270,16 +1271,27 @@ class UploadManager extends EventEmitter {
const clouddrop = new ClouddropUploader(task.apiKey);
return clouddrop.upload(task.file, progressCb, signal, throttle);
} else {
const baselineOpts = {};
if (task.hoster === 'byse.sx') {
baselineOpts.byseBaseline = await this._getBaseline('byse.sx', task.apiKey, signal);
if (fileProbe && fileProbe.ok !== false) baselineOpts.probeIsVideoLike = fileProbe.isVideoLike === true;
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe);
}
if (task.hoster === 'doodstream.com') baselineOpts.doodBaseline = await this._getBaseline('doodstream.com', task.apiKey, signal);
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, baselineOpts);
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
}
}
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
return recoveryClaim.runExclusive(async () => {
const options = { recoveryClaim };
if (hosterName === 'byse.sx') {
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
} else {
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
}
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
}, signal);
}
_getBaseline(hosterName, apiKey, signal) {
if (!apiKey) return Promise.resolve(null);
const key = `${hosterName}:${apiKey}`;