Harden same-name hoster recovery
Serialize recovery windows per hoster account and normalized filename, reserve remote codes across the batch, and keep cancellation responsive while waiting. Reject semantic and malformed file-list baselines instead of treating them as empty accounts, with focused regression coverage and release allowlist updates.
This commit is contained in:
+126
-12
@@ -523,12 +523,17 @@ async function _requestFileList(url, signal, phase, hosterName) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const apiStatus = Number(data && data.status);
|
const apiStatus = Number(data && data.status);
|
||||||
if (apiStatus >= 400) {
|
const statusText = typeof data.status === 'string' ? data.status.trim().toLowerCase() : '';
|
||||||
|
const semanticFailure = data.success === false
|
||||||
|
|| data.ok === false
|
||||||
|
|| data.status === false
|
||||||
|
|| /^(?:error|failed|failure|denied|invalid|rejected)$/.test(statusText);
|
||||||
|
if (apiStatus >= 400 || semanticFailure) {
|
||||||
const retryable = apiStatus === 429 || apiStatus >= 500;
|
const retryable = apiStatus === 429 || apiStatus >= 500;
|
||||||
throw createTransportError(`${hosterName}: Dateiliste wurde abgelehnt`, {
|
throw createTransportError(`${hosterName}: Dateiliste wurde abgelehnt`, {
|
||||||
phase,
|
phase,
|
||||||
endpoint: url,
|
endpoint: url,
|
||||||
httpStatus: apiStatus,
|
httpStatus: apiStatus >= 100 ? apiStatus : response.statusCode,
|
||||||
contentType,
|
contentType,
|
||||||
body: text,
|
body: text,
|
||||||
retryable,
|
retryable,
|
||||||
@@ -539,6 +544,18 @@ async function _requestFileList(url, signal, phase, hosterName) {
|
|||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function _requireFileList(data, candidates, phase, hosterName, url) {
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (Array.isArray(candidate)) return candidate;
|
||||||
|
}
|
||||||
|
throw createTransportError(`${hosterName}: Dateiliste hatte ein ungültiges Format`, {
|
||||||
|
phase,
|
||||||
|
endpoint: url,
|
||||||
|
httpStatus: 200,
|
||||||
|
contentType: 'application/json'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
||||||
// Byse's file-list endpoint. Returns up to 100 most-recent files — enough
|
// Byse's file-list endpoint. Returns up to 100 most-recent files — enough
|
||||||
// to match the upload we just did against what the server has. The API
|
// to match the upload we just did against what the server has. The API
|
||||||
@@ -546,9 +563,11 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
|||||||
// { status, msg, files: [...] }.
|
// { status, msg, files: [...] }.
|
||||||
const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`;
|
const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`;
|
||||||
const data = await _requestFileList(url, signal, phase, 'Byse');
|
const data = await _requestFileList(url, signal, phase, 'Byse');
|
||||||
const src = Array.isArray(data.files) ? data.files
|
const src = _requireFileList(data, [
|
||||||
: (data.result && Array.isArray(data.result.files) ? data.result.files
|
data.files,
|
||||||
: (Array.isArray(data.result) ? data.result : []));
|
data.result && data.result.files,
|
||||||
|
data.result
|
||||||
|
], phase, 'Byse', url);
|
||||||
return src.map(f => ({
|
return src.map(f => ({
|
||||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||||
file_name: String(f.title || f.name || f.file_name || '').trim()
|
file_name: String(f.title || f.name || f.file_name || '').trim()
|
||||||
@@ -559,7 +578,82 @@ function _normalizeFileTitle(s) {
|
|||||||
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) {
|
function _createAbortError() {
|
||||||
|
const error = new Error('Operation aborted');
|
||||||
|
error.name = 'AbortError';
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function _waitForRecoveryTurn(predecessor, signal) {
|
||||||
|
if (!signal) return predecessor;
|
||||||
|
if (signal.aborted) return Promise.reject(_createAbortError());
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const onAbort = () => {
|
||||||
|
signal.removeEventListener('abort', onAbort);
|
||||||
|
reject(_createAbortError());
|
||||||
|
};
|
||||||
|
signal.addEventListener('abort', onAbort, { once: true });
|
||||||
|
predecessor.then(
|
||||||
|
() => {
|
||||||
|
signal.removeEventListener('abort', onAbort);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
error => {
|
||||||
|
signal.removeEventListener('abort', onAbort);
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRecoveryClaimRegistry() {
|
||||||
|
const scopes = new Map();
|
||||||
|
return {
|
||||||
|
forUpload(hosterName, apiKey, fileName) {
|
||||||
|
const identity = crypto.createHash('sha256')
|
||||||
|
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`)
|
||||||
|
.digest('hex');
|
||||||
|
let codes = scopes.get(identity);
|
||||||
|
if (!codes) {
|
||||||
|
codes = {
|
||||||
|
values: new Set(),
|
||||||
|
tail: Promise.resolve()
|
||||||
|
};
|
||||||
|
scopes.set(identity, codes);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
has(code) {
|
||||||
|
return codes.values.has(String(code || '').trim());
|
||||||
|
},
|
||||||
|
reserve(code) {
|
||||||
|
const normalized = String(code || '').trim();
|
||||||
|
if (!normalized || codes.values.has(normalized)) return false;
|
||||||
|
codes.values.add(normalized);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
async runExclusive(operation, signal) {
|
||||||
|
const predecessor = codes.tail;
|
||||||
|
let release;
|
||||||
|
const current = new Promise(resolve => {
|
||||||
|
release = resolve;
|
||||||
|
});
|
||||||
|
codes.tail = predecessor.then(() => current);
|
||||||
|
try {
|
||||||
|
await _waitForRecoveryTurn(predecessor, signal);
|
||||||
|
return await operation();
|
||||||
|
} finally {
|
||||||
|
release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
},
|
||||||
|
clear() {
|
||||||
|
scopes.clear();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
|
||||||
if (!(baselineCodes instanceof Set)) return null;
|
if (!(baselineCodes instanceof Set)) return null;
|
||||||
const expected = _normalizeFileTitle(fileName);
|
const expected = _normalizeFileTitle(fileName);
|
||||||
const POLL_ATTEMPTS = 15;
|
const POLL_ATTEMPTS = 15;
|
||||||
@@ -573,10 +667,13 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal)
|
|||||||
// poller could claim job B's newly appeared file and return the wrong
|
// poller could claim job B's newly appeared file and return the wrong
|
||||||
// URL. At the cost of a few false-negatives when byse mangles the
|
// URL. At the cost of a few false-negatives when byse mangles the
|
||||||
// filename beyond our normalizer, correctness for parallel uploads wins.
|
// filename beyond our normalizer, correctness for parallel uploads wins.
|
||||||
const matches = newFiles.filter(f => _normalizeFileTitle(f.file_name) === expected);
|
const matches = newFiles
|
||||||
|
.filter(f => _normalizeFileTitle(f.file_name) === expected)
|
||||||
|
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
|
||||||
if (matches.length > 1) return null;
|
if (matches.length > 1) return null;
|
||||||
if (matches.length === 1) {
|
if (matches.length === 1) {
|
||||||
const match = matches[0];
|
const match = matches[0];
|
||||||
|
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
|
||||||
return {
|
return {
|
||||||
download_url: `https://byse.sx/d/${match.file_code}`,
|
download_url: `https://byse.sx/d/${match.file_code}`,
|
||||||
embed_url: `https://byse.sx/e/${match.file_code}`,
|
embed_url: `https://byse.sx/e/${match.file_code}`,
|
||||||
@@ -601,7 +698,7 @@ async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll')
|
|||||||
// recovery only needs the most recent uploads, so page 1 newest-first suffices.
|
// recovery only needs the most recent uploads, so page 1 newest-first suffices.
|
||||||
const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`;
|
const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`;
|
||||||
const data = await _requestFileList(url, signal, phase, 'Doodstream');
|
const data = await _requestFileList(url, signal, phase, 'Doodstream');
|
||||||
const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : [];
|
const files = _requireFileList(data, [data && data.result && data.result.files], phase, 'Doodstream', url);
|
||||||
return files.map(f => ({
|
return files.map(f => ({
|
||||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||||
file_name: String(f.title || f.file_name || f.name || '').trim()
|
file_name: String(f.title || f.file_name || f.name || '').trim()
|
||||||
@@ -610,7 +707,7 @@ async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll')
|
|||||||
|
|
||||||
const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test
|
const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test
|
||||||
|
|
||||||
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal) {
|
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
|
||||||
if (!(baselineCodes instanceof Set)) return null;
|
if (!(baselineCodes instanceof Set)) return null;
|
||||||
// Same recovery byse uses: the upload POST returned no filecode, but the file
|
// Same recovery byse uses: the upload POST returned no filecode, but the file
|
||||||
// may register in the account a little later. Poll the list for a NEW file
|
// may register in the account a little later. Poll the list for a NEW file
|
||||||
@@ -624,10 +721,13 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s
|
|||||||
if (signal && signal.aborted) return null;
|
if (signal && signal.aborted) return null;
|
||||||
const list = await _fetchDoodstreamFileList(apiKey, signal);
|
const list = await _fetchDoodstreamFileList(apiKey, signal);
|
||||||
const fresh = list.filter(f => !baselineCodes.has(f.file_code));
|
const fresh = list.filter(f => !baselineCodes.has(f.file_code));
|
||||||
const matches = fresh.filter(f => _normalizeFileTitle(f.file_name) === expected);
|
const matches = fresh
|
||||||
|
.filter(f => _normalizeFileTitle(f.file_name) === expected)
|
||||||
|
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
|
||||||
if (matches.length > 1) return null;
|
if (matches.length > 1) return null;
|
||||||
if (matches.length === 1) {
|
if (matches.length === 1) {
|
||||||
const match = matches[0];
|
const match = matches[0];
|
||||||
|
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
|
||||||
return {
|
return {
|
||||||
download_url: `https://doodstream.com/d/${match.file_code}`,
|
download_url: `https://doodstream.com/d/${match.file_code}`,
|
||||||
embed_url: `https://doodstream.com/e/${match.file_code}`,
|
embed_url: `https://doodstream.com/e/${match.file_code}`,
|
||||||
@@ -783,6 +883,19 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
parseErr = err;
|
parseErr = err;
|
||||||
}
|
}
|
||||||
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
||||||
|
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
|
||||||
|
if (!opts.recoveryClaim.reserve(result.file_code)) {
|
||||||
|
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
||||||
|
phase: 'upload-result',
|
||||||
|
endpoint: targetUrl,
|
||||||
|
httpStatus: statusCode,
|
||||||
|
contentType: headers && headers['content-type'],
|
||||||
|
body: rawBody,
|
||||||
|
retryable: true,
|
||||||
|
hosterTransient: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -808,7 +921,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
// even after our uploader gave up.
|
// even after our uploader gave up.
|
||||||
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
||||||
const fileName = path.basename(filePath);
|
const fileName = path.basename(filePath);
|
||||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal);
|
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
||||||
if (polled) return polled;
|
if (polled) return polled;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -817,7 +930,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
// the file did register, claim its code instead of failing the upload.
|
// the file did register, claim its code instead of failing the upload.
|
||||||
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
||||||
const fileName = path.basename(filePath);
|
const fileName = path.basename(filePath);
|
||||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal);
|
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
||||||
if (polled) return polled;
|
if (polled) return polled;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -893,6 +1006,7 @@ async function prefetchBaseline(hosterName, apiKey, signal) {
|
|||||||
module.exports = {
|
module.exports = {
|
||||||
uploadFile,
|
uploadFile,
|
||||||
prefetchBaseline,
|
prefetchBaseline,
|
||||||
|
createRecoveryClaimRegistry,
|
||||||
HOSTER_CONFIGS,
|
HOSTER_CONFIGS,
|
||||||
__test: {
|
__test: {
|
||||||
extractUploadServerUrl,
|
extractUploadServerUrl,
|
||||||
|
|||||||
+22
-10
@@ -3,7 +3,7 @@ const path = require('path');
|
|||||||
const { assertUploadConfirmation } = require('./upload-confirmation');
|
const { assertUploadConfirmation } = require('./upload-confirmation');
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const { uploadFile, prefetchBaseline } = require('./hosters');
|
const { uploadFile, prefetchBaseline, createRecoveryClaimRegistry } = require('./hosters');
|
||||||
const VidmolyUploader = require('./vidmoly-upload');
|
const VidmolyUploader = require('./vidmoly-upload');
|
||||||
const VoeUploader = require('./voe-upload');
|
const VoeUploader = require('./voe-upload');
|
||||||
const DoodstreamUploader = require('./doodstream-upload');
|
const DoodstreamUploader = require('./doodstream-upload');
|
||||||
@@ -52,6 +52,7 @@ class UploadManager extends EventEmitter {
|
|||||||
this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file
|
this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file
|
||||||
this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
|
this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
|
||||||
this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
|
this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
|
||||||
|
this._recoveryClaims = createRecoveryClaimRegistry();
|
||||||
}
|
}
|
||||||
|
|
||||||
updateAccountPools(accountPools) {
|
updateAccountPools(accountPools) {
|
||||||
@@ -68,6 +69,7 @@ class UploadManager extends EventEmitter {
|
|||||||
this._suspectGoodAccounts.clear();
|
this._suspectGoodAccounts.clear();
|
||||||
this._doodApiKeyCache.clear();
|
this._doodApiKeyCache.clear();
|
||||||
this._baselineCache.clear();
|
this._baselineCache.clear();
|
||||||
|
this._recoveryClaims.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
switchAccount(hoster, fallbackAccount) {
|
switchAccount(hoster, fallbackAccount) {
|
||||||
@@ -348,6 +350,7 @@ class UploadManager extends EventEmitter {
|
|||||||
for (const jobId of pendingCancelledJobIds) this.cancelledJobIds.add(jobId);
|
for (const jobId of pendingCancelledJobIds) this.cancelledJobIds.add(jobId);
|
||||||
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
||||||
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
||||||
|
this._recoveryClaims.clear();
|
||||||
this.semaphores = {};
|
this.semaphores = {};
|
||||||
this.globalSemaphore = null;
|
this.globalSemaphore = null;
|
||||||
this.globalThrottle = null;
|
this.globalThrottle = null;
|
||||||
@@ -1258,9 +1261,7 @@ class UploadManager extends EventEmitter {
|
|||||||
const apiKey = await this._resolveDoodstreamApiKey(task);
|
const apiKey = await this._resolveDoodstreamApiKey(task);
|
||||||
if (apiKey) {
|
if (apiKey) {
|
||||||
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
|
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||||
return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle, {
|
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe);
|
||||||
doodBaseline: await this._getBaseline('doodstream.com', apiKey, signal)
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
|
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||||
const dood = new DoodstreamUploader();
|
const dood = new DoodstreamUploader();
|
||||||
@@ -1270,16 +1271,27 @@ class UploadManager extends EventEmitter {
|
|||||||
const clouddrop = new ClouddropUploader(task.apiKey);
|
const clouddrop = new ClouddropUploader(task.apiKey);
|
||||||
return clouddrop.upload(task.file, progressCb, signal, throttle);
|
return clouddrop.upload(task.file, progressCb, signal, throttle);
|
||||||
} else {
|
} else {
|
||||||
const baselineOpts = {};
|
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
||||||
if (task.hoster === 'byse.sx') {
|
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe);
|
||||||
baselineOpts.byseBaseline = await this._getBaseline('byse.sx', task.apiKey, signal);
|
|
||||||
if (fileProbe && fileProbe.ok !== false) baselineOpts.probeIsVideoLike = fileProbe.isVideoLike === true;
|
|
||||||
}
|
}
|
||||||
if (task.hoster === 'doodstream.com') baselineOpts.doodBaseline = await this._getBaseline('doodstream.com', task.apiKey, signal);
|
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
|
||||||
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, baselineOpts);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
|
||||||
|
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
|
||||||
|
return recoveryClaim.runExclusive(async () => {
|
||||||
|
const options = { recoveryClaim };
|
||||||
|
if (hosterName === 'byse.sx') {
|
||||||
|
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||||
|
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
|
||||||
|
} else {
|
||||||
|
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||||
|
}
|
||||||
|
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
|
||||||
|
}, signal);
|
||||||
|
}
|
||||||
|
|
||||||
_getBaseline(hosterName, apiKey, signal) {
|
_getBaseline(hosterName, apiKey, signal) {
|
||||||
if (!apiKey) return Promise.resolve(null);
|
if (!apiKey) return Promise.resolve(null);
|
||||||
const key = `${hosterName}:${apiKey}`;
|
const key = `${hosterName}:${apiKey}`;
|
||||||
|
|||||||
@@ -116,6 +116,7 @@ const sourceFiles = [
|
|||||||
'tests/history-retention.test.js',
|
'tests/history-retention.test.js',
|
||||||
'tests/hosters.test.js',
|
'tests/hosters.test.js',
|
||||||
'tests/hoster-recovery-provenance.test.js',
|
'tests/hoster-recovery-provenance.test.js',
|
||||||
|
'tests/hoster-recovery-safety.test.js',
|
||||||
'tests/i18n.test.js',
|
'tests/i18n.test.js',
|
||||||
'tests/ip-allowlist.test.js',
|
'tests/ip-allowlist.test.js',
|
||||||
'tests/log-mode.test.js',
|
'tests/log-mode.test.js',
|
||||||
@@ -158,6 +159,7 @@ const sourceFiles = [
|
|||||||
'tests/upload-confirmation.test.js',
|
'tests/upload-confirmation.test.js',
|
||||||
'tests/upload-diagnostics.test.js',
|
'tests/upload-diagnostics.test.js',
|
||||||
'tests/upload-manager.test.js',
|
'tests/upload-manager.test.js',
|
||||||
|
'tests/upload-manager-recovery-claims.test.js',
|
||||||
'tests/upload-recovery.test.js',
|
'tests/upload-recovery.test.js',
|
||||||
'tests/upload-start-reservation.test.js',
|
'tests/upload-start-reservation.test.js',
|
||||||
'tests/session-report.test.js',
|
'tests/session-report.test.js',
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
const { after, before, test } = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const os = require('node:os');
|
||||||
|
const path = require('node:path');
|
||||||
|
|
||||||
|
let requestRouter = async () => ({ statusCode: 200, headers: {}, body: { text: async () => '{}' } });
|
||||||
|
const undici = require('undici');
|
||||||
|
const originalRequest = undici.request;
|
||||||
|
undici.request = (...args) => requestRouter(...args);
|
||||||
|
delete require.cache[require.resolve('../lib/hosters')];
|
||||||
|
const hosters = require('../lib/hosters');
|
||||||
|
|
||||||
|
let tempRoot;
|
||||||
|
let uploadPath;
|
||||||
|
let originalFetch;
|
||||||
|
|
||||||
|
before(() => {
|
||||||
|
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mhu-recovery-safety-'));
|
||||||
|
uploadPath = path.join(tempRoot, 'Shared Episode.mkv');
|
||||||
|
fs.writeFileSync(uploadPath, Buffer.alloc(2048, 7));
|
||||||
|
originalFetch = global.fetch;
|
||||||
|
hosters.__test.DOODSTREAM_POLL.attempts = 1;
|
||||||
|
hosters.__test.DOODSTREAM_POLL.delayMs = 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
after(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
undici.request = originalRequest;
|
||||||
|
delete require.cache[require.resolve('../lib/hosters')];
|
||||||
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
function stubUploadServer() {
|
||||||
|
global.fetch = async () => ({
|
||||||
|
status: 200,
|
||||||
|
text: async () => JSON.stringify({ status: 200, result: 'https://node1.cloudatacdn.com/upload/01' })
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function response(body, statusCode = 200) {
|
||||||
|
return {
|
||||||
|
statusCode,
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: { text: async () => typeof body === 'string' ? body : JSON.stringify(body) }
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function drain(body) {
|
||||||
|
if (!body || typeof body[Symbol.asyncIterator] !== 'function') return;
|
||||||
|
for await (const chunk of body) {
|
||||||
|
if (chunk && chunk.length === -1) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createClaim() {
|
||||||
|
const codes = new Set();
|
||||||
|
return {
|
||||||
|
has: (code) => codes.has(String(code)),
|
||||||
|
reserve(code) {
|
||||||
|
const normalized = String(code || '').trim();
|
||||||
|
if (!normalized || codes.has(normalized)) return false;
|
||||||
|
codes.add(normalized);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('parallel same-name recovery cannot reuse a directly confirmed remote file', async () => {
|
||||||
|
stubUploadServer();
|
||||||
|
const recoveryClaim = createClaim();
|
||||||
|
let uploadCalls = 0;
|
||||||
|
requestRouter = async (url, options) => {
|
||||||
|
if (/\/api\/file\/list/.test(String(url))) {
|
||||||
|
return response({
|
||||||
|
status: 200,
|
||||||
|
result: { files: [{ file_code: 'SHARED_REMOTE_CODE', title: path.basename(uploadPath) }] }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
await drain(options && options.body);
|
||||||
|
uploadCalls++;
|
||||||
|
if (uploadCalls === 1) {
|
||||||
|
return response({
|
||||||
|
status: 200,
|
||||||
|
result: [{ filecode: 'SHARED_REMOTE_CODE', download_url: 'https://doodstream.com/d/SHARED_REMOTE_CODE' }]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return response({ status: 200, msg: 'OK' });
|
||||||
|
};
|
||||||
|
|
||||||
|
const results = await Promise.allSettled([
|
||||||
|
hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null, {
|
||||||
|
doodBaseline: new Set(),
|
||||||
|
recoveryClaim
|
||||||
|
}),
|
||||||
|
hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null, {
|
||||||
|
doodBaseline: new Set(),
|
||||||
|
recoveryClaim
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(results.filter(result => result.status === 'fulfilled').length, 1);
|
||||||
|
assert.equal(results.filter(result => result.status === 'rejected').length, 1);
|
||||||
|
assert.equal(results.find(result => result.status === 'fulfilled').value.file_code, 'SHARED_REMOTE_CODE');
|
||||||
|
assert.equal(results.find(result => result.status === 'rejected').reason.hosterTransient, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a direct response rejects a remote code already reserved in its recovery scope', async () => {
|
||||||
|
stubUploadServer();
|
||||||
|
const recoveryClaim = createClaim();
|
||||||
|
recoveryClaim.reserve('ALREADY_RESERVED_CODE');
|
||||||
|
requestRouter = async (url, options) => {
|
||||||
|
if (/\/api\/file\/list/.test(String(url))) {
|
||||||
|
return response({ status: 200, result: { files: [] } });
|
||||||
|
}
|
||||||
|
await drain(options && options.body);
|
||||||
|
return response({
|
||||||
|
status: 200,
|
||||||
|
result: [{
|
||||||
|
filecode: 'ALREADY_RESERVED_CODE',
|
||||||
|
download_url: 'https://doodstream.com/d/ALREADY_RESERVED_CODE'
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
() => hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null, {
|
||||||
|
doodBaseline: new Set(),
|
||||||
|
recoveryClaim
|
||||||
|
}),
|
||||||
|
error => {
|
||||||
|
assert.equal(error.hosterTransient, true);
|
||||||
|
assert.equal(error.diagnostic.phase, 'upload-result');
|
||||||
|
assert.equal(error.diagnostic.http, 200);
|
||||||
|
assert.doesNotMatch(error.message, /ALREADY_RESERVED_CODE/);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const fixture of [
|
||||||
|
{ name: 'semantic error payload', payload: { status: 'error', msg: 'invalid key' } },
|
||||||
|
{ name: 'missing files list', payload: { status: 200, result: {} } }
|
||||||
|
]) {
|
||||||
|
test(`doodstream rejects a ${fixture.name} as a recovery baseline`, async () => {
|
||||||
|
stubUploadServer();
|
||||||
|
let listCalls = 0;
|
||||||
|
requestRouter = async (url, options) => {
|
||||||
|
if (/\/api\/file\/list/.test(String(url))) {
|
||||||
|
listCalls++;
|
||||||
|
return response(fixture.payload);
|
||||||
|
}
|
||||||
|
await drain(options && options.body);
|
||||||
|
return response({ status: 200, msg: 'OK' });
|
||||||
|
};
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
() => hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null),
|
||||||
|
error => {
|
||||||
|
assert.equal(error.diagnostic.phase, 'recovery-baseline');
|
||||||
|
assert.equal(error.diagnostic.http, 200);
|
||||||
|
assert.equal(error.hosterTransient, true);
|
||||||
|
assert.doesNotMatch(error.message, /invalid key/i);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.equal(listCalls, 1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test('byse rejects a missing files list as a recovery baseline', async () => {
|
||||||
|
global.fetch = async () => ({
|
||||||
|
status: 200,
|
||||||
|
text: async () => JSON.stringify({ status: 200, result: 'https://byse-upload.invalid/upload/01' })
|
||||||
|
});
|
||||||
|
let listCalls = 0;
|
||||||
|
requestRouter = async (url, options) => {
|
||||||
|
if (/\/file\/list/.test(String(url))) {
|
||||||
|
listCalls++;
|
||||||
|
return response({ status: 200, result: {} });
|
||||||
|
}
|
||||||
|
await drain(options && options.body);
|
||||||
|
return response({ status: 200, msg: 'OK' });
|
||||||
|
};
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
() => hosters.uploadFile('byse.sx', uploadPath, 'VALIDKEY', null, null, null),
|
||||||
|
error => {
|
||||||
|
assert.equal(error.diagnostic.phase, 'recovery-baseline');
|
||||||
|
assert.equal(error.diagnostic.http, 200);
|
||||||
|
assert.equal(error.hosterTransient, true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.equal(listCalls, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a cancelled recovery lock waiter exits before the active lease finishes', async () => {
|
||||||
|
const registry = hosters.createRecoveryClaimRegistry();
|
||||||
|
const claim = registry.forUpload('doodstream.com', 'ACCOUNT_KEY', 'Shared Episode.mkv');
|
||||||
|
let releaseFirst;
|
||||||
|
const firstBlocked = new Promise(resolve => {
|
||||||
|
releaseFirst = resolve;
|
||||||
|
});
|
||||||
|
let markFirstEntered;
|
||||||
|
const firstEntered = new Promise(resolve => {
|
||||||
|
markFirstEntered = resolve;
|
||||||
|
});
|
||||||
|
const first = claim.runExclusive(async () => {
|
||||||
|
markFirstEntered();
|
||||||
|
await firstBlocked;
|
||||||
|
});
|
||||||
|
await firstEntered;
|
||||||
|
const abortController = new AbortController();
|
||||||
|
const second = claim.runExclusive(async () => 'unexpected', abortController.signal);
|
||||||
|
abortController.abort();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const outcome = await Promise.race([
|
||||||
|
second.then(value => ({ status: 'fulfilled', value }), error => ({ status: 'rejected', error })),
|
||||||
|
new Promise(resolve => setTimeout(() => resolve({ status: 'timeout' }), 100))
|
||||||
|
]);
|
||||||
|
assert.equal(outcome.status, 'rejected');
|
||||||
|
assert.equal(outcome.error.name, 'AbortError');
|
||||||
|
} finally {
|
||||||
|
releaseFirst();
|
||||||
|
await first;
|
||||||
|
await second.catch(() => {});
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
const { after, before, test } = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const os = require('node:os');
|
||||||
|
const path = require('node:path');
|
||||||
|
|
||||||
|
const hosters = require('../lib/hosters');
|
||||||
|
const originalUploadFile = hosters.uploadFile;
|
||||||
|
const originalPrefetchBaseline = hosters.prefetchBaseline;
|
||||||
|
let tempRoot;
|
||||||
|
let firstPath;
|
||||||
|
let secondPath;
|
||||||
|
let UploadManager;
|
||||||
|
|
||||||
|
before(() => {
|
||||||
|
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mhu-manager-recovery-'));
|
||||||
|
const firstDir = path.join(tempRoot, 'first');
|
||||||
|
const secondDir = path.join(tempRoot, 'second');
|
||||||
|
fs.mkdirSync(firstDir);
|
||||||
|
fs.mkdirSync(secondDir);
|
||||||
|
firstPath = path.join(firstDir, 'Shared Episode.mkv');
|
||||||
|
secondPath = path.join(secondDir, 'shared-episode.mp4');
|
||||||
|
fs.writeFileSync(firstPath, Buffer.alloc(1024, 1));
|
||||||
|
fs.writeFileSync(secondPath, Buffer.alloc(1024, 2));
|
||||||
|
});
|
||||||
|
|
||||||
|
after(() => {
|
||||||
|
hosters.uploadFile = originalUploadFile;
|
||||||
|
hosters.prefetchBaseline = originalPrefetchBaseline;
|
||||||
|
delete require.cache[require.resolve('../lib/upload-manager')];
|
||||||
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
function loadManager(uploadFile) {
|
||||||
|
hosters.uploadFile = uploadFile;
|
||||||
|
hosters.prefetchBaseline = async () => new Set();
|
||||||
|
delete require.cache[require.resolve('../lib/upload-manager')];
|
||||||
|
UploadManager = require('../lib/upload-manager');
|
||||||
|
}
|
||||||
|
|
||||||
|
function settings(parallelCount) {
|
||||||
|
return {
|
||||||
|
'byse.sx': {
|
||||||
|
retries: 0,
|
||||||
|
parallelCount,
|
||||||
|
maxSpeedKbs: 0,
|
||||||
|
restartBelowKbs: 0,
|
||||||
|
timeIntervalSec: 0,
|
||||||
|
maxSizeMb: 0
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runBatch(manager, tasks) {
|
||||||
|
let summary;
|
||||||
|
manager.once('batch-done', value => {
|
||||||
|
summary = value;
|
||||||
|
});
|
||||||
|
await manager.startBatch(tasks);
|
||||||
|
return summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
test('a batch shares recovery claims across normalized same-name jobs', async () => {
|
||||||
|
let unsafeCalls = 0;
|
||||||
|
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||||
|
const claim = options && options.recoveryClaim;
|
||||||
|
if (claim && claim.reserve('SHARED_REMOTE_CODE')) {
|
||||||
|
return {
|
||||||
|
file_code: 'SHARED_REMOTE_CODE',
|
||||||
|
download_url: 'https://byse.sx/d/SHARED_REMOTE_CODE'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (!claim) {
|
||||||
|
unsafeCalls++;
|
||||||
|
return {
|
||||||
|
file_code: `UNSAFE_${unsafeCalls}`,
|
||||||
|
download_url: `https://byse.sx/d/UNSAFE_${unsafeCalls}`
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const error = new Error('Remote recovery candidate already claimed');
|
||||||
|
error.hosterTransient = true;
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
const manager = new UploadManager(settings(2));
|
||||||
|
|
||||||
|
const summary = await runBatch(manager, [
|
||||||
|
{ jobId: 'same-name-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
|
||||||
|
{ jobId: 'same-name-b', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(summary.succeeded, 1);
|
||||||
|
assert.equal(summary.failed, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('recovery claims stay isolated between accounts', async () => {
|
||||||
|
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||||
|
if (!options || !options.recoveryClaim) {
|
||||||
|
throw new Error('Missing recovery claim');
|
||||||
|
}
|
||||||
|
if (!options.recoveryClaim.reserve('SHARED_REMOTE_CODE')) {
|
||||||
|
const error = new Error('Remote recovery candidate already claimed');
|
||||||
|
error.hosterTransient = true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
file_code: 'SHARED_REMOTE_CODE',
|
||||||
|
download_url: 'https://byse.sx/d/SHARED_REMOTE_CODE'
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const manager = new UploadManager(settings(2));
|
||||||
|
|
||||||
|
const summary = await runBatch(manager, [
|
||||||
|
{ jobId: 'account-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_A' },
|
||||||
|
{ jobId: 'account-b', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_B' }
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(summary.succeeded, 2);
|
||||||
|
assert.equal(summary.failed, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('normalized same-name recovery sections never overlap', async () => {
|
||||||
|
let active = 0;
|
||||||
|
let maximumActive = 0;
|
||||||
|
let sequence = 0;
|
||||||
|
loadManager(async () => {
|
||||||
|
active++;
|
||||||
|
maximumActive = Math.max(maximumActive, active);
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 20));
|
||||||
|
active--;
|
||||||
|
sequence++;
|
||||||
|
return {
|
||||||
|
file_code: `SERIAL_${sequence}`,
|
||||||
|
download_url: `https://byse.sx/d/SERIAL_${sequence}`
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const manager = new UploadManager(settings(2));
|
||||||
|
|
||||||
|
const summary = await runBatch(manager, [
|
||||||
|
{ jobId: 'serialized-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
|
||||||
|
{ jobId: 'serialized-b', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(summary.succeeded, 2);
|
||||||
|
assert.equal(maximumActive, 1);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user