Harden same-name hoster recovery
Serialize recovery windows per hoster account and normalized filename, reserve remote codes across the batch, and keep cancellation responsive while waiting. Reject semantic and malformed file-list baselines instead of treating them as empty accounts, with focused regression coverage and release allowlist updates.
This commit is contained in:
+126
-12
@@ -523,12 +523,17 @@ async function _requestFileList(url, signal, phase, hosterName) {
|
||||
}
|
||||
|
||||
const apiStatus = Number(data && data.status);
|
||||
if (apiStatus >= 400) {
|
||||
const statusText = typeof data.status === 'string' ? data.status.trim().toLowerCase() : '';
|
||||
const semanticFailure = data.success === false
|
||||
|| data.ok === false
|
||||
|| data.status === false
|
||||
|| /^(?:error|failed|failure|denied|invalid|rejected)$/.test(statusText);
|
||||
if (apiStatus >= 400 || semanticFailure) {
|
||||
const retryable = apiStatus === 429 || apiStatus >= 500;
|
||||
throw createTransportError(`${hosterName}: Dateiliste wurde abgelehnt`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: apiStatus,
|
||||
httpStatus: apiStatus >= 100 ? apiStatus : response.statusCode,
|
||||
contentType,
|
||||
body: text,
|
||||
retryable,
|
||||
@@ -539,6 +544,18 @@ async function _requestFileList(url, signal, phase, hosterName) {
|
||||
return data;
|
||||
}
|
||||
|
||||
function _requireFileList(data, candidates, phase, hosterName, url) {
|
||||
for (const candidate of candidates) {
|
||||
if (Array.isArray(candidate)) return candidate;
|
||||
}
|
||||
throw createTransportError(`${hosterName}: Dateiliste hatte ein ungültiges Format`, {
|
||||
phase,
|
||||
endpoint: url,
|
||||
httpStatus: 200,
|
||||
contentType: 'application/json'
|
||||
});
|
||||
}
|
||||
|
||||
async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
||||
// Byse's file-list endpoint. Returns up to 100 most-recent files — enough
|
||||
// to match the upload we just did against what the server has. The API
|
||||
@@ -546,9 +563,11 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
||||
// { status, msg, files: [...] }.
|
||||
const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`;
|
||||
const data = await _requestFileList(url, signal, phase, 'Byse');
|
||||
const src = Array.isArray(data.files) ? data.files
|
||||
: (data.result && Array.isArray(data.result.files) ? data.result.files
|
||||
: (Array.isArray(data.result) ? data.result : []));
|
||||
const src = _requireFileList(data, [
|
||||
data.files,
|
||||
data.result && data.result.files,
|
||||
data.result
|
||||
], phase, 'Byse', url);
|
||||
return src.map(f => ({
|
||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||
file_name: String(f.title || f.name || f.file_name || '').trim()
|
||||
@@ -559,7 +578,82 @@ function _normalizeFileTitle(s) {
|
||||
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
||||
}
|
||||
|
||||
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) {
|
||||
function _createAbortError() {
|
||||
const error = new Error('Operation aborted');
|
||||
error.name = 'AbortError';
|
||||
return error;
|
||||
}
|
||||
|
||||
function _waitForRecoveryTurn(predecessor, signal) {
|
||||
if (!signal) return predecessor;
|
||||
if (signal.aborted) return Promise.reject(_createAbortError());
|
||||
return new Promise((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
reject(_createAbortError());
|
||||
};
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
predecessor.then(
|
||||
() => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
resolve();
|
||||
},
|
||||
error => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
reject(error);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function createRecoveryClaimRegistry() {
|
||||
const scopes = new Map();
|
||||
return {
|
||||
forUpload(hosterName, apiKey, fileName) {
|
||||
const identity = crypto.createHash('sha256')
|
||||
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`)
|
||||
.digest('hex');
|
||||
let codes = scopes.get(identity);
|
||||
if (!codes) {
|
||||
codes = {
|
||||
values: new Set(),
|
||||
tail: Promise.resolve()
|
||||
};
|
||||
scopes.set(identity, codes);
|
||||
}
|
||||
return {
|
||||
has(code) {
|
||||
return codes.values.has(String(code || '').trim());
|
||||
},
|
||||
reserve(code) {
|
||||
const normalized = String(code || '').trim();
|
||||
if (!normalized || codes.values.has(normalized)) return false;
|
||||
codes.values.add(normalized);
|
||||
return true;
|
||||
},
|
||||
async runExclusive(operation, signal) {
|
||||
const predecessor = codes.tail;
|
||||
let release;
|
||||
const current = new Promise(resolve => {
|
||||
release = resolve;
|
||||
});
|
||||
codes.tail = predecessor.then(() => current);
|
||||
try {
|
||||
await _waitForRecoveryTurn(predecessor, signal);
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
};
|
||||
},
|
||||
clear() {
|
||||
scopes.clear();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
|
||||
if (!(baselineCodes instanceof Set)) return null;
|
||||
const expected = _normalizeFileTitle(fileName);
|
||||
const POLL_ATTEMPTS = 15;
|
||||
@@ -573,10 +667,13 @@ async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal)
|
||||
// poller could claim job B's newly appeared file and return the wrong
|
||||
// URL. At the cost of a few false-negatives when byse mangles the
|
||||
// filename beyond our normalizer, correctness for parallel uploads wins.
|
||||
const matches = newFiles.filter(f => _normalizeFileTitle(f.file_name) === expected);
|
||||
const matches = newFiles
|
||||
.filter(f => _normalizeFileTitle(f.file_name) === expected)
|
||||
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
|
||||
if (matches.length > 1) return null;
|
||||
if (matches.length === 1) {
|
||||
const match = matches[0];
|
||||
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
|
||||
return {
|
||||
download_url: `https://byse.sx/d/${match.file_code}`,
|
||||
embed_url: `https://byse.sx/e/${match.file_code}`,
|
||||
@@ -601,7 +698,7 @@ async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll')
|
||||
// recovery only needs the most recent uploads, so page 1 newest-first suffices.
|
||||
const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`;
|
||||
const data = await _requestFileList(url, signal, phase, 'Doodstream');
|
||||
const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : [];
|
||||
const files = _requireFileList(data, [data && data.result && data.result.files], phase, 'Doodstream', url);
|
||||
return files.map(f => ({
|
||||
file_code: String(f.file_code || f.filecode || '').trim(),
|
||||
file_name: String(f.title || f.file_name || f.name || '').trim()
|
||||
@@ -610,7 +707,7 @@ async function _fetchDoodstreamFileList(apiKey, signal, phase = 'recovery-poll')
|
||||
|
||||
const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test
|
||||
|
||||
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal) {
|
||||
async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal, recoveryClaim) {
|
||||
if (!(baselineCodes instanceof Set)) return null;
|
||||
// Same recovery byse uses: the upload POST returned no filecode, but the file
|
||||
// may register in the account a little later. Poll the list for a NEW file
|
||||
@@ -624,10 +721,13 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s
|
||||
if (signal && signal.aborted) return null;
|
||||
const list = await _fetchDoodstreamFileList(apiKey, signal);
|
||||
const fresh = list.filter(f => !baselineCodes.has(f.file_code));
|
||||
const matches = fresh.filter(f => _normalizeFileTitle(f.file_name) === expected);
|
||||
const matches = fresh
|
||||
.filter(f => _normalizeFileTitle(f.file_name) === expected)
|
||||
.filter(f => !recoveryClaim || typeof recoveryClaim.has !== 'function' || !recoveryClaim.has(f.file_code));
|
||||
if (matches.length > 1) return null;
|
||||
if (matches.length === 1) {
|
||||
const match = matches[0];
|
||||
if (recoveryClaim && typeof recoveryClaim.reserve === 'function' && !recoveryClaim.reserve(match.file_code)) return null;
|
||||
return {
|
||||
download_url: `https://doodstream.com/d/${match.file_code}`,
|
||||
embed_url: `https://doodstream.com/e/${match.file_code}`,
|
||||
@@ -783,6 +883,19 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
parseErr = err;
|
||||
}
|
||||
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
||||
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
|
||||
if (!opts.recoveryClaim.reserve(result.file_code)) {
|
||||
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
retryable: true,
|
||||
hosterTransient: true
|
||||
});
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -808,7 +921,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// even after our uploader gave up.
|
||||
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
||||
const fileName = path.basename(filePath);
|
||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal);
|
||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
}
|
||||
|
||||
@@ -817,7 +930,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// the file did register, claim its code instead of failing the upload.
|
||||
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
||||
const fileName = path.basename(filePath);
|
||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal);
|
||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
}
|
||||
|
||||
@@ -893,6 +1006,7 @@ async function prefetchBaseline(hosterName, apiKey, signal) {
|
||||
module.exports = {
|
||||
uploadFile,
|
||||
prefetchBaseline,
|
||||
createRecoveryClaimRegistry,
|
||||
HOSTER_CONFIGS,
|
||||
__test: {
|
||||
extractUploadServerUrl,
|
||||
|
||||
+22
-10
@@ -3,7 +3,7 @@ const path = require('path');
|
||||
const { assertUploadConfirmation } = require('./upload-confirmation');
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const { uploadFile, prefetchBaseline } = require('./hosters');
|
||||
const { uploadFile, prefetchBaseline, createRecoveryClaimRegistry } = require('./hosters');
|
||||
const VidmolyUploader = require('./vidmoly-upload');
|
||||
const VoeUploader = require('./voe-upload');
|
||||
const DoodstreamUploader = require('./doodstream-upload');
|
||||
@@ -52,6 +52,7 @@ class UploadManager extends EventEmitter {
|
||||
this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file
|
||||
this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
|
||||
this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
|
||||
this._recoveryClaims = createRecoveryClaimRegistry();
|
||||
}
|
||||
|
||||
updateAccountPools(accountPools) {
|
||||
@@ -68,6 +69,7 @@ class UploadManager extends EventEmitter {
|
||||
this._suspectGoodAccounts.clear();
|
||||
this._doodApiKeyCache.clear();
|
||||
this._baselineCache.clear();
|
||||
this._recoveryClaims.clear();
|
||||
}
|
||||
|
||||
switchAccount(hoster, fallbackAccount) {
|
||||
@@ -348,6 +350,7 @@ class UploadManager extends EventEmitter {
|
||||
for (const jobId of pendingCancelledJobIds) this.cancelledJobIds.add(jobId);
|
||||
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
||||
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
||||
this._recoveryClaims.clear();
|
||||
this.semaphores = {};
|
||||
this.globalSemaphore = null;
|
||||
this.globalThrottle = null;
|
||||
@@ -1258,9 +1261,7 @@ class UploadManager extends EventEmitter {
|
||||
const apiKey = await this._resolveDoodstreamApiKey(task);
|
||||
if (apiKey) {
|
||||
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||
return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle, {
|
||||
doodBaseline: await this._getBaseline('doodstream.com', apiKey, signal)
|
||||
});
|
||||
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe);
|
||||
}
|
||||
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||
const dood = new DoodstreamUploader();
|
||||
@@ -1270,16 +1271,27 @@ class UploadManager extends EventEmitter {
|
||||
const clouddrop = new ClouddropUploader(task.apiKey);
|
||||
return clouddrop.upload(task.file, progressCb, signal, throttle);
|
||||
} else {
|
||||
const baselineOpts = {};
|
||||
if (task.hoster === 'byse.sx') {
|
||||
baselineOpts.byseBaseline = await this._getBaseline('byse.sx', task.apiKey, signal);
|
||||
if (fileProbe && fileProbe.ok !== false) baselineOpts.probeIsVideoLike = fileProbe.isVideoLike === true;
|
||||
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
||||
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe);
|
||||
}
|
||||
if (task.hoster === 'doodstream.com') baselineOpts.doodBaseline = await this._getBaseline('doodstream.com', task.apiKey, signal);
|
||||
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, baselineOpts);
|
||||
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
|
||||
}
|
||||
}
|
||||
|
||||
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
|
||||
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
|
||||
return recoveryClaim.runExclusive(async () => {
|
||||
const options = { recoveryClaim };
|
||||
if (hosterName === 'byse.sx') {
|
||||
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
|
||||
} else {
|
||||
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||
}
|
||||
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
|
||||
}, signal);
|
||||
}
|
||||
|
||||
_getBaseline(hosterName, apiKey, signal) {
|
||||
if (!apiKey) return Promise.resolve(null);
|
||||
const key = `${hosterName}:${apiKey}`;
|
||||
|
||||
@@ -116,6 +116,7 @@ const sourceFiles = [
|
||||
'tests/history-retention.test.js',
|
||||
'tests/hosters.test.js',
|
||||
'tests/hoster-recovery-provenance.test.js',
|
||||
'tests/hoster-recovery-safety.test.js',
|
||||
'tests/i18n.test.js',
|
||||
'tests/ip-allowlist.test.js',
|
||||
'tests/log-mode.test.js',
|
||||
@@ -158,6 +159,7 @@ const sourceFiles = [
|
||||
'tests/upload-confirmation.test.js',
|
||||
'tests/upload-diagnostics.test.js',
|
||||
'tests/upload-manager.test.js',
|
||||
'tests/upload-manager-recovery-claims.test.js',
|
||||
'tests/upload-recovery.test.js',
|
||||
'tests/upload-start-reservation.test.js',
|
||||
'tests/session-report.test.js',
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
const { after, before, test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
|
||||
let requestRouter = async () => ({ statusCode: 200, headers: {}, body: { text: async () => '{}' } });
|
||||
const undici = require('undici');
|
||||
const originalRequest = undici.request;
|
||||
undici.request = (...args) => requestRouter(...args);
|
||||
delete require.cache[require.resolve('../lib/hosters')];
|
||||
const hosters = require('../lib/hosters');
|
||||
|
||||
let tempRoot;
|
||||
let uploadPath;
|
||||
let originalFetch;
|
||||
|
||||
before(() => {
|
||||
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mhu-recovery-safety-'));
|
||||
uploadPath = path.join(tempRoot, 'Shared Episode.mkv');
|
||||
fs.writeFileSync(uploadPath, Buffer.alloc(2048, 7));
|
||||
originalFetch = global.fetch;
|
||||
hosters.__test.DOODSTREAM_POLL.attempts = 1;
|
||||
hosters.__test.DOODSTREAM_POLL.delayMs = 0;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
global.fetch = originalFetch;
|
||||
undici.request = originalRequest;
|
||||
delete require.cache[require.resolve('../lib/hosters')];
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function stubUploadServer() {
|
||||
global.fetch = async () => ({
|
||||
status: 200,
|
||||
text: async () => JSON.stringify({ status: 200, result: 'https://node1.cloudatacdn.com/upload/01' })
|
||||
});
|
||||
}
|
||||
|
||||
function response(body, statusCode = 200) {
|
||||
return {
|
||||
statusCode,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: { text: async () => typeof body === 'string' ? body : JSON.stringify(body) }
|
||||
};
|
||||
}
|
||||
|
||||
async function drain(body) {
|
||||
if (!body || typeof body[Symbol.asyncIterator] !== 'function') return;
|
||||
for await (const chunk of body) {
|
||||
if (chunk && chunk.length === -1) break;
|
||||
}
|
||||
}
|
||||
|
||||
function createClaim() {
|
||||
const codes = new Set();
|
||||
return {
|
||||
has: (code) => codes.has(String(code)),
|
||||
reserve(code) {
|
||||
const normalized = String(code || '').trim();
|
||||
if (!normalized || codes.has(normalized)) return false;
|
||||
codes.add(normalized);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
test('parallel same-name recovery cannot reuse a directly confirmed remote file', async () => {
|
||||
stubUploadServer();
|
||||
const recoveryClaim = createClaim();
|
||||
let uploadCalls = 0;
|
||||
requestRouter = async (url, options) => {
|
||||
if (/\/api\/file\/list/.test(String(url))) {
|
||||
return response({
|
||||
status: 200,
|
||||
result: { files: [{ file_code: 'SHARED_REMOTE_CODE', title: path.basename(uploadPath) }] }
|
||||
});
|
||||
}
|
||||
await drain(options && options.body);
|
||||
uploadCalls++;
|
||||
if (uploadCalls === 1) {
|
||||
return response({
|
||||
status: 200,
|
||||
result: [{ filecode: 'SHARED_REMOTE_CODE', download_url: 'https://doodstream.com/d/SHARED_REMOTE_CODE' }]
|
||||
});
|
||||
}
|
||||
return response({ status: 200, msg: 'OK' });
|
||||
};
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null, {
|
||||
doodBaseline: new Set(),
|
||||
recoveryClaim
|
||||
}),
|
||||
hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null, {
|
||||
doodBaseline: new Set(),
|
||||
recoveryClaim
|
||||
})
|
||||
]);
|
||||
|
||||
assert.equal(results.filter(result => result.status === 'fulfilled').length, 1);
|
||||
assert.equal(results.filter(result => result.status === 'rejected').length, 1);
|
||||
assert.equal(results.find(result => result.status === 'fulfilled').value.file_code, 'SHARED_REMOTE_CODE');
|
||||
assert.equal(results.find(result => result.status === 'rejected').reason.hosterTransient, true);
|
||||
});
|
||||
|
||||
test('a direct response rejects a remote code already reserved in its recovery scope', async () => {
|
||||
stubUploadServer();
|
||||
const recoveryClaim = createClaim();
|
||||
recoveryClaim.reserve('ALREADY_RESERVED_CODE');
|
||||
requestRouter = async (url, options) => {
|
||||
if (/\/api\/file\/list/.test(String(url))) {
|
||||
return response({ status: 200, result: { files: [] } });
|
||||
}
|
||||
await drain(options && options.body);
|
||||
return response({
|
||||
status: 200,
|
||||
result: [{
|
||||
filecode: 'ALREADY_RESERVED_CODE',
|
||||
download_url: 'https://doodstream.com/d/ALREADY_RESERVED_CODE'
|
||||
}]
|
||||
});
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() => hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null, {
|
||||
doodBaseline: new Set(),
|
||||
recoveryClaim
|
||||
}),
|
||||
error => {
|
||||
assert.equal(error.hosterTransient, true);
|
||||
assert.equal(error.diagnostic.phase, 'upload-result');
|
||||
assert.equal(error.diagnostic.http, 200);
|
||||
assert.doesNotMatch(error.message, /ALREADY_RESERVED_CODE/);
|
||||
return true;
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
for (const fixture of [
|
||||
{ name: 'semantic error payload', payload: { status: 'error', msg: 'invalid key' } },
|
||||
{ name: 'missing files list', payload: { status: 200, result: {} } }
|
||||
]) {
|
||||
test(`doodstream rejects a ${fixture.name} as a recovery baseline`, async () => {
|
||||
stubUploadServer();
|
||||
let listCalls = 0;
|
||||
requestRouter = async (url, options) => {
|
||||
if (/\/api\/file\/list/.test(String(url))) {
|
||||
listCalls++;
|
||||
return response(fixture.payload);
|
||||
}
|
||||
await drain(options && options.body);
|
||||
return response({ status: 200, msg: 'OK' });
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() => hosters.uploadFile('doodstream.com', uploadPath, 'VALIDKEY', null, null, null),
|
||||
error => {
|
||||
assert.equal(error.diagnostic.phase, 'recovery-baseline');
|
||||
assert.equal(error.diagnostic.http, 200);
|
||||
assert.equal(error.hosterTransient, true);
|
||||
assert.doesNotMatch(error.message, /invalid key/i);
|
||||
return true;
|
||||
}
|
||||
);
|
||||
assert.equal(listCalls, 1);
|
||||
});
|
||||
}
|
||||
|
||||
test('byse rejects a missing files list as a recovery baseline', async () => {
|
||||
global.fetch = async () => ({
|
||||
status: 200,
|
||||
text: async () => JSON.stringify({ status: 200, result: 'https://byse-upload.invalid/upload/01' })
|
||||
});
|
||||
let listCalls = 0;
|
||||
requestRouter = async (url, options) => {
|
||||
if (/\/file\/list/.test(String(url))) {
|
||||
listCalls++;
|
||||
return response({ status: 200, result: {} });
|
||||
}
|
||||
await drain(options && options.body);
|
||||
return response({ status: 200, msg: 'OK' });
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() => hosters.uploadFile('byse.sx', uploadPath, 'VALIDKEY', null, null, null),
|
||||
error => {
|
||||
assert.equal(error.diagnostic.phase, 'recovery-baseline');
|
||||
assert.equal(error.diagnostic.http, 200);
|
||||
assert.equal(error.hosterTransient, true);
|
||||
return true;
|
||||
}
|
||||
);
|
||||
assert.equal(listCalls, 1);
|
||||
});
|
||||
|
||||
test('a cancelled recovery lock waiter exits before the active lease finishes', async () => {
|
||||
const registry = hosters.createRecoveryClaimRegistry();
|
||||
const claim = registry.forUpload('doodstream.com', 'ACCOUNT_KEY', 'Shared Episode.mkv');
|
||||
let releaseFirst;
|
||||
const firstBlocked = new Promise(resolve => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
let markFirstEntered;
|
||||
const firstEntered = new Promise(resolve => {
|
||||
markFirstEntered = resolve;
|
||||
});
|
||||
const first = claim.runExclusive(async () => {
|
||||
markFirstEntered();
|
||||
await firstBlocked;
|
||||
});
|
||||
await firstEntered;
|
||||
const abortController = new AbortController();
|
||||
const second = claim.runExclusive(async () => 'unexpected', abortController.signal);
|
||||
abortController.abort();
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
second.then(value => ({ status: 'fulfilled', value }), error => ({ status: 'rejected', error })),
|
||||
new Promise(resolve => setTimeout(() => resolve({ status: 'timeout' }), 100))
|
||||
]);
|
||||
assert.equal(outcome.status, 'rejected');
|
||||
assert.equal(outcome.error.name, 'AbortError');
|
||||
} finally {
|
||||
releaseFirst();
|
||||
await first;
|
||||
await second.catch(() => {});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
const { after, before, test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
|
||||
const hosters = require('../lib/hosters');
|
||||
const originalUploadFile = hosters.uploadFile;
|
||||
const originalPrefetchBaseline = hosters.prefetchBaseline;
|
||||
let tempRoot;
|
||||
let firstPath;
|
||||
let secondPath;
|
||||
let UploadManager;
|
||||
|
||||
before(() => {
|
||||
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mhu-manager-recovery-'));
|
||||
const firstDir = path.join(tempRoot, 'first');
|
||||
const secondDir = path.join(tempRoot, 'second');
|
||||
fs.mkdirSync(firstDir);
|
||||
fs.mkdirSync(secondDir);
|
||||
firstPath = path.join(firstDir, 'Shared Episode.mkv');
|
||||
secondPath = path.join(secondDir, 'shared-episode.mp4');
|
||||
fs.writeFileSync(firstPath, Buffer.alloc(1024, 1));
|
||||
fs.writeFileSync(secondPath, Buffer.alloc(1024, 2));
|
||||
});
|
||||
|
||||
after(() => {
|
||||
hosters.uploadFile = originalUploadFile;
|
||||
hosters.prefetchBaseline = originalPrefetchBaseline;
|
||||
delete require.cache[require.resolve('../lib/upload-manager')];
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function loadManager(uploadFile) {
|
||||
hosters.uploadFile = uploadFile;
|
||||
hosters.prefetchBaseline = async () => new Set();
|
||||
delete require.cache[require.resolve('../lib/upload-manager')];
|
||||
UploadManager = require('../lib/upload-manager');
|
||||
}
|
||||
|
||||
function settings(parallelCount) {
|
||||
return {
|
||||
'byse.sx': {
|
||||
retries: 0,
|
||||
parallelCount,
|
||||
maxSpeedKbs: 0,
|
||||
restartBelowKbs: 0,
|
||||
timeIntervalSec: 0,
|
||||
maxSizeMb: 0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function runBatch(manager, tasks) {
|
||||
let summary;
|
||||
manager.once('batch-done', value => {
|
||||
summary = value;
|
||||
});
|
||||
await manager.startBatch(tasks);
|
||||
return summary;
|
||||
}
|
||||
|
||||
test('a batch shares recovery claims across normalized same-name jobs', async () => {
|
||||
let unsafeCalls = 0;
|
||||
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||
const claim = options && options.recoveryClaim;
|
||||
if (claim && claim.reserve('SHARED_REMOTE_CODE')) {
|
||||
return {
|
||||
file_code: 'SHARED_REMOTE_CODE',
|
||||
download_url: 'https://byse.sx/d/SHARED_REMOTE_CODE'
|
||||
};
|
||||
}
|
||||
if (!claim) {
|
||||
unsafeCalls++;
|
||||
return {
|
||||
file_code: `UNSAFE_${unsafeCalls}`,
|
||||
download_url: `https://byse.sx/d/UNSAFE_${unsafeCalls}`
|
||||
};
|
||||
}
|
||||
const error = new Error('Remote recovery candidate already claimed');
|
||||
error.hosterTransient = true;
|
||||
throw error;
|
||||
});
|
||||
const manager = new UploadManager(settings(2));
|
||||
|
||||
const summary = await runBatch(manager, [
|
||||
{ jobId: 'same-name-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
|
||||
{ jobId: 'same-name-b', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||
]);
|
||||
|
||||
assert.equal(summary.succeeded, 1);
|
||||
assert.equal(summary.failed, 1);
|
||||
});
|
||||
|
||||
test('recovery claims stay isolated between accounts', async () => {
|
||||
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||
if (!options || !options.recoveryClaim) {
|
||||
throw new Error('Missing recovery claim');
|
||||
}
|
||||
if (!options.recoveryClaim.reserve('SHARED_REMOTE_CODE')) {
|
||||
const error = new Error('Remote recovery candidate already claimed');
|
||||
error.hosterTransient = true;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
file_code: 'SHARED_REMOTE_CODE',
|
||||
download_url: 'https://byse.sx/d/SHARED_REMOTE_CODE'
|
||||
};
|
||||
});
|
||||
const manager = new UploadManager(settings(2));
|
||||
|
||||
const summary = await runBatch(manager, [
|
||||
{ jobId: 'account-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_A' },
|
||||
{ jobId: 'account-b', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_B' }
|
||||
]);
|
||||
|
||||
assert.equal(summary.succeeded, 2);
|
||||
assert.equal(summary.failed, 0);
|
||||
});
|
||||
|
||||
test('normalized same-name recovery sections never overlap', async () => {
|
||||
let active = 0;
|
||||
let maximumActive = 0;
|
||||
let sequence = 0;
|
||||
loadManager(async () => {
|
||||
active++;
|
||||
maximumActive = Math.max(maximumActive, active);
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
active--;
|
||||
sequence++;
|
||||
return {
|
||||
file_code: `SERIAL_${sequence}`,
|
||||
download_url: `https://byse.sx/d/SERIAL_${sequence}`
|
||||
};
|
||||
});
|
||||
const manager = new UploadManager(settings(2));
|
||||
|
||||
const summary = await runBatch(manager, [
|
||||
{ jobId: 'serialized-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
|
||||
{ jobId: 'serialized-b', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||
]);
|
||||
|
||||
assert.equal(summary.succeeded, 2);
|
||||
assert.equal(maximumActive, 1);
|
||||
});
|
||||
Reference in New Issue
Block a user