Harden batch-wide recovery identity claims

Scope remote code ownership to normalized hoster and account identities while retaining title-only recovery serialization and canonical Unicode title matching.

Mark post-upload ambiguity and duplicate identities as uncertain so retries, account fallback, and later same-title jobs fail closed instead of reporting unsafe success.

Acquire recovery title leases before hoster and global semaphores, revalidate failed-account overrides before upload, and clear claim state at batch boundaries.

Add deterministic concurrent coverage for same-code rejection, distinct-code parallel success, uncertainty propagation, semaphore fairness, account isolation, Unicode equivalence, and registry lifetime.
This commit is contained in:
Sucukdeluxe
2026-08-13 23:09:03 +02:00
parent dd14381e43
commit c78160a521
7 changed files with 640 additions and 141 deletions
+122 -34
View File
@@ -575,7 +575,40 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
} }
function _normalizeFileTitle(s) { function _normalizeFileTitle(s) {
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, ''); return String(s || '')
.normalize('NFKD')
.toLowerCase()
.replace(/\.[\p{Letter}\p{Number}]+$/u, '')
.replace(/\p{Mark}+/gu, '')
.replace(/[^\p{Letter}\p{Number}]+/gu, '');
}
function _normalizeRecoveryHoster(value) {
return String(value || '').normalize('NFKC').trim().toLowerCase();
}
function _normalizeRecoveryAccount(value) {
return String(value || '').normalize('NFKC').trim();
}
function _createRecoveryUncertainError() {
const error = new Error('Upload-Ergebnis für diesen Titel ist wegen eines möglichen Remote-Commits unsicher');
error.remoteCommitUncertain = true;
error.hosterTransient = true;
return error;
}
function _markRecoveryUncertain(recoveryClaim, error) {
if (!recoveryClaim) return error;
if (typeof recoveryClaim.markUncertain === 'function') {
return recoveryClaim.markUncertain(error);
}
const uncertainError = error && typeof error === 'object'
? error
: _createRecoveryUncertainError();
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
} }
function _createAbortError() { function _createAbortError() {
@@ -607,40 +640,69 @@ function _waitForRecoveryTurn(predecessor, signal) {
} }
function createRecoveryClaimRegistry() { function createRecoveryClaimRegistry() {
const scopes = new Map(); const accounts = new Map();
let nextClaimId = 1;
return { return {
forUpload(hosterName, apiKey, fileName) { forUpload(hosterName, apiKey, fileName) {
const identity = crypto.createHash('sha256') const accountIdentity = crypto.createHash('sha256')
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`) .update(`${_normalizeRecoveryHoster(hosterName)}\0${_normalizeRecoveryAccount(apiKey)}`)
.digest('hex'); .digest('hex');
let codes = scopes.get(identity); let account = accounts.get(accountIdentity);
if (!codes) { if (!account) {
codes = { account = {
values: new Set(), codes: new Map(),
tail: Promise.resolve() titles: new Map()
}; };
scopes.set(identity, codes); accounts.set(accountIdentity, account);
} }
const titleIdentity = _normalizeFileTitle(fileName);
let title = account.titles.get(titleIdentity);
if (!title) {
title = {
tail: Promise.resolve(),
uncertain: false
};
account.titles.set(titleIdentity, title);
}
const claimId = nextClaimId++;
return { return {
has(code) { has(code) {
return codes.values.has(String(code || '').trim()); return account.codes.has(String(code || '').trim());
}, },
reserve(code) { reserve(code) {
const normalized = String(code || '').trim(); const normalized = String(code || '').trim();
if (!normalized || codes.values.has(normalized)) return false; if (!normalized) return false;
codes.values.add(normalized); if (account.codes.has(normalized)) {
return account.codes.get(normalized) === claimId;
}
account.codes.set(normalized, claimId);
return true; return true;
}, },
markUncertain(error) {
title.uncertain = true;
const uncertainError = error && typeof error === 'object'
? error
: _createRecoveryUncertainError();
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
},
isUncertain() {
return title.uncertain;
},
async runExclusive(operation, signal) { async runExclusive(operation, signal) {
const predecessor = codes.tail; const predecessor = title.tail;
let release; let release;
const current = new Promise(resolve => { const current = new Promise(resolve => {
release = resolve; release = resolve;
}); });
codes.tail = predecessor.then(() => current); title.tail = predecessor.then(() => current, () => current);
try { try {
await _waitForRecoveryTurn(predecessor, signal); await _waitForRecoveryTurn(predecessor, signal);
return await operation(); if (title.uncertain) throw _createRecoveryUncertainError();
const result = await operation();
if (title.uncertain) throw _createRecoveryUncertainError();
return result;
} finally { } finally {
release(); release();
} }
@@ -648,7 +710,7 @@ function createRecoveryClaimRegistry() {
}; };
}, },
clear() { clear() {
scopes.clear(); accounts.clear();
} }
}; };
} }
@@ -805,23 +867,28 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
bodyTimeout: UPLOAD_TIMEOUT bodyTimeout: UPLOAD_TIMEOUT
}); });
} catch (err) { } catch (err) {
if (signal && signal.aborted) throw err; const error = signal && signal.aborted ? err : createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
throw createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
phase: 'upload-request', phase: 'upload-request',
endpoint: targetUrl, endpoint: targetUrl,
retryable: true, retryable: true,
transientNetwork: true transientNetwork: true
}); });
throw _markRecoveryUncertain(opts && opts.recoveryClaim, error);
} }
const { body, statusCode, headers } = uploadResponse; const { body, statusCode, headers } = uploadResponse;
const rawBody = await body.text(); let rawBody;
try {
rawBody = await body.text();
} catch (err) {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
}
let payload = null; let payload = null;
try { try {
payload = rawBody ? JSON.parse(rawBody) : {}; payload = rawBody ? JSON.parse(rawBody) : {};
} catch { } catch {
throw createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, { throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
phase: 'upload-response', phase: 'upload-response',
endpoint: targetUrl, endpoint: targetUrl,
httpStatus: statusCode, httpStatus: statusCode,
@@ -829,7 +896,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
body: rawBody, body: rawBody,
retryable: statusCode >= 500, retryable: statusCode >= 500,
transientNetwork: statusCode >= 500 transientNetwork: statusCode >= 500
}); }));
} }
// Normalize valid-but-not-object JSON (JSON.parse('null') → null; // Normalize valid-but-not-object JSON (JSON.parse('null') → null;
// JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this // JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this
@@ -843,7 +910,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
} }
if (statusCode < 200 || statusCode >= 300) { if (statusCode < 200 || statusCode >= 300) {
throw createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, { const error = createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
phase: 'upload-response', phase: 'upload-response',
endpoint: targetUrl, endpoint: targetUrl,
httpStatus: statusCode, httpStatus: statusCode,
@@ -852,10 +919,13 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
retryable: statusCode === 429 || statusCode >= 500, retryable: statusCode === 429 || statusCode >= 500,
transientNetwork: statusCode >= 500 transientNetwork: statusCode >= 500
}); });
throw statusCode >= 500
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
: error;
} }
if (payload.status && [401, 403, 429, 500].includes(payload.status)) { if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, { const error = createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
phase: 'upload-response', phase: 'upload-response',
endpoint: targetUrl, endpoint: targetUrl,
httpStatus: Number(payload.status), httpStatus: Number(payload.status),
@@ -864,6 +934,9 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
retryable: Number(payload.status) === 429 || Number(payload.status) >= 500, retryable: Number(payload.status) === 429 || Number(payload.status) >= 500,
transientNetwork: Number(payload.status) >= 500 transientNetwork: Number(payload.status) >= 500
}); });
throw Number(payload.status) >= 500
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
: error;
} }
let result = null; let result = null;
@@ -885,7 +958,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
if (result && (result.file_code || result.download_url || result.embed_url)) { if (result && (result.file_code || result.download_url || result.embed_url)) {
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') { if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
if (!opts.recoveryClaim.reserve(result.file_code)) { if (!opts.recoveryClaim.reserve(result.file_code)) {
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, { const error = createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
phase: 'upload-result', phase: 'upload-result',
endpoint: targetUrl, endpoint: targetUrl,
httpStatus: statusCode, httpStatus: statusCode,
@@ -894,6 +967,8 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
retryable: true, retryable: true,
hosterTransient: true hosterTransient: true
}); });
error.remoteIdentityClaimed = true;
throw _markRecoveryUncertain(opts.recoveryClaim, error);
} }
} }
return result; return result;
@@ -921,8 +996,12 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// even after our uploader gave up. // even after our uploader gave up.
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) { if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
const fileName = path.basename(filePath); const fileName = path.basename(filePath);
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim); try {
if (polled) return polled; const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
} catch (err) {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
}
} }
// Doodstream: the doodapi upload POST returned no filecode (the same backend // Doodstream: the doodapi upload POST returned no filecode (the same backend
@@ -930,21 +1009,29 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// the file did register, claim its code instead of failing the upload. // the file did register, claim its code instead of failing the upload.
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) { if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
const fileName = path.basename(filePath); const fileName = path.basename(filePath);
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim); try {
if (polled) return polled; const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
} catch (err) {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
}
} }
if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) { if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) {
byseBaselineError.hosterTransient = true; byseBaselineError.hosterTransient = true;
throw byseBaselineError; throw _markRecoveryUncertain(opts && opts.recoveryClaim, byseBaselineError);
} }
if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) { if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) {
doodBaselineError.hosterTransient = true; doodBaselineError.hosterTransient = true;
throw doodBaselineError; throw _markRecoveryUncertain(opts && opts.recoveryClaim, doodBaselineError);
} }
if (parseErr) throw parseErr; if (parseErr) {
throw explicitlyRejected
? parseErr
: _markRecoveryUncertain(opts && opts.recoveryClaim, parseErr);
}
if (payload.success === false) { if (payload.success === false) {
throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, { throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, {
@@ -970,7 +1057,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// so tag it hosterTransient: the upload-manager then fails this file WITHOUT // so tag it hosterTransient: the upload-manager then fails this file WITHOUT
// blacklisting the account (same protection the web path got in 3.3.29) and // blacklisting the account (same protection the web path got in 3.3.29) and
// the account stays usable for the next retry/batch. // the account stays usable for the next retry/batch.
throw createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, { throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
phase: 'upload-result', phase: 'upload-result',
endpoint: targetUrl, endpoint: targetUrl,
httpStatus: statusCode, httpStatus: statusCode,
@@ -978,7 +1065,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
body: rawBody, body: rawBody,
retryable: true, retryable: true,
hosterTransient: true hosterTransient: true
}); }));
} }
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, { throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
phase: 'upload-result', phase: 'upload-result',
@@ -1007,6 +1094,7 @@ module.exports = {
uploadFile, uploadFile,
prefetchBaseline, prefetchBaseline,
createRecoveryClaimRegistry, createRecoveryClaimRegistry,
normalizeRecoveryTitle: _normalizeFileTitle,
HOSTER_CONFIGS, HOSTER_CONFIGS,
__test: { __test: {
extractUploadServerUrl, extractUploadServerUrl,
+157 -73
View File
@@ -71,7 +71,7 @@ class UploadManager extends EventEmitter {
this._suspectGoodAccounts.clear(); this._suspectGoodAccounts.clear();
this._doodApiKeyCache.clear(); this._doodApiKeyCache.clear();
this._baselineCache.clear(); this._baselineCache.clear();
this._recoveryClaims.clear(); if (!this.running) this._recoveryClaims.clear();
} }
switchAccount(hoster, fallbackAccount) { switchAccount(hoster, fallbackAccount) {
@@ -134,6 +134,41 @@ class UploadManager extends EventEmitter {
return true; return true;
} }
_swapFailedAccount(task, jobId, fileName) {
if (!task.accountId || !this._failedAccounts.has(task.hoster + ':' + task.accountId)) return false;
const override = this._accountOverrides.get(task.hoster);
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
this._rotLog('pre-job-swap', {
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
});
task.accountId = override.id;
task.username = override.username;
task.password = override.password;
task.apiKey = override.apiKey;
return true;
}
this._rotLog('pre-job-swap-blocked', {
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
hasOverride: !!override,
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
});
return false;
}
async _coordinateAccountFailure(task, err, signal, jobId) {
if (!task.accountId || (err && err.remoteCommitUncertain === true)) return;
if (!this._shouldSkipRetryOnAccountError(err)) return;
const key = task.hoster + ':' + task.accountId;
if (this._failedAccounts.has(key)) return;
this._failedAccounts.set(key, true);
this._rotLog('mark-failed', {
jobId, hoster: task.hoster, fileName: path.basename(task.file),
accountId: task.accountId, lastError: err && err.message ? err.message : String(err)
});
this.emit('account-failed', { hoster: task.hoster, accountId: task.accountId });
await this._sleep(800, signal);
}
_rotLog(event, data) { _rotLog(event, data) {
this.emit('rot-log', { ts: Date.now(), event, ...data }); this.emit('rot-log', { ts: Date.now(), event, ...data });
} }
@@ -161,6 +196,7 @@ class UploadManager extends EventEmitter {
// which takes priority in _shouldSkipRetryOnAccountError. // which takes priority in _shouldSkipRetryOnAccountError.
_isFileRejectedError(err) { _isFileRejectedError(err) {
if (!err) return false; if (!err) return false;
if (err.remoteCommitUncertain === true) return false;
if (err.transientNetwork === true) return false; if (err.transientNetwork === true) return false;
if (err.accountError === true) return false; // explicit account-level wins if (err.accountError === true) return false; // explicit account-level wins
if (err.fileRejected === true) return true; if (err.fileRejected === true) return true;
@@ -355,6 +391,7 @@ class UploadManager extends EventEmitter {
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance) this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
this._recoveryClaims.clear(); this._recoveryClaims.clear();
this._recoveryClaims = createRecoveryClaimRegistry();
this.semaphores = {}; this.semaphores = {};
this.globalSemaphore = null; this.globalSemaphore = null;
this.globalThrottle = null; this.globalThrottle = null;
@@ -440,13 +477,12 @@ class UploadManager extends EventEmitter {
files files
}; };
this._recoveryClaims.clear();
this.emit('batch-done', summary); this.emit('batch-done', summary);
} }
async _runJob(task, results, batchSignal) { async _runJob(task, results, batchSignal) {
const settings = this._getSettings(task.hoster); const settings = this._getSettings(task.hoster);
const hosterSemaphore = this._getSemaphore(task.hoster);
const globalSemaphore = this._getGlobalSemaphore();
const uploadId = crypto.randomBytes(8).toString('hex'); const uploadId = crypto.randomBytes(8).toString('hex');
const jobId = task.jobId || uploadId; const jobId = task.jobId || uploadId;
const fileName = path.basename(task.file); const fileName = path.basename(task.file);
@@ -466,8 +502,6 @@ class UploadManager extends EventEmitter {
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal); const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
this.jobAbortControllers.set(jobId, jobAbortController); this.jobAbortControllers.set(jobId, jobAbortController);
let hosterSlotAcquired = false;
let globalSlotAcquired = false;
let finalResultRecorded = false; let finalResultRecorded = false;
let finalStatus = 'error'; let finalStatus = 'error';
let lastError = null; let lastError = null;
@@ -549,9 +583,6 @@ class UploadManager extends EventEmitter {
// queueJobs array; the first event it actually needs from main is the // queueJobs array; the first event it actually needs from main is the
// 'getting-server' / 'uploading' transition for the jobs that the // 'getting-server' / 'uploading' transition for the jobs that the
// semaphore lets through. // semaphore lets through.
await hosterSemaphore.acquire(signal);
hosterSlotAcquired = true;
let fileProbe = null; let fileProbe = null;
try { try {
fileProbe = await probeFileHead(task.file, 512); fileProbe = await probeFileHead(task.file, 512);
@@ -566,11 +597,6 @@ class UploadManager extends EventEmitter {
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
}); });
if (globalSemaphore) {
await globalSemaphore.acquire(signal);
globalSlotAcquired = true;
}
if (settings.timeIntervalSec > 0) { if (settings.timeIntervalSec > 0) {
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal); await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal);
} }
@@ -580,24 +606,7 @@ class UploadManager extends EventEmitter {
// of burning a guaranteed-to-fail upload attempt. Critical at scale: // of burning a guaranteed-to-fail upload attempt. Critical at scale:
// with 500 queued jobs and 1 parallel slot, without this check every // with 500 queued jobs and 1 parallel slot, without this check every
// job still hits the original dead account first. // job still hits the original dead account first.
if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) { this._swapFailedAccount(task, jobId, fileName);
const override = this._accountOverrides.get(task.hoster);
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
this._rotLog('pre-job-swap', {
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
});
task.accountId = override.id;
task.username = override.username;
task.password = override.password;
task.apiKey = override.apiKey;
} else {
this._rotLog('pre-job-swap-blocked', {
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
hasOverride: !!override,
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
});
}
}
// A previous file of at least this size already got a suspect rejection // A previous file of at least this size already got a suspect rejection
// on this exact account — skip the guaranteed-to-fail multi-GB upload // on this exact account — skip the guaranteed-to-fail multi-GB upload
@@ -735,7 +744,7 @@ class UploadManager extends EventEmitter {
} catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ } } catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ }
}; };
const result = await this._executeUpload(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe); const result = await this._executeUploadWithAdmission(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe, true, jobId);
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted'); if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
@@ -784,6 +793,11 @@ class UploadManager extends EventEmitter {
break; break;
} }
if (err && err.remoteCommitUncertain === true) {
lastError = err;
break;
}
if (isSpeedRestart && attempt < maxAttempts) { if (isSpeedRestart && attempt < maxAttempts) {
lastError = new Error('Geschwindigkeit zu niedrig - Neustart'); lastError = new Error('Geschwindigkeit zu niedrig - Neustart');
await this._sleep(3000, signal); await this._sleep(3000, signal);
@@ -1047,7 +1061,7 @@ class UploadManager extends EventEmitter {
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } } ? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
: hosterThrottle || globalThrottle; : hosterThrottle || globalThrottle;
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe); const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, true, jobId);
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted'); if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
this.activeJobs.delete(uploadId); this.activeJobs.delete(uploadId);
this.sessionBytes += fileSize; this.sessionBytes += fileSize;
@@ -1104,9 +1118,6 @@ class UploadManager extends EventEmitter {
this.activeJobs.delete(uploadId); this.activeJobs.delete(uploadId);
this.jobAbortControllers.delete(jobId); this.jobAbortControllers.delete(jobId);
cleanupSignals(); cleanupSignals();
// Release in reverse order of acquire (global first, then hoster)
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
if (hosterSlotAcquired) hosterSemaphore.release();
this.emit('job-settled', { this.emit('job-settled', {
jobId, jobId,
sourceCleanupToken: task.sourceCleanupToken || null, sourceCleanupToken: task.sourceCleanupToken || null,
@@ -1190,7 +1201,7 @@ class UploadManager extends EventEmitter {
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } } ? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
: hosterThrottle || globalThrottle; : hosterThrottle || globalThrottle;
try { try {
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe); const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, false, jobId);
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted'); if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
this.activeJobs.delete(uploadId); this.activeJobs.delete(uploadId);
this.sessionBytes += fileSize; this.sessionBytes += fileSize;
@@ -1210,6 +1221,7 @@ class UploadManager extends EventEmitter {
}); });
} }
if (signal.aborted || this.stopAfterActive) break; if (signal.aborted || this.stopAfterActive) break;
if (err && err.remoteCommitUncertain === true) throw err;
if (err && err.suspectReject === true) { if (err && err.suspectReject === true) {
this._noteSuspectReject(task.hoster, account.id, fileSize); this._noteSuspectReject(task.hoster, account.id, fileSize);
} }
@@ -1241,16 +1253,95 @@ class UploadManager extends EventEmitter {
return null; return null;
} }
async _executeUpload(task, progressCb, signal, throttle, fileProbe) { async _executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, coordinateAccountFailure = true, jobId = task.jobId) {
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe); while (true) {
return assertUploadConfirmation(result, task.hoster); const context = await this._createRecoveryContext(task);
let retryAdmission = false;
const operation = async () => {
const hosterSemaphore = this._getSemaphore(task.hoster);
const globalSemaphore = this._getGlobalSemaphore();
let hosterSlotAcquired = false;
let globalSlotAcquired = false;
try {
await hosterSemaphore.acquire(signal);
hosterSlotAcquired = true;
if (globalSemaphore) {
await globalSemaphore.acquire(signal);
globalSlotAcquired = true;
}
if (this._swapFailedAccount(task, jobId, path.basename(task.file))) {
retryAdmission = true;
return null;
}
try {
return await this._executeUpload(task, progressCb, signal, throttle, fileProbe, context);
} catch (err) {
if (coordinateAccountFailure) await this._coordinateAccountFailure(task, err, signal, jobId);
if (context.recoveryClaim && err && err.remoteCommitUncertain === true) {
throw context.recoveryClaim.markUncertain(err);
}
throw err;
}
} finally {
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
if (hosterSlotAcquired) hosterSemaphore.release();
}
};
const result = context.recoveryClaim
? await context.recoveryClaim.runExclusive(operation, signal)
: await operation();
if (retryAdmission) continue;
return result;
}
} }
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) { async _executeUpload(task, progressCb, signal, throttle, fileProbe, context) {
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context);
let confirmed;
try {
confirmed = assertUploadConfirmation(result, task.hoster);
} catch (err) {
if (context.recoveryClaim) throw context.recoveryClaim.markUncertain(err);
throw err;
}
return confirmed;
}
async _createRecoveryContext(task) {
const fileName = path.basename(task.file);
if ((task.hoster === 'vidmoly.me' || task.hoster === 'voe.sx') && task.username) {
const accountIdentity = task.accountId !== null && task.accountId !== undefined
? task.accountId
: String(task.username || '').trim().toLowerCase();
return {
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
doodApiKey: null
};
}
if (task.hoster === 'doodstream.com' && task.username) {
const doodApiKey = await this._resolveDoodstreamApiKey(task);
const accountIdentity = doodApiKey || (task.accountId !== null && task.accountId !== undefined
? task.accountId
: String(task.username || '').trim().toLowerCase());
return {
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
doodApiKey
};
}
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
return {
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, task.apiKey, fileName),
doodApiKey: null
};
}
return { recoveryClaim: null, doodApiKey: null };
}
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context) {
if (task.hoster === 'vidmoly.me' && task.username) { if (task.hoster === 'vidmoly.me' && task.username) {
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle); return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle, context.recoveryClaim);
} else if (task.hoster === 'voe.sx' && task.username) { } else if (task.hoster === 'voe.sx' && task.username) {
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle); return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle, context.recoveryClaim);
} else if (task.hoster === 'doodstream.com' && task.username) { } else if (task.hoster === 'doodstream.com' && task.username) {
// Login-path reliability fix: the web-form upload returns the filecode in // Login-path reliability fix: the web-form upload returns the filecode in
// an HTML form that comes back empty for large files (doodstream backend // an HTML form that comes back empty for large files (doodstream backend
@@ -1258,54 +1349,47 @@ class UploadManager extends EventEmitter {
// session ONCE per batch and upload via the official API instead — it // session ONCE per batch and upload via the official API instead — it
// returns result[0].filecode directly and has no empty-form failure mode. // returns result[0].filecode directly and has no empty-form failure mode.
// Falls back to the web-form upload if no valid key can be derived. // Falls back to the web-form upload if no valid key can be derived.
const apiKey = await this._resolveDoodstreamApiKey(task); const apiKey = context.doodApiKey;
if (apiKey) { if (apiKey) {
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) }); this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe); return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
} }
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) }); this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
const dood = new DoodstreamUploader(); const dood = new DoodstreamUploader();
await dood.login(task.username, task.password); await dood.login(task.username, task.password);
return dood.upload(task.file, progressCb, signal, throttle); const result = await dood.upload(task.file, progressCb, signal, throttle);
if (result && result.file_code && !context.recoveryClaim.reserve(result.file_code)) {
const error = new Error('Doodstream Upload lieferte eine bereits zugeordnete Remote-Identität');
error.remoteIdentityClaimed = true;
throw context.recoveryClaim.markUncertain(error);
}
return result;
} else if (task.hoster === 'clouddrop.cc') { } else if (task.hoster === 'clouddrop.cc') {
const clouddrop = new ClouddropUploader(task.apiKey); const clouddrop = new ClouddropUploader(task.apiKey);
return clouddrop.upload(task.file, progressCb, signal, throttle); return clouddrop.upload(task.file, progressCb, signal, throttle);
} else { } else {
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') { if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe); return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
} }
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {}); return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
} }
} }
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle) { async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle, recoveryClaim) {
const accountIdentity = task.accountId !== null && task.accountId !== undefined const uploader = new UploaderClass(recoveryClaim);
? task.accountId await uploader.login(task.username, task.password);
: String(task.username || '').trim().toLowerCase(); return uploader.upload(task.file, progressCb, signal, throttle);
const recoveryClaim = this._recoveryClaims.forUpload(
task.hoster,
accountIdentity,
path.basename(task.file)
);
return recoveryClaim.runExclusive(async () => {
const uploader = new UploaderClass(recoveryClaim);
await uploader.login(task.username, task.password);
return uploader.upload(task.file, progressCb, signal, throttle);
}, signal);
} }
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) { async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe, recoveryClaim) {
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath)); const options = { recoveryClaim };
return recoveryClaim.runExclusive(async () => { if (hosterName === 'byse.sx') {
const options = { recoveryClaim }; options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
if (hosterName === 'byse.sx') { if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal); } else {
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true; options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
} else { }
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal); return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
}
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
}, signal);
} }
_getBaseline(hosterName, apiKey, signal) { _getBaseline(hosterName, apiKey, signal) {
+38 -16
View File
@@ -3,6 +3,7 @@ const path = require('path');
const crypto = require('crypto'); const crypto = require('crypto');
const { request } = require('undici'); const { request } = require('undici');
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error'); const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
const { normalizeRecoveryTitle } = require('./hosters');
const BASE_URL = 'https://vidmoly.me'; const BASE_URL = 'https://vidmoly.me';
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
@@ -275,13 +276,13 @@ class VidmolyUploader {
bodyTimeout: UPLOAD_TIMEOUT bodyTimeout: UPLOAD_TIMEOUT
}); });
} catch (err) { } catch (err) {
if (signal && signal.aborted) throw err; const error = signal && signal.aborted ? err : createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
throw createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
phase: 'upload-request', phase: 'upload-request',
endpoint: targetUrl, endpoint: targetUrl,
retryable: true, retryable: true,
transientNetwork: true transientNetwork: true
}); });
throw this._markRemoteCommitUncertain(error);
} }
const { body, statusCode, headers } = uploadResponse; const { body, statusCode, headers } = uploadResponse;
@@ -295,17 +296,25 @@ class VidmolyUploader {
// Always drain the original body to prevent connection leak // Always drain the original body to prevent connection leak
try { await body.text(); } catch {} try { await body.text(); } catch {}
if (location) { if (location) {
const resultRes = await this._fetch(new URL(location, uploadUrl).href); try {
resultHtml = await resultRes.text(); const resultRes = await this._fetch(new URL(location, uploadUrl).href);
resultHtml = await resultRes.text();
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
} else { } else {
resultHtml = ''; resultHtml = '';
} }
} else { } else {
resultHtml = await body.text(); try {
resultHtml = await body.text();
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
} }
if (statusCode >= 400) { if (statusCode >= 400) {
throw createTransportError('Vidmoly Upload fehlgeschlagen', { const error = createTransportError('Vidmoly Upload fehlgeschlagen', {
phase: 'upload-response', phase: 'upload-response',
endpoint: targetUrl, endpoint: targetUrl,
httpStatus: statusCode, httpStatus: statusCode,
@@ -314,6 +323,7 @@ class VidmolyUploader {
retryable: statusCode === 429 || statusCode >= 500, retryable: statusCode === 429 || statusCode >= 500,
transientNetwork: statusCode >= 500 transientNetwork: statusCode >= 500
}); });
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
} }
// Try JSON first. The current transit server returns // Try JSON first. The current transit server returns
@@ -355,23 +365,35 @@ class VidmolyUploader {
} catch (primaryErr) { } catch (primaryErr) {
if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr; if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr;
if (baselineCodes) { if (baselineCodes) {
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal); try {
if (fallback) return fallback; const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
if (fallback) return fallback;
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
} }
if (baselineError) { if (baselineError) {
baselineError.hosterTransient = true; baselineError.hosterTransient = true;
throw baselineError; throw this._markRemoteCommitUncertain(baselineError);
} }
throw primaryErr; throw this._markRemoteCommitUncertain(primaryErr);
} }
} }
_normalizeTitle(value) { _normalizeTitle(value) {
return String(value || '') return normalizeRecoveryTitle(value);
.toLowerCase() }
.normalize('NFKD')
.replace(/\.[a-z0-9]+$/i, '') _markRemoteCommitUncertain(error) {
.replace(/[^a-z0-9]+/g, ''); if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
return this.recoveryClaim.markUncertain(error);
}
const uncertainError = error && typeof error === 'object'
? error
: new Error('Vidmoly Upload-Ergebnis ist unsicher');
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
} }
_buildUrlsFromCode(fileCode, phase = 'upload-result') { _buildUrlsFromCode(fileCode, phase = 'upload-result') {
@@ -387,7 +409,7 @@ class VidmolyUploader {
hosterTransient: true hosterTransient: true
}); });
error.remoteIdentityClaimed = true; error.remoteIdentityClaimed = true;
throw error; throw this._markRemoteCommitUncertain(error);
} }
return { return {
+34 -15
View File
@@ -3,6 +3,7 @@ const path = require('path');
const crypto = require('crypto'); const crypto = require('crypto');
const { request } = require('undici'); const { request } = require('undici');
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error'); const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
const { normalizeRecoveryTitle } = require('./hosters');
const BASE_URL = 'https://voe.sx'; const BASE_URL = 'https://voe.sx';
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
@@ -352,22 +353,27 @@ class VoeUploader {
bodyTimeout: UPLOAD_TIMEOUT bodyTimeout: UPLOAD_TIMEOUT
}); });
} catch (err) { } catch (err) {
if (signal && signal.aborted) throw err; const error = signal && signal.aborted ? err : createTransportError('VOE Upload konnte nicht übertragen werden', {
throw createTransportError('VOE Upload konnte nicht übertragen werden', {
phase: 'upload-request', phase: 'upload-request',
endpoint: uploadServer, endpoint: uploadServer,
retryable: true, retryable: true,
transientNetwork: true transientNetwork: true
}); });
throw this._markRemoteCommitUncertain(error);
} }
const { body, headers, statusCode } = uploadResponse; const { body, headers, statusCode } = uploadResponse;
this._parseCookiesFromHeaders(headers || {}); this._parseCookiesFromHeaders(headers || {});
const rawBody = await body.text(); let rawBody;
try {
rawBody = await body.text();
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
if (statusCode < 200 || statusCode >= 300) { if (statusCode < 200 || statusCode >= 300) {
throw createTransportError('VOE Upload fehlgeschlagen', { const error = createTransportError('VOE Upload fehlgeschlagen', {
phase: 'upload-response', phase: 'upload-response',
endpoint: uploadServer, endpoint: uploadServer,
httpStatus: statusCode, httpStatus: statusCode,
@@ -376,6 +382,7 @@ class VoeUploader {
retryable: statusCode === 429 || statusCode >= 500, retryable: statusCode === 429 || statusCode >= 500,
transientNetwork: statusCode >= 500 transientNetwork: statusCode >= 500
}); });
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
} }
// Try JSON response // Try JSON response
@@ -407,23 +414,27 @@ class VoeUploader {
// Fallback: poll the file list to find the newly uploaded file // Fallback: poll the file list to find the newly uploaded file
if (baselineCodes) { if (baselineCodes) {
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal); try {
if (result) return result; const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
if (result) return result;
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
} }
if (baselineError) { if (baselineError) {
baselineError.hosterTransient = true; baselineError.hosterTransient = true;
throw baselineError; throw this._markRemoteCommitUncertain(baselineError);
} }
throw createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', { throw this._markRemoteCommitUncertain(createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
phase: 'upload-result', phase: 'upload-result',
endpoint: uploadServer, endpoint: uploadServer,
contentType: headers && headers['content-type'], contentType: headers && headers['content-type'],
body: rawBody, body: rawBody,
hosterTransient: true, hosterTransient: true,
retryable: true retryable: true
}); }));
} }
async _resolveUploadedFile(fileName, baselineCodes, signal) { async _resolveUploadedFile(fileName, baselineCodes, signal) {
@@ -478,11 +489,19 @@ class VoeUploader {
} }
_normalizeTitle(value) { _normalizeTitle(value) {
return String(value || '') return normalizeRecoveryTitle(value);
.toLowerCase() }
.normalize('NFKD')
.replace(/\.[a-z0-9]+$/i, '') _markRemoteCommitUncertain(error) {
.replace(/[^a-z0-9]+/g, ''); if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
return this.recoveryClaim.markUncertain(error);
}
const uncertainError = error && typeof error === 'object'
? error
: new Error('VOE Upload-Ergebnis ist unsicher');
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
} }
_buildUrls(fileCode, phase = 'upload-result') { _buildUrls(fileCode, phase = 'upload-result') {
@@ -498,7 +517,7 @@ class VoeUploader {
hosterTransient: true hosterTransient: true
}); });
error.remoteIdentityClaimed = true; error.remoteIdentityClaimed = true;
throw error; throw this._markRemoteCommitUncertain(error);
} }
return { return {
download_url: `${BASE_URL}/${code}`, download_url: `${BASE_URL}/${code}`,
+43
View File
@@ -187,3 +187,46 @@ test('Vidmoly concurrent same-name recovery accepts distinct remote codes', asyn
assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']); assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']);
}); });
for (const scenario of [
{
label: 'VOE',
hoster: 'voe.sx',
Uploader: VoeUploader,
sharedCode: 'VOE_UNCERTAIN',
lateCode: 'VOE_LATE_CODE',
build(uploader, code) {
return uploader._buildUrls(code);
}
},
{
label: 'Vidmoly',
hoster: 'vidmoly.me',
Uploader: VidmolyUploader,
sharedCode: 'VIDUNCERTAIN',
lateCode: 'VIDLATECODE1',
build(uploader, code) {
return uploader._buildUrlsFromCode(code);
}
}
]) {
test(`${scenario.label} marks a duplicate direct identity uncertain and blocks a later title match`, async () => {
const registry = createRecoveryClaimRegistry();
const firstClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'First Episode.mkv');
const uncertainClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'Second Episode.mkv');
const first = new scenario.Uploader(firstClaim);
const uncertain = new scenario.Uploader(uncertainClaim);
scenario.build(first, scenario.sharedCode);
assert.throws(
() => scenario.build(uncertain, scenario.sharedCode),
err => err.remoteIdentityClaimed === true && err.remoteCommitUncertain === true
);
const laterClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'Second Episode.mp4');
await assert.rejects(
() => laterClaim.runExclusive(async () => scenario.build(new scenario.Uploader(laterClaim), scenario.lateCode)),
err => err.remoteCommitUncertain === true
);
});
}
+72 -1
View File
@@ -1,7 +1,7 @@
const { describe, it } = require('node:test'); const { describe, it } = require('node:test');
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const { __test } = require('../lib/hosters'); const { __test, createRecoveryClaimRegistry } = require('../lib/hosters');
describe('hosters helpers', () => { describe('hosters helpers', () => {
it('extracts VOE file_code from nested result payloads', () => { it('extracts VOE file_code from nested result payloads', () => {
@@ -94,3 +94,74 @@ describe('hosters helpers', () => {
assert.equal(r.embed_url, 'https://byse.sx/e/GOOD123'); assert.equal(r.embed_url, 'https://byse.sx/e/GOOD123');
}); });
}); });
describe('recovery claim registry', () => {
it('claims remote codes across every title of one normalized hoster account', () => {
const registry = createRecoveryClaimRegistry();
const first = registry.forUpload(' VOE.SX ', 'ACCOUNT', 'First Episode.mkv');
const differentTitle = registry.forUpload('voe.sx', 'ACCOUNT', 'Second Episode.mp4');
const differentAccount = registry.forUpload('voe.sx', 'ACCOUNT-B', 'Second Episode.mp4');
assert.equal(first.reserve('REMOTE-CODE'), true);
assert.equal(differentTitle.reserve('REMOTE-CODE'), false);
assert.equal(differentAccount.reserve('REMOTE-CODE'), true);
});
it('serializes canonically equivalent titles without blocking an independent title', async () => {
const registry = createRecoveryClaimRegistry();
const composed = registry.forUpload('voe.sx', 'ACCOUNT', 'Café.mkv');
const decomposed = registry.forUpload('voe.sx', 'ACCOUNT', 'Cafe\u0301.mp4');
const independent = registry.forUpload('voe.sx', 'ACCOUNT', 'Other Episode.mkv');
const events = [];
let releaseFirst;
const firstGate = new Promise(resolve => {
releaseFirst = resolve;
});
assert.equal(composed.reserve('UNICODE-CODE'), true);
assert.equal(decomposed.reserve('UNICODE-CODE'), false);
const first = composed.runExclusive(async () => {
events.push('first-started');
await firstGate;
events.push('first-finished');
});
await new Promise(resolve => setImmediate(resolve));
const equivalent = decomposed.runExclusive(async () => {
events.push('equivalent-started');
});
const other = independent.runExclusive(async () => {
events.push('independent-started');
});
await new Promise(resolve => setImmediate(resolve));
assert.deepEqual(events, ['first-started', 'independent-started']);
releaseFirst();
await Promise.all([first, equivalent, other]);
assert.deepEqual(events, ['first-started', 'independent-started', 'first-finished', 'equivalent-started']);
});
it('fails closed for later jobs after a title becomes uncertain', async () => {
const registry = createRecoveryClaimRegistry();
const first = registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv');
const later = registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4');
const error = first.markUncertain(new Error('Remote commit could not be confirmed'));
assert.equal(error.remoteCommitUncertain, true);
assert.equal(error.hosterTransient, true);
await assert.rejects(
() => later.runExclusive(async () => 'unsafe-success'),
err => err.remoteCommitUncertain === true && err.hosterTransient === true
);
});
it('drops every claim when the registry is cleared', () => {
const registry = createRecoveryClaimRegistry();
const first = registry.forUpload('voe.sx', 'ACCOUNT', 'Episode.mkv');
assert.equal(first.reserve('REMOTE-CODE'), true);
registry.clear();
const nextBatch = registry.forUpload('voe.sx', 'ACCOUNT', 'Episode.mkv');
assert.equal(nextBatch.reserve('REMOTE-CODE'), true);
});
});
+174 -2
View File
@@ -56,12 +56,12 @@ function settings(hoster, parallelCount) {
}; };
} }
async function runBatch(manager, tasks) { async function runBatch(manager, tasks, options) {
let summary; let summary;
manager.once('batch-done', value => { manager.once('batch-done', value => {
summary = value; summary = value;
}); });
await manager.startBatch(tasks); await manager.startBatch(tasks, options);
return summary; return summary;
} }
@@ -88,6 +88,56 @@ function waitFor(promise, timeoutMs, message) {
]).finally(() => clearTimeout(timer)); ]).finally(() => clearTimeout(timer));
} }
async function assertTitleWaiterLeavesSlotAvailable(hosterParallel, globalSettings = {}) {
let releaseFirst;
let markFirstStarted;
let markIndependentStarted;
const firstGate = new Promise(resolve => {
releaseFirst = resolve;
});
const firstStarted = new Promise(resolve => {
markFirstStarted = resolve;
});
const independentStarted = new Promise(resolve => {
markIndependentStarted = resolve;
});
let sequence = 0;
loadManager(async (hoster, file) => {
if (file === firstPath) {
markFirstStarted();
await firstGate;
}
if (file === distinctPath) markIndependentStarted();
sequence++;
return {
file_code: `ADMISSION_${sequence}`,
download_url: `https://byse.sx/d/ADMISSION_${sequence}`
};
});
const manager = new UploadManager(settings('byse.sx', hosterParallel), globalSettings);
const batch = runBatch(manager, [
{ jobId: 'admission-first', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
]);
await waitFor(firstStarted, 500, 'First upload did not start');
const added = manager.addJobs([
{ jobId: 'admission-waiter', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
{ jobId: 'admission-independent', file: distinctPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
]);
assert.equal(added.added, 2);
let admissionError = null;
try {
await waitFor(independentStarted, 500, 'Independent title was blocked behind a title-lock waiter');
} catch (err) {
admissionError = err;
} finally {
releaseFirst();
}
const summary = await batch;
if (admissionError) throw admissionError;
assert.equal(summary.succeeded, 3);
}
test('a batch shares recovery claims across normalized same-name jobs', async () => { test('a batch shares recovery claims across normalized same-name jobs', async () => {
let unsafeCalls = 0; let unsafeCalls = 0;
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => { loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
@@ -172,6 +222,93 @@ test('normalized same-name recovery sections never overlap', async () => {
assert.equal(maximumActive, 1); assert.equal(maximumActive, 1);
}); });
test('a title-lock waiter does not consume a scarce upload slot', async () => {
await assertTitleWaiterLeavesSlotAvailable(2);
});
test('a title-lock waiter does not consume a scarce global upload slot', async () => {
await assertTitleWaiterLeavesSlotAvailable(3, { parallelUploadCount: 2 });
});
test('an uncertain remote commit blocks retries, account fallback, and later same-title success', async () => {
const calls = [];
let markFirstStarted;
let releaseUncertain;
const firstStarted = new Promise(resolve => {
markFirstStarted = resolve;
});
const uncertainGate = new Promise(resolve => {
releaseUncertain = resolve;
});
loadManager(async (hoster, file, apiKey) => {
calls.push({ file, apiKey });
if (file === firstPath) {
markFirstStarted();
await uncertainGate;
const error = new Error('Remote commit could not be confirmed');
error.remoteCommitUncertain = true;
throw error;
}
return {
file_code: 'LATE_REMOTE_CODE',
download_url: 'https://byse.sx/d/LATE_REMOTE_CODE'
};
});
const hosterSettings = settings('byse.sx', 2);
hosterSettings['byse.sx'].retries = 2;
const manager = new UploadManager(hosterSettings);
const fallback = { id: 'ACCOUNT_B', apiKey: 'ACCOUNT_KEY_B' };
const batch = runBatch(manager, [
{
jobId: 'uncertain-first',
file: firstPath,
hoster: 'byse.sx',
accountId: 'ACCOUNT_A',
apiKey: 'ACCOUNT_KEY_A'
}
], { primeOverrides: [['byse.sx', fallback]] });
await waitFor(firstStarted, 500, 'Uncertain predecessor did not start');
const added = manager.addJobs([
{
jobId: 'uncertain-later',
file: secondPath,
hoster: 'byse.sx',
accountId: 'ACCOUNT_A',
apiKey: 'ACCOUNT_KEY_A'
}
]);
assert.equal(added.added, 1);
releaseUncertain();
const summary = await batch;
assert.equal(summary.succeeded, 0);
assert.equal(summary.failed, 2);
assert.deepEqual(calls, [{ file: firstPath, apiKey: 'ACCOUNT_KEY_A' }]);
});
test('recovery claims do not leak into a later batch on the same manager', async () => {
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
if (!options.recoveryClaim.reserve('REUSED_BATCH_CODE')) {
const error = new Error('Remote recovery candidate already claimed');
error.hosterTransient = true;
throw error;
}
return {
file_code: 'REUSED_BATCH_CODE',
download_url: 'https://byse.sx/d/REUSED_BATCH_CODE'
};
});
const manager = new UploadManager(settings('byse.sx', 1));
const task = { jobId: 'batch-one', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' };
const first = await runBatch(manager, [task]);
const second = await runBatch(manager, [{ ...task, jobId: 'batch-two' }]);
assert.equal(first.succeeded, 1);
assert.equal(second.succeeded, 1);
});
for (const scenario of [ for (const scenario of [
{ {
label: 'VOE', label: 'VOE',
@@ -263,6 +400,41 @@ for (const scenario of [
); );
}); });
test(`${scenario.label} uploader instances reject one direct remote code across different titles`, async () => {
await withUploaderMethods(
scenario.Uploader,
async function () {
await new Promise(resolve => setImmediate(resolve));
return scenario.buildResult(this, scenario.sharedCode);
},
async () => {
loadManager();
const manager = new UploadManager(settings(scenario.hoster, 2));
const summary = await runBatch(manager, [
{
jobId: `${scenario.label}-different-title-a`,
file: firstPath,
hoster: scenario.hoster,
accountId: 'LOGIN_ACCOUNT',
username: 'account@example.test',
password: 'password'
},
{
jobId: `${scenario.label}-different-title-b`,
file: distinctPath,
hoster: scenario.hoster,
accountId: 'LOGIN_ACCOUNT',
username: 'account@example.test',
password: 'password'
}
]);
assert.equal(summary.succeeded, 1);
assert.equal(summary.failed, 1);
}
);
});
test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => { test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => {
let active = 0; let active = 0;
let maximumActive = 0; let maximumActive = 0;