Harden batch-wide recovery identity claims
Scope remote code ownership to normalized hoster and account identities while retaining title-only recovery serialization and canonical Unicode title matching. Mark post-upload ambiguity and duplicate identities as uncertain so retries, account fallback, and later same-title jobs fail closed instead of reporting unsafe success. Acquire recovery title leases before hoster and global semaphores, revalidate failed-account overrides before upload, and clear claim state at batch boundaries. Add deterministic concurrent coverage for same-code rejection, distinct-code parallel success, uncertainty propagation, semaphore fairness, account isolation, Unicode equivalence, and registry lifetime.
This commit is contained in:
+122
-34
@@ -575,7 +575,40 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
||||
}
|
||||
|
||||
function _normalizeFileTitle(s) {
|
||||
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
||||
return String(s || '')
|
||||
.normalize('NFKD')
|
||||
.toLowerCase()
|
||||
.replace(/\.[\p{Letter}\p{Number}]+$/u, '')
|
||||
.replace(/\p{Mark}+/gu, '')
|
||||
.replace(/[^\p{Letter}\p{Number}]+/gu, '');
|
||||
}
|
||||
|
||||
function _normalizeRecoveryHoster(value) {
|
||||
return String(value || '').normalize('NFKC').trim().toLowerCase();
|
||||
}
|
||||
|
||||
function _normalizeRecoveryAccount(value) {
|
||||
return String(value || '').normalize('NFKC').trim();
|
||||
}
|
||||
|
||||
function _createRecoveryUncertainError() {
|
||||
const error = new Error('Upload-Ergebnis für diesen Titel ist wegen eines möglichen Remote-Commits unsicher');
|
||||
error.remoteCommitUncertain = true;
|
||||
error.hosterTransient = true;
|
||||
return error;
|
||||
}
|
||||
|
||||
function _markRecoveryUncertain(recoveryClaim, error) {
|
||||
if (!recoveryClaim) return error;
|
||||
if (typeof recoveryClaim.markUncertain === 'function') {
|
||||
return recoveryClaim.markUncertain(error);
|
||||
}
|
||||
const uncertainError = error && typeof error === 'object'
|
||||
? error
|
||||
: _createRecoveryUncertainError();
|
||||
uncertainError.remoteCommitUncertain = true;
|
||||
uncertainError.hosterTransient = true;
|
||||
return uncertainError;
|
||||
}
|
||||
|
||||
function _createAbortError() {
|
||||
@@ -607,40 +640,69 @@ function _waitForRecoveryTurn(predecessor, signal) {
|
||||
}
|
||||
|
||||
function createRecoveryClaimRegistry() {
|
||||
const scopes = new Map();
|
||||
const accounts = new Map();
|
||||
let nextClaimId = 1;
|
||||
return {
|
||||
forUpload(hosterName, apiKey, fileName) {
|
||||
const identity = crypto.createHash('sha256')
|
||||
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`)
|
||||
const accountIdentity = crypto.createHash('sha256')
|
||||
.update(`${_normalizeRecoveryHoster(hosterName)}\0${_normalizeRecoveryAccount(apiKey)}`)
|
||||
.digest('hex');
|
||||
let codes = scopes.get(identity);
|
||||
if (!codes) {
|
||||
codes = {
|
||||
values: new Set(),
|
||||
tail: Promise.resolve()
|
||||
let account = accounts.get(accountIdentity);
|
||||
if (!account) {
|
||||
account = {
|
||||
codes: new Map(),
|
||||
titles: new Map()
|
||||
};
|
||||
scopes.set(identity, codes);
|
||||
accounts.set(accountIdentity, account);
|
||||
}
|
||||
const titleIdentity = _normalizeFileTitle(fileName);
|
||||
let title = account.titles.get(titleIdentity);
|
||||
if (!title) {
|
||||
title = {
|
||||
tail: Promise.resolve(),
|
||||
uncertain: false
|
||||
};
|
||||
account.titles.set(titleIdentity, title);
|
||||
}
|
||||
const claimId = nextClaimId++;
|
||||
return {
|
||||
has(code) {
|
||||
return codes.values.has(String(code || '').trim());
|
||||
return account.codes.has(String(code || '').trim());
|
||||
},
|
||||
reserve(code) {
|
||||
const normalized = String(code || '').trim();
|
||||
if (!normalized || codes.values.has(normalized)) return false;
|
||||
codes.values.add(normalized);
|
||||
if (!normalized) return false;
|
||||
if (account.codes.has(normalized)) {
|
||||
return account.codes.get(normalized) === claimId;
|
||||
}
|
||||
account.codes.set(normalized, claimId);
|
||||
return true;
|
||||
},
|
||||
markUncertain(error) {
|
||||
title.uncertain = true;
|
||||
const uncertainError = error && typeof error === 'object'
|
||||
? error
|
||||
: _createRecoveryUncertainError();
|
||||
uncertainError.remoteCommitUncertain = true;
|
||||
uncertainError.hosterTransient = true;
|
||||
return uncertainError;
|
||||
},
|
||||
isUncertain() {
|
||||
return title.uncertain;
|
||||
},
|
||||
async runExclusive(operation, signal) {
|
||||
const predecessor = codes.tail;
|
||||
const predecessor = title.tail;
|
||||
let release;
|
||||
const current = new Promise(resolve => {
|
||||
release = resolve;
|
||||
});
|
||||
codes.tail = predecessor.then(() => current);
|
||||
title.tail = predecessor.then(() => current, () => current);
|
||||
try {
|
||||
await _waitForRecoveryTurn(predecessor, signal);
|
||||
return await operation();
|
||||
if (title.uncertain) throw _createRecoveryUncertainError();
|
||||
const result = await operation();
|
||||
if (title.uncertain) throw _createRecoveryUncertainError();
|
||||
return result;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
@@ -648,7 +710,7 @@ function createRecoveryClaimRegistry() {
|
||||
};
|
||||
},
|
||||
clear() {
|
||||
scopes.clear();
|
||||
accounts.clear();
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -805,23 +867,28 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
bodyTimeout: UPLOAD_TIMEOUT
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
throw createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
|
||||
const error = signal && signal.aborted ? err : createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
|
||||
phase: 'upload-request',
|
||||
endpoint: targetUrl,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, error);
|
||||
}
|
||||
|
||||
const { body, statusCode, headers } = uploadResponse;
|
||||
|
||||
const rawBody = await body.text();
|
||||
let rawBody;
|
||||
try {
|
||||
rawBody = await body.text();
|
||||
} catch (err) {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||
}
|
||||
let payload = null;
|
||||
try {
|
||||
payload = rawBody ? JSON.parse(rawBody) : {};
|
||||
} catch {
|
||||
throw createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
@@ -829,7 +896,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
body: rawBody,
|
||||
retryable: statusCode >= 500,
|
||||
transientNetwork: statusCode >= 500
|
||||
});
|
||||
}));
|
||||
}
|
||||
// Normalize valid-but-not-object JSON (JSON.parse('null') → null;
|
||||
// JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this
|
||||
@@ -843,7 +910,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
}
|
||||
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
throw createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
|
||||
const error = createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
@@ -852,10 +919,13 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
retryable: statusCode === 429 || statusCode >= 500,
|
||||
transientNetwork: statusCode >= 500
|
||||
});
|
||||
throw statusCode >= 500
|
||||
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
|
||||
: error;
|
||||
}
|
||||
|
||||
if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
|
||||
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||
const error = createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: Number(payload.status),
|
||||
@@ -864,6 +934,9 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
retryable: Number(payload.status) === 429 || Number(payload.status) >= 500,
|
||||
transientNetwork: Number(payload.status) >= 500
|
||||
});
|
||||
throw Number(payload.status) >= 500
|
||||
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
|
||||
: error;
|
||||
}
|
||||
|
||||
let result = null;
|
||||
@@ -885,7 +958,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
||||
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
|
||||
if (!opts.recoveryClaim.reserve(result.file_code)) {
|
||||
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
||||
const error = createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
@@ -894,6 +967,8 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
retryable: true,
|
||||
hosterTransient: true
|
||||
});
|
||||
error.remoteIdentityClaimed = true;
|
||||
throw _markRecoveryUncertain(opts.recoveryClaim, error);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
@@ -921,8 +996,12 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// even after our uploader gave up.
|
||||
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
||||
const fileName = path.basename(filePath);
|
||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
try {
|
||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
} catch (err) {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Doodstream: the doodapi upload POST returned no filecode (the same backend
|
||||
@@ -930,21 +1009,29 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// the file did register, claim its code instead of failing the upload.
|
||||
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
||||
const fileName = path.basename(filePath);
|
||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
try {
|
||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
||||
if (polled) return polled;
|
||||
} catch (err) {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||
}
|
||||
}
|
||||
|
||||
if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) {
|
||||
byseBaselineError.hosterTransient = true;
|
||||
throw byseBaselineError;
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, byseBaselineError);
|
||||
}
|
||||
|
||||
if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) {
|
||||
doodBaselineError.hosterTransient = true;
|
||||
throw doodBaselineError;
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, doodBaselineError);
|
||||
}
|
||||
|
||||
if (parseErr) throw parseErr;
|
||||
if (parseErr) {
|
||||
throw explicitlyRejected
|
||||
? parseErr
|
||||
: _markRecoveryUncertain(opts && opts.recoveryClaim, parseErr);
|
||||
}
|
||||
|
||||
if (payload.success === false) {
|
||||
throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, {
|
||||
@@ -970,7 +1057,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
// so tag it hosterTransient: the upload-manager then fails this file WITHOUT
|
||||
// blacklisting the account (same protection the web path got in 3.3.29) and
|
||||
// the account stays usable for the next retry/batch.
|
||||
throw createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
|
||||
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
|
||||
phase: 'upload-result',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
@@ -978,7 +1065,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
||||
body: rawBody,
|
||||
retryable: true,
|
||||
hosterTransient: true
|
||||
});
|
||||
}));
|
||||
}
|
||||
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||
phase: 'upload-result',
|
||||
@@ -1007,6 +1094,7 @@ module.exports = {
|
||||
uploadFile,
|
||||
prefetchBaseline,
|
||||
createRecoveryClaimRegistry,
|
||||
normalizeRecoveryTitle: _normalizeFileTitle,
|
||||
HOSTER_CONFIGS,
|
||||
__test: {
|
||||
extractUploadServerUrl,
|
||||
|
||||
+157
-73
@@ -71,7 +71,7 @@ class UploadManager extends EventEmitter {
|
||||
this._suspectGoodAccounts.clear();
|
||||
this._doodApiKeyCache.clear();
|
||||
this._baselineCache.clear();
|
||||
this._recoveryClaims.clear();
|
||||
if (!this.running) this._recoveryClaims.clear();
|
||||
}
|
||||
|
||||
switchAccount(hoster, fallbackAccount) {
|
||||
@@ -134,6 +134,41 @@ class UploadManager extends EventEmitter {
|
||||
return true;
|
||||
}
|
||||
|
||||
_swapFailedAccount(task, jobId, fileName) {
|
||||
if (!task.accountId || !this._failedAccounts.has(task.hoster + ':' + task.accountId)) return false;
|
||||
const override = this._accountOverrides.get(task.hoster);
|
||||
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
|
||||
this._rotLog('pre-job-swap', {
|
||||
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
|
||||
});
|
||||
task.accountId = override.id;
|
||||
task.username = override.username;
|
||||
task.password = override.password;
|
||||
task.apiKey = override.apiKey;
|
||||
return true;
|
||||
}
|
||||
this._rotLog('pre-job-swap-blocked', {
|
||||
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
|
||||
hasOverride: !!override,
|
||||
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
async _coordinateAccountFailure(task, err, signal, jobId) {
|
||||
if (!task.accountId || (err && err.remoteCommitUncertain === true)) return;
|
||||
if (!this._shouldSkipRetryOnAccountError(err)) return;
|
||||
const key = task.hoster + ':' + task.accountId;
|
||||
if (this._failedAccounts.has(key)) return;
|
||||
this._failedAccounts.set(key, true);
|
||||
this._rotLog('mark-failed', {
|
||||
jobId, hoster: task.hoster, fileName: path.basename(task.file),
|
||||
accountId: task.accountId, lastError: err && err.message ? err.message : String(err)
|
||||
});
|
||||
this.emit('account-failed', { hoster: task.hoster, accountId: task.accountId });
|
||||
await this._sleep(800, signal);
|
||||
}
|
||||
|
||||
_rotLog(event, data) {
|
||||
this.emit('rot-log', { ts: Date.now(), event, ...data });
|
||||
}
|
||||
@@ -161,6 +196,7 @@ class UploadManager extends EventEmitter {
|
||||
// which takes priority in _shouldSkipRetryOnAccountError.
|
||||
_isFileRejectedError(err) {
|
||||
if (!err) return false;
|
||||
if (err.remoteCommitUncertain === true) return false;
|
||||
if (err.transientNetwork === true) return false;
|
||||
if (err.accountError === true) return false; // explicit account-level wins
|
||||
if (err.fileRejected === true) return true;
|
||||
@@ -355,6 +391,7 @@ class UploadManager extends EventEmitter {
|
||||
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
||||
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
||||
this._recoveryClaims.clear();
|
||||
this._recoveryClaims = createRecoveryClaimRegistry();
|
||||
this.semaphores = {};
|
||||
this.globalSemaphore = null;
|
||||
this.globalThrottle = null;
|
||||
@@ -440,13 +477,12 @@ class UploadManager extends EventEmitter {
|
||||
files
|
||||
};
|
||||
|
||||
this._recoveryClaims.clear();
|
||||
this.emit('batch-done', summary);
|
||||
}
|
||||
|
||||
async _runJob(task, results, batchSignal) {
|
||||
const settings = this._getSettings(task.hoster);
|
||||
const hosterSemaphore = this._getSemaphore(task.hoster);
|
||||
const globalSemaphore = this._getGlobalSemaphore();
|
||||
const uploadId = crypto.randomBytes(8).toString('hex');
|
||||
const jobId = task.jobId || uploadId;
|
||||
const fileName = path.basename(task.file);
|
||||
@@ -466,8 +502,6 @@ class UploadManager extends EventEmitter {
|
||||
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
|
||||
this.jobAbortControllers.set(jobId, jobAbortController);
|
||||
|
||||
let hosterSlotAcquired = false;
|
||||
let globalSlotAcquired = false;
|
||||
let finalResultRecorded = false;
|
||||
let finalStatus = 'error';
|
||||
let lastError = null;
|
||||
@@ -549,9 +583,6 @@ class UploadManager extends EventEmitter {
|
||||
// queueJobs array; the first event it actually needs from main is the
|
||||
// 'getting-server' / 'uploading' transition for the jobs that the
|
||||
// semaphore lets through.
|
||||
await hosterSemaphore.acquire(signal);
|
||||
hosterSlotAcquired = true;
|
||||
|
||||
let fileProbe = null;
|
||||
try {
|
||||
fileProbe = await probeFileHead(task.file, 512);
|
||||
@@ -566,11 +597,6 @@ class UploadManager extends EventEmitter {
|
||||
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
|
||||
});
|
||||
|
||||
if (globalSemaphore) {
|
||||
await globalSemaphore.acquire(signal);
|
||||
globalSlotAcquired = true;
|
||||
}
|
||||
|
||||
if (settings.timeIntervalSec > 0) {
|
||||
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal);
|
||||
}
|
||||
@@ -580,24 +606,7 @@ class UploadManager extends EventEmitter {
|
||||
// of burning a guaranteed-to-fail upload attempt. Critical at scale:
|
||||
// with 500 queued jobs and 1 parallel slot, without this check every
|
||||
// job still hits the original dead account first.
|
||||
if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) {
|
||||
const override = this._accountOverrides.get(task.hoster);
|
||||
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
|
||||
this._rotLog('pre-job-swap', {
|
||||
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
|
||||
});
|
||||
task.accountId = override.id;
|
||||
task.username = override.username;
|
||||
task.password = override.password;
|
||||
task.apiKey = override.apiKey;
|
||||
} else {
|
||||
this._rotLog('pre-job-swap-blocked', {
|
||||
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
|
||||
hasOverride: !!override,
|
||||
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
|
||||
});
|
||||
}
|
||||
}
|
||||
this._swapFailedAccount(task, jobId, fileName);
|
||||
|
||||
// A previous file of at least this size already got a suspect rejection
|
||||
// on this exact account — skip the guaranteed-to-fail multi-GB upload
|
||||
@@ -735,7 +744,7 @@ class UploadManager extends EventEmitter {
|
||||
} catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ }
|
||||
};
|
||||
|
||||
const result = await this._executeUpload(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe);
|
||||
const result = await this._executeUploadWithAdmission(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe, true, jobId);
|
||||
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
|
||||
@@ -784,6 +793,11 @@ class UploadManager extends EventEmitter {
|
||||
break;
|
||||
}
|
||||
|
||||
if (err && err.remoteCommitUncertain === true) {
|
||||
lastError = err;
|
||||
break;
|
||||
}
|
||||
|
||||
if (isSpeedRestart && attempt < maxAttempts) {
|
||||
lastError = new Error('Geschwindigkeit zu niedrig - Neustart');
|
||||
await this._sleep(3000, signal);
|
||||
@@ -1047,7 +1061,7 @@ class UploadManager extends EventEmitter {
|
||||
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
|
||||
: hosterThrottle || globalThrottle;
|
||||
|
||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
||||
const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, true, jobId);
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.sessionBytes += fileSize;
|
||||
@@ -1104,9 +1118,6 @@ class UploadManager extends EventEmitter {
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.jobAbortControllers.delete(jobId);
|
||||
cleanupSignals();
|
||||
// Release in reverse order of acquire (global first, then hoster)
|
||||
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
|
||||
if (hosterSlotAcquired) hosterSemaphore.release();
|
||||
this.emit('job-settled', {
|
||||
jobId,
|
||||
sourceCleanupToken: task.sourceCleanupToken || null,
|
||||
@@ -1190,7 +1201,7 @@ class UploadManager extends EventEmitter {
|
||||
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
|
||||
: hosterThrottle || globalThrottle;
|
||||
try {
|
||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
||||
const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, false, jobId);
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.sessionBytes += fileSize;
|
||||
@@ -1210,6 +1221,7 @@ class UploadManager extends EventEmitter {
|
||||
});
|
||||
}
|
||||
if (signal.aborted || this.stopAfterActive) break;
|
||||
if (err && err.remoteCommitUncertain === true) throw err;
|
||||
if (err && err.suspectReject === true) {
|
||||
this._noteSuspectReject(task.hoster, account.id, fileSize);
|
||||
}
|
||||
@@ -1241,16 +1253,95 @@ class UploadManager extends EventEmitter {
|
||||
return null;
|
||||
}
|
||||
|
||||
async _executeUpload(task, progressCb, signal, throttle, fileProbe) {
|
||||
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe);
|
||||
return assertUploadConfirmation(result, task.hoster);
|
||||
async _executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, coordinateAccountFailure = true, jobId = task.jobId) {
|
||||
while (true) {
|
||||
const context = await this._createRecoveryContext(task);
|
||||
let retryAdmission = false;
|
||||
const operation = async () => {
|
||||
const hosterSemaphore = this._getSemaphore(task.hoster);
|
||||
const globalSemaphore = this._getGlobalSemaphore();
|
||||
let hosterSlotAcquired = false;
|
||||
let globalSlotAcquired = false;
|
||||
try {
|
||||
await hosterSemaphore.acquire(signal);
|
||||
hosterSlotAcquired = true;
|
||||
if (globalSemaphore) {
|
||||
await globalSemaphore.acquire(signal);
|
||||
globalSlotAcquired = true;
|
||||
}
|
||||
if (this._swapFailedAccount(task, jobId, path.basename(task.file))) {
|
||||
retryAdmission = true;
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return await this._executeUpload(task, progressCb, signal, throttle, fileProbe, context);
|
||||
} catch (err) {
|
||||
if (coordinateAccountFailure) await this._coordinateAccountFailure(task, err, signal, jobId);
|
||||
if (context.recoveryClaim && err && err.remoteCommitUncertain === true) {
|
||||
throw context.recoveryClaim.markUncertain(err);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
} finally {
|
||||
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
|
||||
if (hosterSlotAcquired) hosterSemaphore.release();
|
||||
}
|
||||
};
|
||||
const result = context.recoveryClaim
|
||||
? await context.recoveryClaim.runExclusive(operation, signal)
|
||||
: await operation();
|
||||
if (retryAdmission) continue;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) {
|
||||
async _executeUpload(task, progressCb, signal, throttle, fileProbe, context) {
|
||||
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context);
|
||||
let confirmed;
|
||||
try {
|
||||
confirmed = assertUploadConfirmation(result, task.hoster);
|
||||
} catch (err) {
|
||||
if (context.recoveryClaim) throw context.recoveryClaim.markUncertain(err);
|
||||
throw err;
|
||||
}
|
||||
return confirmed;
|
||||
}
|
||||
|
||||
async _createRecoveryContext(task) {
|
||||
const fileName = path.basename(task.file);
|
||||
if ((task.hoster === 'vidmoly.me' || task.hoster === 'voe.sx') && task.username) {
|
||||
const accountIdentity = task.accountId !== null && task.accountId !== undefined
|
||||
? task.accountId
|
||||
: String(task.username || '').trim().toLowerCase();
|
||||
return {
|
||||
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
|
||||
doodApiKey: null
|
||||
};
|
||||
}
|
||||
if (task.hoster === 'doodstream.com' && task.username) {
|
||||
const doodApiKey = await this._resolveDoodstreamApiKey(task);
|
||||
const accountIdentity = doodApiKey || (task.accountId !== null && task.accountId !== undefined
|
||||
? task.accountId
|
||||
: String(task.username || '').trim().toLowerCase());
|
||||
return {
|
||||
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
|
||||
doodApiKey
|
||||
};
|
||||
}
|
||||
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
||||
return {
|
||||
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, task.apiKey, fileName),
|
||||
doodApiKey: null
|
||||
};
|
||||
}
|
||||
return { recoveryClaim: null, doodApiKey: null };
|
||||
}
|
||||
|
||||
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context) {
|
||||
if (task.hoster === 'vidmoly.me' && task.username) {
|
||||
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle);
|
||||
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle, context.recoveryClaim);
|
||||
} else if (task.hoster === 'voe.sx' && task.username) {
|
||||
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle);
|
||||
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle, context.recoveryClaim);
|
||||
} else if (task.hoster === 'doodstream.com' && task.username) {
|
||||
// Login-path reliability fix: the web-form upload returns the filecode in
|
||||
// an HTML form that comes back empty for large files (doodstream backend
|
||||
@@ -1258,54 +1349,47 @@ class UploadManager extends EventEmitter {
|
||||
// session ONCE per batch and upload via the official API instead — it
|
||||
// returns result[0].filecode directly and has no empty-form failure mode.
|
||||
// Falls back to the web-form upload if no valid key can be derived.
|
||||
const apiKey = await this._resolveDoodstreamApiKey(task);
|
||||
const apiKey = context.doodApiKey;
|
||||
if (apiKey) {
|
||||
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe);
|
||||
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
|
||||
}
|
||||
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||
const dood = new DoodstreamUploader();
|
||||
await dood.login(task.username, task.password);
|
||||
return dood.upload(task.file, progressCb, signal, throttle);
|
||||
const result = await dood.upload(task.file, progressCb, signal, throttle);
|
||||
if (result && result.file_code && !context.recoveryClaim.reserve(result.file_code)) {
|
||||
const error = new Error('Doodstream Upload lieferte eine bereits zugeordnete Remote-Identität');
|
||||
error.remoteIdentityClaimed = true;
|
||||
throw context.recoveryClaim.markUncertain(error);
|
||||
}
|
||||
return result;
|
||||
} else if (task.hoster === 'clouddrop.cc') {
|
||||
const clouddrop = new ClouddropUploader(task.apiKey);
|
||||
return clouddrop.upload(task.file, progressCb, signal, throttle);
|
||||
} else {
|
||||
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
||||
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe);
|
||||
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
|
||||
}
|
||||
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
|
||||
}
|
||||
}
|
||||
|
||||
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle) {
|
||||
const accountIdentity = task.accountId !== null && task.accountId !== undefined
|
||||
? task.accountId
|
||||
: String(task.username || '').trim().toLowerCase();
|
||||
const recoveryClaim = this._recoveryClaims.forUpload(
|
||||
task.hoster,
|
||||
accountIdentity,
|
||||
path.basename(task.file)
|
||||
);
|
||||
return recoveryClaim.runExclusive(async () => {
|
||||
const uploader = new UploaderClass(recoveryClaim);
|
||||
await uploader.login(task.username, task.password);
|
||||
return uploader.upload(task.file, progressCb, signal, throttle);
|
||||
}, signal);
|
||||
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle, recoveryClaim) {
|
||||
const uploader = new UploaderClass(recoveryClaim);
|
||||
await uploader.login(task.username, task.password);
|
||||
return uploader.upload(task.file, progressCb, signal, throttle);
|
||||
}
|
||||
|
||||
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
|
||||
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
|
||||
return recoveryClaim.runExclusive(async () => {
|
||||
const options = { recoveryClaim };
|
||||
if (hosterName === 'byse.sx') {
|
||||
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
|
||||
} else {
|
||||
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||
}
|
||||
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
|
||||
}, signal);
|
||||
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe, recoveryClaim) {
|
||||
const options = { recoveryClaim };
|
||||
if (hosterName === 'byse.sx') {
|
||||
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
|
||||
} else {
|
||||
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||
}
|
||||
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
|
||||
}
|
||||
|
||||
_getBaseline(hosterName, apiKey, signal) {
|
||||
|
||||
+38
-16
@@ -3,6 +3,7 @@ const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { request } = require('undici');
|
||||
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
||||
const { normalizeRecoveryTitle } = require('./hosters');
|
||||
|
||||
const BASE_URL = 'https://vidmoly.me';
|
||||
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
||||
@@ -275,13 +276,13 @@ class VidmolyUploader {
|
||||
bodyTimeout: UPLOAD_TIMEOUT
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
throw createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
|
||||
const error = signal && signal.aborted ? err : createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
|
||||
phase: 'upload-request',
|
||||
endpoint: targetUrl,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
throw this._markRemoteCommitUncertain(error);
|
||||
}
|
||||
|
||||
const { body, statusCode, headers } = uploadResponse;
|
||||
@@ -295,17 +296,25 @@ class VidmolyUploader {
|
||||
// Always drain the original body to prevent connection leak
|
||||
try { await body.text(); } catch {}
|
||||
if (location) {
|
||||
const resultRes = await this._fetch(new URL(location, uploadUrl).href);
|
||||
resultHtml = await resultRes.text();
|
||||
try {
|
||||
const resultRes = await this._fetch(new URL(location, uploadUrl).href);
|
||||
resultHtml = await resultRes.text();
|
||||
} catch (err) {
|
||||
throw this._markRemoteCommitUncertain(err);
|
||||
}
|
||||
} else {
|
||||
resultHtml = '';
|
||||
}
|
||||
} else {
|
||||
resultHtml = await body.text();
|
||||
try {
|
||||
resultHtml = await body.text();
|
||||
} catch (err) {
|
||||
throw this._markRemoteCommitUncertain(err);
|
||||
}
|
||||
}
|
||||
|
||||
if (statusCode >= 400) {
|
||||
throw createTransportError('Vidmoly Upload fehlgeschlagen', {
|
||||
const error = createTransportError('Vidmoly Upload fehlgeschlagen', {
|
||||
phase: 'upload-response',
|
||||
endpoint: targetUrl,
|
||||
httpStatus: statusCode,
|
||||
@@ -314,6 +323,7 @@ class VidmolyUploader {
|
||||
retryable: statusCode === 429 || statusCode >= 500,
|
||||
transientNetwork: statusCode >= 500
|
||||
});
|
||||
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
|
||||
}
|
||||
|
||||
// Try JSON first. The current transit server returns
|
||||
@@ -355,23 +365,35 @@ class VidmolyUploader {
|
||||
} catch (primaryErr) {
|
||||
if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr;
|
||||
if (baselineCodes) {
|
||||
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
|
||||
if (fallback) return fallback;
|
||||
try {
|
||||
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
|
||||
if (fallback) return fallback;
|
||||
} catch (err) {
|
||||
throw this._markRemoteCommitUncertain(err);
|
||||
}
|
||||
}
|
||||
if (baselineError) {
|
||||
baselineError.hosterTransient = true;
|
||||
throw baselineError;
|
||||
throw this._markRemoteCommitUncertain(baselineError);
|
||||
}
|
||||
throw primaryErr;
|
||||
throw this._markRemoteCommitUncertain(primaryErr);
|
||||
}
|
||||
}
|
||||
|
||||
_normalizeTitle(value) {
|
||||
return String(value || '')
|
||||
.toLowerCase()
|
||||
.normalize('NFKD')
|
||||
.replace(/\.[a-z0-9]+$/i, '')
|
||||
.replace(/[^a-z0-9]+/g, '');
|
||||
return normalizeRecoveryTitle(value);
|
||||
}
|
||||
|
||||
_markRemoteCommitUncertain(error) {
|
||||
if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
|
||||
return this.recoveryClaim.markUncertain(error);
|
||||
}
|
||||
const uncertainError = error && typeof error === 'object'
|
||||
? error
|
||||
: new Error('Vidmoly Upload-Ergebnis ist unsicher');
|
||||
uncertainError.remoteCommitUncertain = true;
|
||||
uncertainError.hosterTransient = true;
|
||||
return uncertainError;
|
||||
}
|
||||
|
||||
_buildUrlsFromCode(fileCode, phase = 'upload-result') {
|
||||
@@ -387,7 +409,7 @@ class VidmolyUploader {
|
||||
hosterTransient: true
|
||||
});
|
||||
error.remoteIdentityClaimed = true;
|
||||
throw error;
|
||||
throw this._markRemoteCommitUncertain(error);
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
+34
-15
@@ -3,6 +3,7 @@ const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { request } = require('undici');
|
||||
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
||||
const { normalizeRecoveryTitle } = require('./hosters');
|
||||
|
||||
const BASE_URL = 'https://voe.sx';
|
||||
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
||||
@@ -352,22 +353,27 @@ class VoeUploader {
|
||||
bodyTimeout: UPLOAD_TIMEOUT
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal && signal.aborted) throw err;
|
||||
throw createTransportError('VOE Upload konnte nicht übertragen werden', {
|
||||
const error = signal && signal.aborted ? err : createTransportError('VOE Upload konnte nicht übertragen werden', {
|
||||
phase: 'upload-request',
|
||||
endpoint: uploadServer,
|
||||
retryable: true,
|
||||
transientNetwork: true
|
||||
});
|
||||
throw this._markRemoteCommitUncertain(error);
|
||||
}
|
||||
|
||||
const { body, headers, statusCode } = uploadResponse;
|
||||
|
||||
this._parseCookiesFromHeaders(headers || {});
|
||||
|
||||
const rawBody = await body.text();
|
||||
let rawBody;
|
||||
try {
|
||||
rawBody = await body.text();
|
||||
} catch (err) {
|
||||
throw this._markRemoteCommitUncertain(err);
|
||||
}
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
throw createTransportError('VOE Upload fehlgeschlagen', {
|
||||
const error = createTransportError('VOE Upload fehlgeschlagen', {
|
||||
phase: 'upload-response',
|
||||
endpoint: uploadServer,
|
||||
httpStatus: statusCode,
|
||||
@@ -376,6 +382,7 @@ class VoeUploader {
|
||||
retryable: statusCode === 429 || statusCode >= 500,
|
||||
transientNetwork: statusCode >= 500
|
||||
});
|
||||
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
|
||||
}
|
||||
|
||||
// Try JSON response
|
||||
@@ -407,23 +414,27 @@ class VoeUploader {
|
||||
|
||||
// Fallback: poll the file list to find the newly uploaded file
|
||||
if (baselineCodes) {
|
||||
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
|
||||
if (result) return result;
|
||||
try {
|
||||
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
|
||||
if (result) return result;
|
||||
} catch (err) {
|
||||
throw this._markRemoteCommitUncertain(err);
|
||||
}
|
||||
}
|
||||
|
||||
if (baselineError) {
|
||||
baselineError.hosterTransient = true;
|
||||
throw baselineError;
|
||||
throw this._markRemoteCommitUncertain(baselineError);
|
||||
}
|
||||
|
||||
throw createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
|
||||
throw this._markRemoteCommitUncertain(createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
|
||||
phase: 'upload-result',
|
||||
endpoint: uploadServer,
|
||||
contentType: headers && headers['content-type'],
|
||||
body: rawBody,
|
||||
hosterTransient: true,
|
||||
retryable: true
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
async _resolveUploadedFile(fileName, baselineCodes, signal) {
|
||||
@@ -478,11 +489,19 @@ class VoeUploader {
|
||||
}
|
||||
|
||||
_normalizeTitle(value) {
|
||||
return String(value || '')
|
||||
.toLowerCase()
|
||||
.normalize('NFKD')
|
||||
.replace(/\.[a-z0-9]+$/i, '')
|
||||
.replace(/[^a-z0-9]+/g, '');
|
||||
return normalizeRecoveryTitle(value);
|
||||
}
|
||||
|
||||
_markRemoteCommitUncertain(error) {
|
||||
if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
|
||||
return this.recoveryClaim.markUncertain(error);
|
||||
}
|
||||
const uncertainError = error && typeof error === 'object'
|
||||
? error
|
||||
: new Error('VOE Upload-Ergebnis ist unsicher');
|
||||
uncertainError.remoteCommitUncertain = true;
|
||||
uncertainError.hosterTransient = true;
|
||||
return uncertainError;
|
||||
}
|
||||
|
||||
_buildUrls(fileCode, phase = 'upload-result') {
|
||||
@@ -498,7 +517,7 @@ class VoeUploader {
|
||||
hosterTransient: true
|
||||
});
|
||||
error.remoteIdentityClaimed = true;
|
||||
throw error;
|
||||
throw this._markRemoteCommitUncertain(error);
|
||||
}
|
||||
return {
|
||||
download_url: `${BASE_URL}/${code}`,
|
||||
|
||||
@@ -187,3 +187,46 @@ test('Vidmoly concurrent same-name recovery accepts distinct remote codes', asyn
|
||||
|
||||
assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']);
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
{
|
||||
label: 'VOE',
|
||||
hoster: 'voe.sx',
|
||||
Uploader: VoeUploader,
|
||||
sharedCode: 'VOE_UNCERTAIN',
|
||||
lateCode: 'VOE_LATE_CODE',
|
||||
build(uploader, code) {
|
||||
return uploader._buildUrls(code);
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Vidmoly',
|
||||
hoster: 'vidmoly.me',
|
||||
Uploader: VidmolyUploader,
|
||||
sharedCode: 'VIDUNCERTAIN',
|
||||
lateCode: 'VIDLATECODE1',
|
||||
build(uploader, code) {
|
||||
return uploader._buildUrlsFromCode(code);
|
||||
}
|
||||
}
|
||||
]) {
|
||||
test(`${scenario.label} marks a duplicate direct identity uncertain and blocks a later title match`, async () => {
|
||||
const registry = createRecoveryClaimRegistry();
|
||||
const firstClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'First Episode.mkv');
|
||||
const uncertainClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'Second Episode.mkv');
|
||||
const first = new scenario.Uploader(firstClaim);
|
||||
const uncertain = new scenario.Uploader(uncertainClaim);
|
||||
|
||||
scenario.build(first, scenario.sharedCode);
|
||||
assert.throws(
|
||||
() => scenario.build(uncertain, scenario.sharedCode),
|
||||
err => err.remoteIdentityClaimed === true && err.remoteCommitUncertain === true
|
||||
);
|
||||
|
||||
const laterClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'Second Episode.mp4');
|
||||
await assert.rejects(
|
||||
() => laterClaim.runExclusive(async () => scenario.build(new scenario.Uploader(laterClaim), scenario.lateCode)),
|
||||
err => err.remoteCommitUncertain === true
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
+72
-1
@@ -1,7 +1,7 @@
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const { __test } = require('../lib/hosters');
|
||||
const { __test, createRecoveryClaimRegistry } = require('../lib/hosters');
|
||||
|
||||
describe('hosters helpers', () => {
|
||||
it('extracts VOE file_code from nested result payloads', () => {
|
||||
@@ -94,3 +94,74 @@ describe('hosters helpers', () => {
|
||||
assert.equal(r.embed_url, 'https://byse.sx/e/GOOD123');
|
||||
});
|
||||
});
|
||||
|
||||
describe('recovery claim registry', () => {
|
||||
it('claims remote codes across every title of one normalized hoster account', () => {
|
||||
const registry = createRecoveryClaimRegistry();
|
||||
const first = registry.forUpload(' VOE.SX ', 'ACCOUNT', 'First Episode.mkv');
|
||||
const differentTitle = registry.forUpload('voe.sx', 'ACCOUNT', 'Second Episode.mp4');
|
||||
const differentAccount = registry.forUpload('voe.sx', 'ACCOUNT-B', 'Second Episode.mp4');
|
||||
|
||||
assert.equal(first.reserve('REMOTE-CODE'), true);
|
||||
assert.equal(differentTitle.reserve('REMOTE-CODE'), false);
|
||||
assert.equal(differentAccount.reserve('REMOTE-CODE'), true);
|
||||
});
|
||||
|
||||
it('serializes canonically equivalent titles without blocking an independent title', async () => {
|
||||
const registry = createRecoveryClaimRegistry();
|
||||
const composed = registry.forUpload('voe.sx', 'ACCOUNT', 'Café.mkv');
|
||||
const decomposed = registry.forUpload('voe.sx', 'ACCOUNT', 'Cafe\u0301.mp4');
|
||||
const independent = registry.forUpload('voe.sx', 'ACCOUNT', 'Other Episode.mkv');
|
||||
const events = [];
|
||||
let releaseFirst;
|
||||
const firstGate = new Promise(resolve => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
assert.equal(composed.reserve('UNICODE-CODE'), true);
|
||||
assert.equal(decomposed.reserve('UNICODE-CODE'), false);
|
||||
|
||||
const first = composed.runExclusive(async () => {
|
||||
events.push('first-started');
|
||||
await firstGate;
|
||||
events.push('first-finished');
|
||||
});
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
const equivalent = decomposed.runExclusive(async () => {
|
||||
events.push('equivalent-started');
|
||||
});
|
||||
const other = independent.runExclusive(async () => {
|
||||
events.push('independent-started');
|
||||
});
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(events, ['first-started', 'independent-started']);
|
||||
releaseFirst();
|
||||
await Promise.all([first, equivalent, other]);
|
||||
assert.deepEqual(events, ['first-started', 'independent-started', 'first-finished', 'equivalent-started']);
|
||||
});
|
||||
|
||||
it('fails closed for later jobs after a title becomes uncertain', async () => {
|
||||
const registry = createRecoveryClaimRegistry();
|
||||
const first = registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv');
|
||||
const later = registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4');
|
||||
const error = first.markUncertain(new Error('Remote commit could not be confirmed'));
|
||||
|
||||
assert.equal(error.remoteCommitUncertain, true);
|
||||
assert.equal(error.hosterTransient, true);
|
||||
await assert.rejects(
|
||||
() => later.runExclusive(async () => 'unsafe-success'),
|
||||
err => err.remoteCommitUncertain === true && err.hosterTransient === true
|
||||
);
|
||||
});
|
||||
|
||||
it('drops every claim when the registry is cleared', () => {
|
||||
const registry = createRecoveryClaimRegistry();
|
||||
const first = registry.forUpload('voe.sx', 'ACCOUNT', 'Episode.mkv');
|
||||
assert.equal(first.reserve('REMOTE-CODE'), true);
|
||||
|
||||
registry.clear();
|
||||
|
||||
const nextBatch = registry.forUpload('voe.sx', 'ACCOUNT', 'Episode.mkv');
|
||||
assert.equal(nextBatch.reserve('REMOTE-CODE'), true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -56,12 +56,12 @@ function settings(hoster, parallelCount) {
|
||||
};
|
||||
}
|
||||
|
||||
async function runBatch(manager, tasks) {
|
||||
async function runBatch(manager, tasks, options) {
|
||||
let summary;
|
||||
manager.once('batch-done', value => {
|
||||
summary = value;
|
||||
});
|
||||
await manager.startBatch(tasks);
|
||||
await manager.startBatch(tasks, options);
|
||||
return summary;
|
||||
}
|
||||
|
||||
@@ -88,6 +88,56 @@ function waitFor(promise, timeoutMs, message) {
|
||||
]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
|
||||
async function assertTitleWaiterLeavesSlotAvailable(hosterParallel, globalSettings = {}) {
|
||||
let releaseFirst;
|
||||
let markFirstStarted;
|
||||
let markIndependentStarted;
|
||||
const firstGate = new Promise(resolve => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
const firstStarted = new Promise(resolve => {
|
||||
markFirstStarted = resolve;
|
||||
});
|
||||
const independentStarted = new Promise(resolve => {
|
||||
markIndependentStarted = resolve;
|
||||
});
|
||||
let sequence = 0;
|
||||
loadManager(async (hoster, file) => {
|
||||
if (file === firstPath) {
|
||||
markFirstStarted();
|
||||
await firstGate;
|
||||
}
|
||||
if (file === distinctPath) markIndependentStarted();
|
||||
sequence++;
|
||||
return {
|
||||
file_code: `ADMISSION_${sequence}`,
|
||||
download_url: `https://byse.sx/d/ADMISSION_${sequence}`
|
||||
};
|
||||
});
|
||||
const manager = new UploadManager(settings('byse.sx', hosterParallel), globalSettings);
|
||||
const batch = runBatch(manager, [
|
||||
{ jobId: 'admission-first', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||
]);
|
||||
|
||||
await waitFor(firstStarted, 500, 'First upload did not start');
|
||||
const added = manager.addJobs([
|
||||
{ jobId: 'admission-waiter', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
|
||||
{ jobId: 'admission-independent', file: distinctPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||
]);
|
||||
assert.equal(added.added, 2);
|
||||
let admissionError = null;
|
||||
try {
|
||||
await waitFor(independentStarted, 500, 'Independent title was blocked behind a title-lock waiter');
|
||||
} catch (err) {
|
||||
admissionError = err;
|
||||
} finally {
|
||||
releaseFirst();
|
||||
}
|
||||
const summary = await batch;
|
||||
if (admissionError) throw admissionError;
|
||||
assert.equal(summary.succeeded, 3);
|
||||
}
|
||||
|
||||
test('a batch shares recovery claims across normalized same-name jobs', async () => {
|
||||
let unsafeCalls = 0;
|
||||
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||
@@ -172,6 +222,93 @@ test('normalized same-name recovery sections never overlap', async () => {
|
||||
assert.equal(maximumActive, 1);
|
||||
});
|
||||
|
||||
test('a title-lock waiter does not consume a scarce upload slot', async () => {
|
||||
await assertTitleWaiterLeavesSlotAvailable(2);
|
||||
});
|
||||
|
||||
test('a title-lock waiter does not consume a scarce global upload slot', async () => {
|
||||
await assertTitleWaiterLeavesSlotAvailable(3, { parallelUploadCount: 2 });
|
||||
});
|
||||
|
||||
test('an uncertain remote commit blocks retries, account fallback, and later same-title success', async () => {
|
||||
const calls = [];
|
||||
let markFirstStarted;
|
||||
let releaseUncertain;
|
||||
const firstStarted = new Promise(resolve => {
|
||||
markFirstStarted = resolve;
|
||||
});
|
||||
const uncertainGate = new Promise(resolve => {
|
||||
releaseUncertain = resolve;
|
||||
});
|
||||
loadManager(async (hoster, file, apiKey) => {
|
||||
calls.push({ file, apiKey });
|
||||
if (file === firstPath) {
|
||||
markFirstStarted();
|
||||
await uncertainGate;
|
||||
const error = new Error('Remote commit could not be confirmed');
|
||||
error.remoteCommitUncertain = true;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
file_code: 'LATE_REMOTE_CODE',
|
||||
download_url: 'https://byse.sx/d/LATE_REMOTE_CODE'
|
||||
};
|
||||
});
|
||||
const hosterSettings = settings('byse.sx', 2);
|
||||
hosterSettings['byse.sx'].retries = 2;
|
||||
const manager = new UploadManager(hosterSettings);
|
||||
const fallback = { id: 'ACCOUNT_B', apiKey: 'ACCOUNT_KEY_B' };
|
||||
|
||||
const batch = runBatch(manager, [
|
||||
{
|
||||
jobId: 'uncertain-first',
|
||||
file: firstPath,
|
||||
hoster: 'byse.sx',
|
||||
accountId: 'ACCOUNT_A',
|
||||
apiKey: 'ACCOUNT_KEY_A'
|
||||
}
|
||||
], { primeOverrides: [['byse.sx', fallback]] });
|
||||
await waitFor(firstStarted, 500, 'Uncertain predecessor did not start');
|
||||
const added = manager.addJobs([
|
||||
{
|
||||
jobId: 'uncertain-later',
|
||||
file: secondPath,
|
||||
hoster: 'byse.sx',
|
||||
accountId: 'ACCOUNT_A',
|
||||
apiKey: 'ACCOUNT_KEY_A'
|
||||
}
|
||||
]);
|
||||
assert.equal(added.added, 1);
|
||||
releaseUncertain();
|
||||
const summary = await batch;
|
||||
|
||||
assert.equal(summary.succeeded, 0);
|
||||
assert.equal(summary.failed, 2);
|
||||
assert.deepEqual(calls, [{ file: firstPath, apiKey: 'ACCOUNT_KEY_A' }]);
|
||||
});
|
||||
|
||||
test('recovery claims do not leak into a later batch on the same manager', async () => {
|
||||
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||
if (!options.recoveryClaim.reserve('REUSED_BATCH_CODE')) {
|
||||
const error = new Error('Remote recovery candidate already claimed');
|
||||
error.hosterTransient = true;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
file_code: 'REUSED_BATCH_CODE',
|
||||
download_url: 'https://byse.sx/d/REUSED_BATCH_CODE'
|
||||
};
|
||||
});
|
||||
const manager = new UploadManager(settings('byse.sx', 1));
|
||||
const task = { jobId: 'batch-one', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' };
|
||||
|
||||
const first = await runBatch(manager, [task]);
|
||||
const second = await runBatch(manager, [{ ...task, jobId: 'batch-two' }]);
|
||||
|
||||
assert.equal(first.succeeded, 1);
|
||||
assert.equal(second.succeeded, 1);
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
{
|
||||
label: 'VOE',
|
||||
@@ -263,6 +400,41 @@ for (const scenario of [
|
||||
);
|
||||
});
|
||||
|
||||
test(`${scenario.label} uploader instances reject one direct remote code across different titles`, async () => {
|
||||
await withUploaderMethods(
|
||||
scenario.Uploader,
|
||||
async function () {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
return scenario.buildResult(this, scenario.sharedCode);
|
||||
},
|
||||
async () => {
|
||||
loadManager();
|
||||
const manager = new UploadManager(settings(scenario.hoster, 2));
|
||||
const summary = await runBatch(manager, [
|
||||
{
|
||||
jobId: `${scenario.label}-different-title-a`,
|
||||
file: firstPath,
|
||||
hoster: scenario.hoster,
|
||||
accountId: 'LOGIN_ACCOUNT',
|
||||
username: 'account@example.test',
|
||||
password: 'password'
|
||||
},
|
||||
{
|
||||
jobId: `${scenario.label}-different-title-b`,
|
||||
file: distinctPath,
|
||||
hoster: scenario.hoster,
|
||||
accountId: 'LOGIN_ACCOUNT',
|
||||
username: 'account@example.test',
|
||||
password: 'password'
|
||||
}
|
||||
]);
|
||||
|
||||
assert.equal(summary.succeeded, 1);
|
||||
assert.equal(summary.failed, 1);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => {
|
||||
let active = 0;
|
||||
let maximumActive = 0;
|
||||
|
||||
Reference in New Issue
Block a user