Harden batch-wide recovery identity claims
Scope remote code ownership to normalized hoster and account identities while retaining title-only recovery serialization and canonical Unicode title matching. Mark post-upload ambiguity and duplicate identities as uncertain so retries, account fallback, and later same-title jobs fail closed instead of reporting unsafe success. Acquire recovery title leases before hoster and global semaphores, revalidate failed-account overrides before upload, and clear claim state at batch boundaries. Add deterministic concurrent coverage for same-code rejection, distinct-code parallel success, uncertainty propagation, semaphore fairness, account isolation, Unicode equivalence, and registry lifetime.
This commit is contained in:
+118
-30
@@ -575,7 +575,40 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function _normalizeFileTitle(s) {
|
function _normalizeFileTitle(s) {
|
||||||
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
|
return String(s || '')
|
||||||
|
.normalize('NFKD')
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/\.[\p{Letter}\p{Number}]+$/u, '')
|
||||||
|
.replace(/\p{Mark}+/gu, '')
|
||||||
|
.replace(/[^\p{Letter}\p{Number}]+/gu, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function _normalizeRecoveryHoster(value) {
|
||||||
|
return String(value || '').normalize('NFKC').trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function _normalizeRecoveryAccount(value) {
|
||||||
|
return String(value || '').normalize('NFKC').trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function _createRecoveryUncertainError() {
|
||||||
|
const error = new Error('Upload-Ergebnis für diesen Titel ist wegen eines möglichen Remote-Commits unsicher');
|
||||||
|
error.remoteCommitUncertain = true;
|
||||||
|
error.hosterTransient = true;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function _markRecoveryUncertain(recoveryClaim, error) {
|
||||||
|
if (!recoveryClaim) return error;
|
||||||
|
if (typeof recoveryClaim.markUncertain === 'function') {
|
||||||
|
return recoveryClaim.markUncertain(error);
|
||||||
|
}
|
||||||
|
const uncertainError = error && typeof error === 'object'
|
||||||
|
? error
|
||||||
|
: _createRecoveryUncertainError();
|
||||||
|
uncertainError.remoteCommitUncertain = true;
|
||||||
|
uncertainError.hosterTransient = true;
|
||||||
|
return uncertainError;
|
||||||
}
|
}
|
||||||
|
|
||||||
function _createAbortError() {
|
function _createAbortError() {
|
||||||
@@ -607,40 +640,69 @@ function _waitForRecoveryTurn(predecessor, signal) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function createRecoveryClaimRegistry() {
|
function createRecoveryClaimRegistry() {
|
||||||
const scopes = new Map();
|
const accounts = new Map();
|
||||||
|
let nextClaimId = 1;
|
||||||
return {
|
return {
|
||||||
forUpload(hosterName, apiKey, fileName) {
|
forUpload(hosterName, apiKey, fileName) {
|
||||||
const identity = crypto.createHash('sha256')
|
const accountIdentity = crypto.createHash('sha256')
|
||||||
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`)
|
.update(`${_normalizeRecoveryHoster(hosterName)}\0${_normalizeRecoveryAccount(apiKey)}`)
|
||||||
.digest('hex');
|
.digest('hex');
|
||||||
let codes = scopes.get(identity);
|
let account = accounts.get(accountIdentity);
|
||||||
if (!codes) {
|
if (!account) {
|
||||||
codes = {
|
account = {
|
||||||
values: new Set(),
|
codes: new Map(),
|
||||||
tail: Promise.resolve()
|
titles: new Map()
|
||||||
};
|
};
|
||||||
scopes.set(identity, codes);
|
accounts.set(accountIdentity, account);
|
||||||
}
|
}
|
||||||
|
const titleIdentity = _normalizeFileTitle(fileName);
|
||||||
|
let title = account.titles.get(titleIdentity);
|
||||||
|
if (!title) {
|
||||||
|
title = {
|
||||||
|
tail: Promise.resolve(),
|
||||||
|
uncertain: false
|
||||||
|
};
|
||||||
|
account.titles.set(titleIdentity, title);
|
||||||
|
}
|
||||||
|
const claimId = nextClaimId++;
|
||||||
return {
|
return {
|
||||||
has(code) {
|
has(code) {
|
||||||
return codes.values.has(String(code || '').trim());
|
return account.codes.has(String(code || '').trim());
|
||||||
},
|
},
|
||||||
reserve(code) {
|
reserve(code) {
|
||||||
const normalized = String(code || '').trim();
|
const normalized = String(code || '').trim();
|
||||||
if (!normalized || codes.values.has(normalized)) return false;
|
if (!normalized) return false;
|
||||||
codes.values.add(normalized);
|
if (account.codes.has(normalized)) {
|
||||||
|
return account.codes.get(normalized) === claimId;
|
||||||
|
}
|
||||||
|
account.codes.set(normalized, claimId);
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
|
markUncertain(error) {
|
||||||
|
title.uncertain = true;
|
||||||
|
const uncertainError = error && typeof error === 'object'
|
||||||
|
? error
|
||||||
|
: _createRecoveryUncertainError();
|
||||||
|
uncertainError.remoteCommitUncertain = true;
|
||||||
|
uncertainError.hosterTransient = true;
|
||||||
|
return uncertainError;
|
||||||
|
},
|
||||||
|
isUncertain() {
|
||||||
|
return title.uncertain;
|
||||||
|
},
|
||||||
async runExclusive(operation, signal) {
|
async runExclusive(operation, signal) {
|
||||||
const predecessor = codes.tail;
|
const predecessor = title.tail;
|
||||||
let release;
|
let release;
|
||||||
const current = new Promise(resolve => {
|
const current = new Promise(resolve => {
|
||||||
release = resolve;
|
release = resolve;
|
||||||
});
|
});
|
||||||
codes.tail = predecessor.then(() => current);
|
title.tail = predecessor.then(() => current, () => current);
|
||||||
try {
|
try {
|
||||||
await _waitForRecoveryTurn(predecessor, signal);
|
await _waitForRecoveryTurn(predecessor, signal);
|
||||||
return await operation();
|
if (title.uncertain) throw _createRecoveryUncertainError();
|
||||||
|
const result = await operation();
|
||||||
|
if (title.uncertain) throw _createRecoveryUncertainError();
|
||||||
|
return result;
|
||||||
} finally {
|
} finally {
|
||||||
release();
|
release();
|
||||||
}
|
}
|
||||||
@@ -648,7 +710,7 @@ function createRecoveryClaimRegistry() {
|
|||||||
};
|
};
|
||||||
},
|
},
|
||||||
clear() {
|
clear() {
|
||||||
scopes.clear();
|
accounts.clear();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -805,23 +867,28 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
bodyTimeout: UPLOAD_TIMEOUT
|
bodyTimeout: UPLOAD_TIMEOUT
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (signal && signal.aborted) throw err;
|
const error = signal && signal.aborted ? err : createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
|
||||||
throw createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
|
|
||||||
phase: 'upload-request',
|
phase: 'upload-request',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
retryable: true,
|
retryable: true,
|
||||||
transientNetwork: true
|
transientNetwork: true
|
||||||
});
|
});
|
||||||
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, error);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { body, statusCode, headers } = uploadResponse;
|
const { body, statusCode, headers } = uploadResponse;
|
||||||
|
|
||||||
const rawBody = await body.text();
|
let rawBody;
|
||||||
|
try {
|
||||||
|
rawBody = await body.text();
|
||||||
|
} catch (err) {
|
||||||
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||||
|
}
|
||||||
let payload = null;
|
let payload = null;
|
||||||
try {
|
try {
|
||||||
payload = rawBody ? JSON.parse(rawBody) : {};
|
payload = rawBody ? JSON.parse(rawBody) : {};
|
||||||
} catch {
|
} catch {
|
||||||
throw createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
|
||||||
phase: 'upload-response',
|
phase: 'upload-response',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
httpStatus: statusCode,
|
httpStatus: statusCode,
|
||||||
@@ -829,7 +896,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
body: rawBody,
|
body: rawBody,
|
||||||
retryable: statusCode >= 500,
|
retryable: statusCode >= 500,
|
||||||
transientNetwork: statusCode >= 500
|
transientNetwork: statusCode >= 500
|
||||||
});
|
}));
|
||||||
}
|
}
|
||||||
// Normalize valid-but-not-object JSON (JSON.parse('null') → null;
|
// Normalize valid-but-not-object JSON (JSON.parse('null') → null;
|
||||||
// JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this
|
// JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this
|
||||||
@@ -843,7 +910,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (statusCode < 200 || statusCode >= 300) {
|
if (statusCode < 200 || statusCode >= 300) {
|
||||||
throw createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
|
const error = createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
|
||||||
phase: 'upload-response',
|
phase: 'upload-response',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
httpStatus: statusCode,
|
httpStatus: statusCode,
|
||||||
@@ -852,10 +919,13 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
retryable: statusCode === 429 || statusCode >= 500,
|
retryable: statusCode === 429 || statusCode >= 500,
|
||||||
transientNetwork: statusCode >= 500
|
transientNetwork: statusCode >= 500
|
||||||
});
|
});
|
||||||
|
throw statusCode >= 500
|
||||||
|
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
|
||||||
|
: error;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
|
if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
|
||||||
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
const error = createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||||
phase: 'upload-response',
|
phase: 'upload-response',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
httpStatus: Number(payload.status),
|
httpStatus: Number(payload.status),
|
||||||
@@ -864,6 +934,9 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
retryable: Number(payload.status) === 429 || Number(payload.status) >= 500,
|
retryable: Number(payload.status) === 429 || Number(payload.status) >= 500,
|
||||||
transientNetwork: Number(payload.status) >= 500
|
transientNetwork: Number(payload.status) >= 500
|
||||||
});
|
});
|
||||||
|
throw Number(payload.status) >= 500
|
||||||
|
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
|
||||||
|
: error;
|
||||||
}
|
}
|
||||||
|
|
||||||
let result = null;
|
let result = null;
|
||||||
@@ -885,7 +958,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
if (result && (result.file_code || result.download_url || result.embed_url)) {
|
||||||
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
|
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
|
||||||
if (!opts.recoveryClaim.reserve(result.file_code)) {
|
if (!opts.recoveryClaim.reserve(result.file_code)) {
|
||||||
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
const error = createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
|
||||||
phase: 'upload-result',
|
phase: 'upload-result',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
httpStatus: statusCode,
|
httpStatus: statusCode,
|
||||||
@@ -894,6 +967,8 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
retryable: true,
|
retryable: true,
|
||||||
hosterTransient: true
|
hosterTransient: true
|
||||||
});
|
});
|
||||||
|
error.remoteIdentityClaimed = true;
|
||||||
|
throw _markRecoveryUncertain(opts.recoveryClaim, error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
@@ -921,8 +996,12 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
// even after our uploader gave up.
|
// even after our uploader gave up.
|
||||||
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
|
||||||
const fileName = path.basename(filePath);
|
const fileName = path.basename(filePath);
|
||||||
|
try {
|
||||||
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
|
||||||
if (polled) return polled;
|
if (polled) return polled;
|
||||||
|
} catch (err) {
|
||||||
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Doodstream: the doodapi upload POST returned no filecode (the same backend
|
// Doodstream: the doodapi upload POST returned no filecode (the same backend
|
||||||
@@ -930,21 +1009,29 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
// the file did register, claim its code instead of failing the upload.
|
// the file did register, claim its code instead of failing the upload.
|
||||||
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
|
||||||
const fileName = path.basename(filePath);
|
const fileName = path.basename(filePath);
|
||||||
|
try {
|
||||||
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
|
||||||
if (polled) return polled;
|
if (polled) return polled;
|
||||||
|
} catch (err) {
|
||||||
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) {
|
if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) {
|
||||||
byseBaselineError.hosterTransient = true;
|
byseBaselineError.hosterTransient = true;
|
||||||
throw byseBaselineError;
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, byseBaselineError);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) {
|
if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) {
|
||||||
doodBaselineError.hosterTransient = true;
|
doodBaselineError.hosterTransient = true;
|
||||||
throw doodBaselineError;
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, doodBaselineError);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (parseErr) throw parseErr;
|
if (parseErr) {
|
||||||
|
throw explicitlyRejected
|
||||||
|
? parseErr
|
||||||
|
: _markRecoveryUncertain(opts && opts.recoveryClaim, parseErr);
|
||||||
|
}
|
||||||
|
|
||||||
if (payload.success === false) {
|
if (payload.success === false) {
|
||||||
throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, {
|
throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, {
|
||||||
@@ -970,7 +1057,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
// so tag it hosterTransient: the upload-manager then fails this file WITHOUT
|
// so tag it hosterTransient: the upload-manager then fails this file WITHOUT
|
||||||
// blacklisting the account (same protection the web path got in 3.3.29) and
|
// blacklisting the account (same protection the web path got in 3.3.29) and
|
||||||
// the account stays usable for the next retry/batch.
|
// the account stays usable for the next retry/batch.
|
||||||
throw createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
|
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
|
||||||
phase: 'upload-result',
|
phase: 'upload-result',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
httpStatus: statusCode,
|
httpStatus: statusCode,
|
||||||
@@ -978,7 +1065,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
|
|||||||
body: rawBody,
|
body: rawBody,
|
||||||
retryable: true,
|
retryable: true,
|
||||||
hosterTransient: true
|
hosterTransient: true
|
||||||
});
|
}));
|
||||||
}
|
}
|
||||||
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
|
||||||
phase: 'upload-result',
|
phase: 'upload-result',
|
||||||
@@ -1007,6 +1094,7 @@ module.exports = {
|
|||||||
uploadFile,
|
uploadFile,
|
||||||
prefetchBaseline,
|
prefetchBaseline,
|
||||||
createRecoveryClaimRegistry,
|
createRecoveryClaimRegistry,
|
||||||
|
normalizeRecoveryTitle: _normalizeFileTitle,
|
||||||
HOSTER_CONFIGS,
|
HOSTER_CONFIGS,
|
||||||
__test: {
|
__test: {
|
||||||
extractUploadServerUrl,
|
extractUploadServerUrl,
|
||||||
|
|||||||
+146
-62
@@ -71,7 +71,7 @@ class UploadManager extends EventEmitter {
|
|||||||
this._suspectGoodAccounts.clear();
|
this._suspectGoodAccounts.clear();
|
||||||
this._doodApiKeyCache.clear();
|
this._doodApiKeyCache.clear();
|
||||||
this._baselineCache.clear();
|
this._baselineCache.clear();
|
||||||
this._recoveryClaims.clear();
|
if (!this.running) this._recoveryClaims.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
switchAccount(hoster, fallbackAccount) {
|
switchAccount(hoster, fallbackAccount) {
|
||||||
@@ -134,6 +134,41 @@ class UploadManager extends EventEmitter {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_swapFailedAccount(task, jobId, fileName) {
|
||||||
|
if (!task.accountId || !this._failedAccounts.has(task.hoster + ':' + task.accountId)) return false;
|
||||||
|
const override = this._accountOverrides.get(task.hoster);
|
||||||
|
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
|
||||||
|
this._rotLog('pre-job-swap', {
|
||||||
|
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
|
||||||
|
});
|
||||||
|
task.accountId = override.id;
|
||||||
|
task.username = override.username;
|
||||||
|
task.password = override.password;
|
||||||
|
task.apiKey = override.apiKey;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
this._rotLog('pre-job-swap-blocked', {
|
||||||
|
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
|
||||||
|
hasOverride: !!override,
|
||||||
|
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
|
||||||
|
});
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async _coordinateAccountFailure(task, err, signal, jobId) {
|
||||||
|
if (!task.accountId || (err && err.remoteCommitUncertain === true)) return;
|
||||||
|
if (!this._shouldSkipRetryOnAccountError(err)) return;
|
||||||
|
const key = task.hoster + ':' + task.accountId;
|
||||||
|
if (this._failedAccounts.has(key)) return;
|
||||||
|
this._failedAccounts.set(key, true);
|
||||||
|
this._rotLog('mark-failed', {
|
||||||
|
jobId, hoster: task.hoster, fileName: path.basename(task.file),
|
||||||
|
accountId: task.accountId, lastError: err && err.message ? err.message : String(err)
|
||||||
|
});
|
||||||
|
this.emit('account-failed', { hoster: task.hoster, accountId: task.accountId });
|
||||||
|
await this._sleep(800, signal);
|
||||||
|
}
|
||||||
|
|
||||||
_rotLog(event, data) {
|
_rotLog(event, data) {
|
||||||
this.emit('rot-log', { ts: Date.now(), event, ...data });
|
this.emit('rot-log', { ts: Date.now(), event, ...data });
|
||||||
}
|
}
|
||||||
@@ -161,6 +196,7 @@ class UploadManager extends EventEmitter {
|
|||||||
// which takes priority in _shouldSkipRetryOnAccountError.
|
// which takes priority in _shouldSkipRetryOnAccountError.
|
||||||
_isFileRejectedError(err) {
|
_isFileRejectedError(err) {
|
||||||
if (!err) return false;
|
if (!err) return false;
|
||||||
|
if (err.remoteCommitUncertain === true) return false;
|
||||||
if (err.transientNetwork === true) return false;
|
if (err.transientNetwork === true) return false;
|
||||||
if (err.accountError === true) return false; // explicit account-level wins
|
if (err.accountError === true) return false; // explicit account-level wins
|
||||||
if (err.fileRejected === true) return true;
|
if (err.fileRejected === true) return true;
|
||||||
@@ -355,6 +391,7 @@ class UploadManager extends EventEmitter {
|
|||||||
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
||||||
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
||||||
this._recoveryClaims.clear();
|
this._recoveryClaims.clear();
|
||||||
|
this._recoveryClaims = createRecoveryClaimRegistry();
|
||||||
this.semaphores = {};
|
this.semaphores = {};
|
||||||
this.globalSemaphore = null;
|
this.globalSemaphore = null;
|
||||||
this.globalThrottle = null;
|
this.globalThrottle = null;
|
||||||
@@ -440,13 +477,12 @@ class UploadManager extends EventEmitter {
|
|||||||
files
|
files
|
||||||
};
|
};
|
||||||
|
|
||||||
|
this._recoveryClaims.clear();
|
||||||
this.emit('batch-done', summary);
|
this.emit('batch-done', summary);
|
||||||
}
|
}
|
||||||
|
|
||||||
async _runJob(task, results, batchSignal) {
|
async _runJob(task, results, batchSignal) {
|
||||||
const settings = this._getSettings(task.hoster);
|
const settings = this._getSettings(task.hoster);
|
||||||
const hosterSemaphore = this._getSemaphore(task.hoster);
|
|
||||||
const globalSemaphore = this._getGlobalSemaphore();
|
|
||||||
const uploadId = crypto.randomBytes(8).toString('hex');
|
const uploadId = crypto.randomBytes(8).toString('hex');
|
||||||
const jobId = task.jobId || uploadId;
|
const jobId = task.jobId || uploadId;
|
||||||
const fileName = path.basename(task.file);
|
const fileName = path.basename(task.file);
|
||||||
@@ -466,8 +502,6 @@ class UploadManager extends EventEmitter {
|
|||||||
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
|
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
|
||||||
this.jobAbortControllers.set(jobId, jobAbortController);
|
this.jobAbortControllers.set(jobId, jobAbortController);
|
||||||
|
|
||||||
let hosterSlotAcquired = false;
|
|
||||||
let globalSlotAcquired = false;
|
|
||||||
let finalResultRecorded = false;
|
let finalResultRecorded = false;
|
||||||
let finalStatus = 'error';
|
let finalStatus = 'error';
|
||||||
let lastError = null;
|
let lastError = null;
|
||||||
@@ -549,9 +583,6 @@ class UploadManager extends EventEmitter {
|
|||||||
// queueJobs array; the first event it actually needs from main is the
|
// queueJobs array; the first event it actually needs from main is the
|
||||||
// 'getting-server' / 'uploading' transition for the jobs that the
|
// 'getting-server' / 'uploading' transition for the jobs that the
|
||||||
// semaphore lets through.
|
// semaphore lets through.
|
||||||
await hosterSemaphore.acquire(signal);
|
|
||||||
hosterSlotAcquired = true;
|
|
||||||
|
|
||||||
let fileProbe = null;
|
let fileProbe = null;
|
||||||
try {
|
try {
|
||||||
fileProbe = await probeFileHead(task.file, 512);
|
fileProbe = await probeFileHead(task.file, 512);
|
||||||
@@ -566,11 +597,6 @@ class UploadManager extends EventEmitter {
|
|||||||
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
|
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
|
||||||
});
|
});
|
||||||
|
|
||||||
if (globalSemaphore) {
|
|
||||||
await globalSemaphore.acquire(signal);
|
|
||||||
globalSlotAcquired = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (settings.timeIntervalSec > 0) {
|
if (settings.timeIntervalSec > 0) {
|
||||||
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal);
|
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal);
|
||||||
}
|
}
|
||||||
@@ -580,24 +606,7 @@ class UploadManager extends EventEmitter {
|
|||||||
// of burning a guaranteed-to-fail upload attempt. Critical at scale:
|
// of burning a guaranteed-to-fail upload attempt. Critical at scale:
|
||||||
// with 500 queued jobs and 1 parallel slot, without this check every
|
// with 500 queued jobs and 1 parallel slot, without this check every
|
||||||
// job still hits the original dead account first.
|
// job still hits the original dead account first.
|
||||||
if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) {
|
this._swapFailedAccount(task, jobId, fileName);
|
||||||
const override = this._accountOverrides.get(task.hoster);
|
|
||||||
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
|
|
||||||
this._rotLog('pre-job-swap', {
|
|
||||||
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
|
|
||||||
});
|
|
||||||
task.accountId = override.id;
|
|
||||||
task.username = override.username;
|
|
||||||
task.password = override.password;
|
|
||||||
task.apiKey = override.apiKey;
|
|
||||||
} else {
|
|
||||||
this._rotLog('pre-job-swap-blocked', {
|
|
||||||
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
|
|
||||||
hasOverride: !!override,
|
|
||||||
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A previous file of at least this size already got a suspect rejection
|
// A previous file of at least this size already got a suspect rejection
|
||||||
// on this exact account — skip the guaranteed-to-fail multi-GB upload
|
// on this exact account — skip the guaranteed-to-fail multi-GB upload
|
||||||
@@ -735,7 +744,7 @@ class UploadManager extends EventEmitter {
|
|||||||
} catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ }
|
} catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ }
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await this._executeUpload(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe);
|
const result = await this._executeUploadWithAdmission(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe, true, jobId);
|
||||||
|
|
||||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||||
|
|
||||||
@@ -784,6 +793,11 @@ class UploadManager extends EventEmitter {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (err && err.remoteCommitUncertain === true) {
|
||||||
|
lastError = err;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
if (isSpeedRestart && attempt < maxAttempts) {
|
if (isSpeedRestart && attempt < maxAttempts) {
|
||||||
lastError = new Error('Geschwindigkeit zu niedrig - Neustart');
|
lastError = new Error('Geschwindigkeit zu niedrig - Neustart');
|
||||||
await this._sleep(3000, signal);
|
await this._sleep(3000, signal);
|
||||||
@@ -1047,7 +1061,7 @@ class UploadManager extends EventEmitter {
|
|||||||
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
|
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
|
||||||
: hosterThrottle || globalThrottle;
|
: hosterThrottle || globalThrottle;
|
||||||
|
|
||||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, true, jobId);
|
||||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||||
this.activeJobs.delete(uploadId);
|
this.activeJobs.delete(uploadId);
|
||||||
this.sessionBytes += fileSize;
|
this.sessionBytes += fileSize;
|
||||||
@@ -1104,9 +1118,6 @@ class UploadManager extends EventEmitter {
|
|||||||
this.activeJobs.delete(uploadId);
|
this.activeJobs.delete(uploadId);
|
||||||
this.jobAbortControllers.delete(jobId);
|
this.jobAbortControllers.delete(jobId);
|
||||||
cleanupSignals();
|
cleanupSignals();
|
||||||
// Release in reverse order of acquire (global first, then hoster)
|
|
||||||
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
|
|
||||||
if (hosterSlotAcquired) hosterSemaphore.release();
|
|
||||||
this.emit('job-settled', {
|
this.emit('job-settled', {
|
||||||
jobId,
|
jobId,
|
||||||
sourceCleanupToken: task.sourceCleanupToken || null,
|
sourceCleanupToken: task.sourceCleanupToken || null,
|
||||||
@@ -1190,7 +1201,7 @@ class UploadManager extends EventEmitter {
|
|||||||
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
|
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
|
||||||
: hosterThrottle || globalThrottle;
|
: hosterThrottle || globalThrottle;
|
||||||
try {
|
try {
|
||||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, false, jobId);
|
||||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||||
this.activeJobs.delete(uploadId);
|
this.activeJobs.delete(uploadId);
|
||||||
this.sessionBytes += fileSize;
|
this.sessionBytes += fileSize;
|
||||||
@@ -1210,6 +1221,7 @@ class UploadManager extends EventEmitter {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (signal.aborted || this.stopAfterActive) break;
|
if (signal.aborted || this.stopAfterActive) break;
|
||||||
|
if (err && err.remoteCommitUncertain === true) throw err;
|
||||||
if (err && err.suspectReject === true) {
|
if (err && err.suspectReject === true) {
|
||||||
this._noteSuspectReject(task.hoster, account.id, fileSize);
|
this._noteSuspectReject(task.hoster, account.id, fileSize);
|
||||||
}
|
}
|
||||||
@@ -1241,16 +1253,95 @@ class UploadManager extends EventEmitter {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async _executeUpload(task, progressCb, signal, throttle, fileProbe) {
|
async _executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, coordinateAccountFailure = true, jobId = task.jobId) {
|
||||||
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe);
|
while (true) {
|
||||||
return assertUploadConfirmation(result, task.hoster);
|
const context = await this._createRecoveryContext(task);
|
||||||
|
let retryAdmission = false;
|
||||||
|
const operation = async () => {
|
||||||
|
const hosterSemaphore = this._getSemaphore(task.hoster);
|
||||||
|
const globalSemaphore = this._getGlobalSemaphore();
|
||||||
|
let hosterSlotAcquired = false;
|
||||||
|
let globalSlotAcquired = false;
|
||||||
|
try {
|
||||||
|
await hosterSemaphore.acquire(signal);
|
||||||
|
hosterSlotAcquired = true;
|
||||||
|
if (globalSemaphore) {
|
||||||
|
await globalSemaphore.acquire(signal);
|
||||||
|
globalSlotAcquired = true;
|
||||||
|
}
|
||||||
|
if (this._swapFailedAccount(task, jobId, path.basename(task.file))) {
|
||||||
|
retryAdmission = true;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await this._executeUpload(task, progressCb, signal, throttle, fileProbe, context);
|
||||||
|
} catch (err) {
|
||||||
|
if (coordinateAccountFailure) await this._coordinateAccountFailure(task, err, signal, jobId);
|
||||||
|
if (context.recoveryClaim && err && err.remoteCommitUncertain === true) {
|
||||||
|
throw context.recoveryClaim.markUncertain(err);
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
|
||||||
|
if (hosterSlotAcquired) hosterSemaphore.release();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const result = context.recoveryClaim
|
||||||
|
? await context.recoveryClaim.runExclusive(operation, signal)
|
||||||
|
: await operation();
|
||||||
|
if (retryAdmission) continue;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) {
|
async _executeUpload(task, progressCb, signal, throttle, fileProbe, context) {
|
||||||
|
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context);
|
||||||
|
let confirmed;
|
||||||
|
try {
|
||||||
|
confirmed = assertUploadConfirmation(result, task.hoster);
|
||||||
|
} catch (err) {
|
||||||
|
if (context.recoveryClaim) throw context.recoveryClaim.markUncertain(err);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
return confirmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
async _createRecoveryContext(task) {
|
||||||
|
const fileName = path.basename(task.file);
|
||||||
|
if ((task.hoster === 'vidmoly.me' || task.hoster === 'voe.sx') && task.username) {
|
||||||
|
const accountIdentity = task.accountId !== null && task.accountId !== undefined
|
||||||
|
? task.accountId
|
||||||
|
: String(task.username || '').trim().toLowerCase();
|
||||||
|
return {
|
||||||
|
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
|
||||||
|
doodApiKey: null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (task.hoster === 'doodstream.com' && task.username) {
|
||||||
|
const doodApiKey = await this._resolveDoodstreamApiKey(task);
|
||||||
|
const accountIdentity = doodApiKey || (task.accountId !== null && task.accountId !== undefined
|
||||||
|
? task.accountId
|
||||||
|
: String(task.username || '').trim().toLowerCase());
|
||||||
|
return {
|
||||||
|
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
|
||||||
|
doodApiKey
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
||||||
|
return {
|
||||||
|
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, task.apiKey, fileName),
|
||||||
|
doodApiKey: null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { recoveryClaim: null, doodApiKey: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context) {
|
||||||
if (task.hoster === 'vidmoly.me' && task.username) {
|
if (task.hoster === 'vidmoly.me' && task.username) {
|
||||||
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle);
|
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle, context.recoveryClaim);
|
||||||
} else if (task.hoster === 'voe.sx' && task.username) {
|
} else if (task.hoster === 'voe.sx' && task.username) {
|
||||||
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle);
|
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle, context.recoveryClaim);
|
||||||
} else if (task.hoster === 'doodstream.com' && task.username) {
|
} else if (task.hoster === 'doodstream.com' && task.username) {
|
||||||
// Login-path reliability fix: the web-form upload returns the filecode in
|
// Login-path reliability fix: the web-form upload returns the filecode in
|
||||||
// an HTML form that comes back empty for large files (doodstream backend
|
// an HTML form that comes back empty for large files (doodstream backend
|
||||||
@@ -1258,45 +1349,39 @@ class UploadManager extends EventEmitter {
|
|||||||
// session ONCE per batch and upload via the official API instead — it
|
// session ONCE per batch and upload via the official API instead — it
|
||||||
// returns result[0].filecode directly and has no empty-form failure mode.
|
// returns result[0].filecode directly and has no empty-form failure mode.
|
||||||
// Falls back to the web-form upload if no valid key can be derived.
|
// Falls back to the web-form upload if no valid key can be derived.
|
||||||
const apiKey = await this._resolveDoodstreamApiKey(task);
|
const apiKey = context.doodApiKey;
|
||||||
if (apiKey) {
|
if (apiKey) {
|
||||||
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
|
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||||
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe);
|
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
|
||||||
}
|
}
|
||||||
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
|
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
|
||||||
const dood = new DoodstreamUploader();
|
const dood = new DoodstreamUploader();
|
||||||
await dood.login(task.username, task.password);
|
await dood.login(task.username, task.password);
|
||||||
return dood.upload(task.file, progressCb, signal, throttle);
|
const result = await dood.upload(task.file, progressCb, signal, throttle);
|
||||||
|
if (result && result.file_code && !context.recoveryClaim.reserve(result.file_code)) {
|
||||||
|
const error = new Error('Doodstream Upload lieferte eine bereits zugeordnete Remote-Identität');
|
||||||
|
error.remoteIdentityClaimed = true;
|
||||||
|
throw context.recoveryClaim.markUncertain(error);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
} else if (task.hoster === 'clouddrop.cc') {
|
} else if (task.hoster === 'clouddrop.cc') {
|
||||||
const clouddrop = new ClouddropUploader(task.apiKey);
|
const clouddrop = new ClouddropUploader(task.apiKey);
|
||||||
return clouddrop.upload(task.file, progressCb, signal, throttle);
|
return clouddrop.upload(task.file, progressCb, signal, throttle);
|
||||||
} else {
|
} else {
|
||||||
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
|
||||||
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe);
|
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
|
||||||
}
|
}
|
||||||
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
|
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle) {
|
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle, recoveryClaim) {
|
||||||
const accountIdentity = task.accountId !== null && task.accountId !== undefined
|
|
||||||
? task.accountId
|
|
||||||
: String(task.username || '').trim().toLowerCase();
|
|
||||||
const recoveryClaim = this._recoveryClaims.forUpload(
|
|
||||||
task.hoster,
|
|
||||||
accountIdentity,
|
|
||||||
path.basename(task.file)
|
|
||||||
);
|
|
||||||
return recoveryClaim.runExclusive(async () => {
|
|
||||||
const uploader = new UploaderClass(recoveryClaim);
|
const uploader = new UploaderClass(recoveryClaim);
|
||||||
await uploader.login(task.username, task.password);
|
await uploader.login(task.username, task.password);
|
||||||
return uploader.upload(task.file, progressCb, signal, throttle);
|
return uploader.upload(task.file, progressCb, signal, throttle);
|
||||||
}, signal);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
|
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe, recoveryClaim) {
|
||||||
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
|
|
||||||
return recoveryClaim.runExclusive(async () => {
|
|
||||||
const options = { recoveryClaim };
|
const options = { recoveryClaim };
|
||||||
if (hosterName === 'byse.sx') {
|
if (hosterName === 'byse.sx') {
|
||||||
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||||
@@ -1305,7 +1390,6 @@ class UploadManager extends EventEmitter {
|
|||||||
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
|
||||||
}
|
}
|
||||||
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
|
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
|
||||||
}, signal);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_getBaseline(hosterName, apiKey, signal) {
|
_getBaseline(hosterName, apiKey, signal) {
|
||||||
|
|||||||
+33
-11
@@ -3,6 +3,7 @@ const path = require('path');
|
|||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const { request } = require('undici');
|
const { request } = require('undici');
|
||||||
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
||||||
|
const { normalizeRecoveryTitle } = require('./hosters');
|
||||||
|
|
||||||
const BASE_URL = 'https://vidmoly.me';
|
const BASE_URL = 'https://vidmoly.me';
|
||||||
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
||||||
@@ -275,13 +276,13 @@ class VidmolyUploader {
|
|||||||
bodyTimeout: UPLOAD_TIMEOUT
|
bodyTimeout: UPLOAD_TIMEOUT
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (signal && signal.aborted) throw err;
|
const error = signal && signal.aborted ? err : createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
|
||||||
throw createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
|
|
||||||
phase: 'upload-request',
|
phase: 'upload-request',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
retryable: true,
|
retryable: true,
|
||||||
transientNetwork: true
|
transientNetwork: true
|
||||||
});
|
});
|
||||||
|
throw this._markRemoteCommitUncertain(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { body, statusCode, headers } = uploadResponse;
|
const { body, statusCode, headers } = uploadResponse;
|
||||||
@@ -295,17 +296,25 @@ class VidmolyUploader {
|
|||||||
// Always drain the original body to prevent connection leak
|
// Always drain the original body to prevent connection leak
|
||||||
try { await body.text(); } catch {}
|
try { await body.text(); } catch {}
|
||||||
if (location) {
|
if (location) {
|
||||||
|
try {
|
||||||
const resultRes = await this._fetch(new URL(location, uploadUrl).href);
|
const resultRes = await this._fetch(new URL(location, uploadUrl).href);
|
||||||
resultHtml = await resultRes.text();
|
resultHtml = await resultRes.text();
|
||||||
|
} catch (err) {
|
||||||
|
throw this._markRemoteCommitUncertain(err);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
resultHtml = '';
|
resultHtml = '';
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
try {
|
||||||
resultHtml = await body.text();
|
resultHtml = await body.text();
|
||||||
|
} catch (err) {
|
||||||
|
throw this._markRemoteCommitUncertain(err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (statusCode >= 400) {
|
if (statusCode >= 400) {
|
||||||
throw createTransportError('Vidmoly Upload fehlgeschlagen', {
|
const error = createTransportError('Vidmoly Upload fehlgeschlagen', {
|
||||||
phase: 'upload-response',
|
phase: 'upload-response',
|
||||||
endpoint: targetUrl,
|
endpoint: targetUrl,
|
||||||
httpStatus: statusCode,
|
httpStatus: statusCode,
|
||||||
@@ -314,6 +323,7 @@ class VidmolyUploader {
|
|||||||
retryable: statusCode === 429 || statusCode >= 500,
|
retryable: statusCode === 429 || statusCode >= 500,
|
||||||
transientNetwork: statusCode >= 500
|
transientNetwork: statusCode >= 500
|
||||||
});
|
});
|
||||||
|
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try JSON first. The current transit server returns
|
// Try JSON first. The current transit server returns
|
||||||
@@ -355,23 +365,35 @@ class VidmolyUploader {
|
|||||||
} catch (primaryErr) {
|
} catch (primaryErr) {
|
||||||
if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr;
|
if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr;
|
||||||
if (baselineCodes) {
|
if (baselineCodes) {
|
||||||
|
try {
|
||||||
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
|
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
|
||||||
if (fallback) return fallback;
|
if (fallback) return fallback;
|
||||||
|
} catch (err) {
|
||||||
|
throw this._markRemoteCommitUncertain(err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (baselineError) {
|
if (baselineError) {
|
||||||
baselineError.hosterTransient = true;
|
baselineError.hosterTransient = true;
|
||||||
throw baselineError;
|
throw this._markRemoteCommitUncertain(baselineError);
|
||||||
}
|
}
|
||||||
throw primaryErr;
|
throw this._markRemoteCommitUncertain(primaryErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_normalizeTitle(value) {
|
_normalizeTitle(value) {
|
||||||
return String(value || '')
|
return normalizeRecoveryTitle(value);
|
||||||
.toLowerCase()
|
}
|
||||||
.normalize('NFKD')
|
|
||||||
.replace(/\.[a-z0-9]+$/i, '')
|
_markRemoteCommitUncertain(error) {
|
||||||
.replace(/[^a-z0-9]+/g, '');
|
if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
|
||||||
|
return this.recoveryClaim.markUncertain(error);
|
||||||
|
}
|
||||||
|
const uncertainError = error && typeof error === 'object'
|
||||||
|
? error
|
||||||
|
: new Error('Vidmoly Upload-Ergebnis ist unsicher');
|
||||||
|
uncertainError.remoteCommitUncertain = true;
|
||||||
|
uncertainError.hosterTransient = true;
|
||||||
|
return uncertainError;
|
||||||
}
|
}
|
||||||
|
|
||||||
_buildUrlsFromCode(fileCode, phase = 'upload-result') {
|
_buildUrlsFromCode(fileCode, phase = 'upload-result') {
|
||||||
@@ -387,7 +409,7 @@ class VidmolyUploader {
|
|||||||
hosterTransient: true
|
hosterTransient: true
|
||||||
});
|
});
|
||||||
error.remoteIdentityClaimed = true;
|
error.remoteIdentityClaimed = true;
|
||||||
throw error;
|
throw this._markRemoteCommitUncertain(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
+32
-13
@@ -3,6 +3,7 @@ const path = require('path');
|
|||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const { request } = require('undici');
|
const { request } = require('undici');
|
||||||
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
|
||||||
|
const { normalizeRecoveryTitle } = require('./hosters');
|
||||||
|
|
||||||
const BASE_URL = 'https://voe.sx';
|
const BASE_URL = 'https://voe.sx';
|
||||||
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
||||||
@@ -352,22 +353,27 @@ class VoeUploader {
|
|||||||
bodyTimeout: UPLOAD_TIMEOUT
|
bodyTimeout: UPLOAD_TIMEOUT
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (signal && signal.aborted) throw err;
|
const error = signal && signal.aborted ? err : createTransportError('VOE Upload konnte nicht übertragen werden', {
|
||||||
throw createTransportError('VOE Upload konnte nicht übertragen werden', {
|
|
||||||
phase: 'upload-request',
|
phase: 'upload-request',
|
||||||
endpoint: uploadServer,
|
endpoint: uploadServer,
|
||||||
retryable: true,
|
retryable: true,
|
||||||
transientNetwork: true
|
transientNetwork: true
|
||||||
});
|
});
|
||||||
|
throw this._markRemoteCommitUncertain(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { body, headers, statusCode } = uploadResponse;
|
const { body, headers, statusCode } = uploadResponse;
|
||||||
|
|
||||||
this._parseCookiesFromHeaders(headers || {});
|
this._parseCookiesFromHeaders(headers || {});
|
||||||
|
|
||||||
const rawBody = await body.text();
|
let rawBody;
|
||||||
|
try {
|
||||||
|
rawBody = await body.text();
|
||||||
|
} catch (err) {
|
||||||
|
throw this._markRemoteCommitUncertain(err);
|
||||||
|
}
|
||||||
if (statusCode < 200 || statusCode >= 300) {
|
if (statusCode < 200 || statusCode >= 300) {
|
||||||
throw createTransportError('VOE Upload fehlgeschlagen', {
|
const error = createTransportError('VOE Upload fehlgeschlagen', {
|
||||||
phase: 'upload-response',
|
phase: 'upload-response',
|
||||||
endpoint: uploadServer,
|
endpoint: uploadServer,
|
||||||
httpStatus: statusCode,
|
httpStatus: statusCode,
|
||||||
@@ -376,6 +382,7 @@ class VoeUploader {
|
|||||||
retryable: statusCode === 429 || statusCode >= 500,
|
retryable: statusCode === 429 || statusCode >= 500,
|
||||||
transientNetwork: statusCode >= 500
|
transientNetwork: statusCode >= 500
|
||||||
});
|
});
|
||||||
|
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try JSON response
|
// Try JSON response
|
||||||
@@ -407,23 +414,27 @@ class VoeUploader {
|
|||||||
|
|
||||||
// Fallback: poll the file list to find the newly uploaded file
|
// Fallback: poll the file list to find the newly uploaded file
|
||||||
if (baselineCodes) {
|
if (baselineCodes) {
|
||||||
|
try {
|
||||||
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
|
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
|
||||||
if (result) return result;
|
if (result) return result;
|
||||||
|
} catch (err) {
|
||||||
|
throw this._markRemoteCommitUncertain(err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (baselineError) {
|
if (baselineError) {
|
||||||
baselineError.hosterTransient = true;
|
baselineError.hosterTransient = true;
|
||||||
throw baselineError;
|
throw this._markRemoteCommitUncertain(baselineError);
|
||||||
}
|
}
|
||||||
|
|
||||||
throw createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
|
throw this._markRemoteCommitUncertain(createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
|
||||||
phase: 'upload-result',
|
phase: 'upload-result',
|
||||||
endpoint: uploadServer,
|
endpoint: uploadServer,
|
||||||
contentType: headers && headers['content-type'],
|
contentType: headers && headers['content-type'],
|
||||||
body: rawBody,
|
body: rawBody,
|
||||||
hosterTransient: true,
|
hosterTransient: true,
|
||||||
retryable: true
|
retryable: true
|
||||||
});
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
async _resolveUploadedFile(fileName, baselineCodes, signal) {
|
async _resolveUploadedFile(fileName, baselineCodes, signal) {
|
||||||
@@ -478,11 +489,19 @@ class VoeUploader {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_normalizeTitle(value) {
|
_normalizeTitle(value) {
|
||||||
return String(value || '')
|
return normalizeRecoveryTitle(value);
|
||||||
.toLowerCase()
|
}
|
||||||
.normalize('NFKD')
|
|
||||||
.replace(/\.[a-z0-9]+$/i, '')
|
_markRemoteCommitUncertain(error) {
|
||||||
.replace(/[^a-z0-9]+/g, '');
|
if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
|
||||||
|
return this.recoveryClaim.markUncertain(error);
|
||||||
|
}
|
||||||
|
const uncertainError = error && typeof error === 'object'
|
||||||
|
? error
|
||||||
|
: new Error('VOE Upload-Ergebnis ist unsicher');
|
||||||
|
uncertainError.remoteCommitUncertain = true;
|
||||||
|
uncertainError.hosterTransient = true;
|
||||||
|
return uncertainError;
|
||||||
}
|
}
|
||||||
|
|
||||||
_buildUrls(fileCode, phase = 'upload-result') {
|
_buildUrls(fileCode, phase = 'upload-result') {
|
||||||
@@ -498,7 +517,7 @@ class VoeUploader {
|
|||||||
hosterTransient: true
|
hosterTransient: true
|
||||||
});
|
});
|
||||||
error.remoteIdentityClaimed = true;
|
error.remoteIdentityClaimed = true;
|
||||||
throw error;
|
throw this._markRemoteCommitUncertain(error);
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
download_url: `${BASE_URL}/${code}`,
|
download_url: `${BASE_URL}/${code}`,
|
||||||
|
|||||||
@@ -187,3 +187,46 @@ test('Vidmoly concurrent same-name recovery accepts distinct remote codes', asyn
|
|||||||
|
|
||||||
assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']);
|
assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
for (const scenario of [
|
||||||
|
{
|
||||||
|
label: 'VOE',
|
||||||
|
hoster: 'voe.sx',
|
||||||
|
Uploader: VoeUploader,
|
||||||
|
sharedCode: 'VOE_UNCERTAIN',
|
||||||
|
lateCode: 'VOE_LATE_CODE',
|
||||||
|
build(uploader, code) {
|
||||||
|
return uploader._buildUrls(code);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Vidmoly',
|
||||||
|
hoster: 'vidmoly.me',
|
||||||
|
Uploader: VidmolyUploader,
|
||||||
|
sharedCode: 'VIDUNCERTAIN',
|
||||||
|
lateCode: 'VIDLATECODE1',
|
||||||
|
build(uploader, code) {
|
||||||
|
return uploader._buildUrlsFromCode(code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]) {
|
||||||
|
test(`${scenario.label} marks a duplicate direct identity uncertain and blocks a later title match`, async () => {
|
||||||
|
const registry = createRecoveryClaimRegistry();
|
||||||
|
const firstClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'First Episode.mkv');
|
||||||
|
const uncertainClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'Second Episode.mkv');
|
||||||
|
const first = new scenario.Uploader(firstClaim);
|
||||||
|
const uncertain = new scenario.Uploader(uncertainClaim);
|
||||||
|
|
||||||
|
scenario.build(first, scenario.sharedCode);
|
||||||
|
assert.throws(
|
||||||
|
() => scenario.build(uncertain, scenario.sharedCode),
|
||||||
|
err => err.remoteIdentityClaimed === true && err.remoteCommitUncertain === true
|
||||||
|
);
|
||||||
|
|
||||||
|
const laterClaim = registry.forUpload(scenario.hoster, 'ACCOUNT', 'Second Episode.mp4');
|
||||||
|
await assert.rejects(
|
||||||
|
() => laterClaim.runExclusive(async () => scenario.build(new scenario.Uploader(laterClaim), scenario.lateCode)),
|
||||||
|
err => err.remoteCommitUncertain === true
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
+72
-1
@@ -1,7 +1,7 @@
|
|||||||
const { describe, it } = require('node:test');
|
const { describe, it } = require('node:test');
|
||||||
const assert = require('node:assert/strict');
|
const assert = require('node:assert/strict');
|
||||||
|
|
||||||
const { __test } = require('../lib/hosters');
|
const { __test, createRecoveryClaimRegistry } = require('../lib/hosters');
|
||||||
|
|
||||||
describe('hosters helpers', () => {
|
describe('hosters helpers', () => {
|
||||||
it('extracts VOE file_code from nested result payloads', () => {
|
it('extracts VOE file_code from nested result payloads', () => {
|
||||||
@@ -94,3 +94,74 @@ describe('hosters helpers', () => {
|
|||||||
assert.equal(r.embed_url, 'https://byse.sx/e/GOOD123');
|
assert.equal(r.embed_url, 'https://byse.sx/e/GOOD123');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('recovery claim registry', () => {
|
||||||
|
it('claims remote codes across every title of one normalized hoster account', () => {
|
||||||
|
const registry = createRecoveryClaimRegistry();
|
||||||
|
const first = registry.forUpload(' VOE.SX ', 'ACCOUNT', 'First Episode.mkv');
|
||||||
|
const differentTitle = registry.forUpload('voe.sx', 'ACCOUNT', 'Second Episode.mp4');
|
||||||
|
const differentAccount = registry.forUpload('voe.sx', 'ACCOUNT-B', 'Second Episode.mp4');
|
||||||
|
|
||||||
|
assert.equal(first.reserve('REMOTE-CODE'), true);
|
||||||
|
assert.equal(differentTitle.reserve('REMOTE-CODE'), false);
|
||||||
|
assert.equal(differentAccount.reserve('REMOTE-CODE'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('serializes canonically equivalent titles without blocking an independent title', async () => {
|
||||||
|
const registry = createRecoveryClaimRegistry();
|
||||||
|
const composed = registry.forUpload('voe.sx', 'ACCOUNT', 'Café.mkv');
|
||||||
|
const decomposed = registry.forUpload('voe.sx', 'ACCOUNT', 'Cafe\u0301.mp4');
|
||||||
|
const independent = registry.forUpload('voe.sx', 'ACCOUNT', 'Other Episode.mkv');
|
||||||
|
const events = [];
|
||||||
|
let releaseFirst;
|
||||||
|
const firstGate = new Promise(resolve => {
|
||||||
|
releaseFirst = resolve;
|
||||||
|
});
|
||||||
|
assert.equal(composed.reserve('UNICODE-CODE'), true);
|
||||||
|
assert.equal(decomposed.reserve('UNICODE-CODE'), false);
|
||||||
|
|
||||||
|
const first = composed.runExclusive(async () => {
|
||||||
|
events.push('first-started');
|
||||||
|
await firstGate;
|
||||||
|
events.push('first-finished');
|
||||||
|
});
|
||||||
|
await new Promise(resolve => setImmediate(resolve));
|
||||||
|
const equivalent = decomposed.runExclusive(async () => {
|
||||||
|
events.push('equivalent-started');
|
||||||
|
});
|
||||||
|
const other = independent.runExclusive(async () => {
|
||||||
|
events.push('independent-started');
|
||||||
|
});
|
||||||
|
await new Promise(resolve => setImmediate(resolve));
|
||||||
|
|
||||||
|
assert.deepEqual(events, ['first-started', 'independent-started']);
|
||||||
|
releaseFirst();
|
||||||
|
await Promise.all([first, equivalent, other]);
|
||||||
|
assert.deepEqual(events, ['first-started', 'independent-started', 'first-finished', 'equivalent-started']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fails closed for later jobs after a title becomes uncertain', async () => {
|
||||||
|
const registry = createRecoveryClaimRegistry();
|
||||||
|
const first = registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv');
|
||||||
|
const later = registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4');
|
||||||
|
const error = first.markUncertain(new Error('Remote commit could not be confirmed'));
|
||||||
|
|
||||||
|
assert.equal(error.remoteCommitUncertain, true);
|
||||||
|
assert.equal(error.hosterTransient, true);
|
||||||
|
await assert.rejects(
|
||||||
|
() => later.runExclusive(async () => 'unsafe-success'),
|
||||||
|
err => err.remoteCommitUncertain === true && err.hosterTransient === true
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('drops every claim when the registry is cleared', () => {
|
||||||
|
const registry = createRecoveryClaimRegistry();
|
||||||
|
const first = registry.forUpload('voe.sx', 'ACCOUNT', 'Episode.mkv');
|
||||||
|
assert.equal(first.reserve('REMOTE-CODE'), true);
|
||||||
|
|
||||||
|
registry.clear();
|
||||||
|
|
||||||
|
const nextBatch = registry.forUpload('voe.sx', 'ACCOUNT', 'Episode.mkv');
|
||||||
|
assert.equal(nextBatch.reserve('REMOTE-CODE'), true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -56,12 +56,12 @@ function settings(hoster, parallelCount) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runBatch(manager, tasks) {
|
async function runBatch(manager, tasks, options) {
|
||||||
let summary;
|
let summary;
|
||||||
manager.once('batch-done', value => {
|
manager.once('batch-done', value => {
|
||||||
summary = value;
|
summary = value;
|
||||||
});
|
});
|
||||||
await manager.startBatch(tasks);
|
await manager.startBatch(tasks, options);
|
||||||
return summary;
|
return summary;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +88,56 @@ function waitFor(promise, timeoutMs, message) {
|
|||||||
]).finally(() => clearTimeout(timer));
|
]).finally(() => clearTimeout(timer));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function assertTitleWaiterLeavesSlotAvailable(hosterParallel, globalSettings = {}) {
|
||||||
|
let releaseFirst;
|
||||||
|
let markFirstStarted;
|
||||||
|
let markIndependentStarted;
|
||||||
|
const firstGate = new Promise(resolve => {
|
||||||
|
releaseFirst = resolve;
|
||||||
|
});
|
||||||
|
const firstStarted = new Promise(resolve => {
|
||||||
|
markFirstStarted = resolve;
|
||||||
|
});
|
||||||
|
const independentStarted = new Promise(resolve => {
|
||||||
|
markIndependentStarted = resolve;
|
||||||
|
});
|
||||||
|
let sequence = 0;
|
||||||
|
loadManager(async (hoster, file) => {
|
||||||
|
if (file === firstPath) {
|
||||||
|
markFirstStarted();
|
||||||
|
await firstGate;
|
||||||
|
}
|
||||||
|
if (file === distinctPath) markIndependentStarted();
|
||||||
|
sequence++;
|
||||||
|
return {
|
||||||
|
file_code: `ADMISSION_${sequence}`,
|
||||||
|
download_url: `https://byse.sx/d/ADMISSION_${sequence}`
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const manager = new UploadManager(settings('byse.sx', hosterParallel), globalSettings);
|
||||||
|
const batch = runBatch(manager, [
|
||||||
|
{ jobId: 'admission-first', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||||
|
]);
|
||||||
|
|
||||||
|
await waitFor(firstStarted, 500, 'First upload did not start');
|
||||||
|
const added = manager.addJobs([
|
||||||
|
{ jobId: 'admission-waiter', file: secondPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
|
||||||
|
{ jobId: 'admission-independent', file: distinctPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }
|
||||||
|
]);
|
||||||
|
assert.equal(added.added, 2);
|
||||||
|
let admissionError = null;
|
||||||
|
try {
|
||||||
|
await waitFor(independentStarted, 500, 'Independent title was blocked behind a title-lock waiter');
|
||||||
|
} catch (err) {
|
||||||
|
admissionError = err;
|
||||||
|
} finally {
|
||||||
|
releaseFirst();
|
||||||
|
}
|
||||||
|
const summary = await batch;
|
||||||
|
if (admissionError) throw admissionError;
|
||||||
|
assert.equal(summary.succeeded, 3);
|
||||||
|
}
|
||||||
|
|
||||||
test('a batch shares recovery claims across normalized same-name jobs', async () => {
|
test('a batch shares recovery claims across normalized same-name jobs', async () => {
|
||||||
let unsafeCalls = 0;
|
let unsafeCalls = 0;
|
||||||
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||||
@@ -172,6 +222,93 @@ test('normalized same-name recovery sections never overlap', async () => {
|
|||||||
assert.equal(maximumActive, 1);
|
assert.equal(maximumActive, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('a title-lock waiter does not consume a scarce upload slot', async () => {
|
||||||
|
await assertTitleWaiterLeavesSlotAvailable(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a title-lock waiter does not consume a scarce global upload slot', async () => {
|
||||||
|
await assertTitleWaiterLeavesSlotAvailable(3, { parallelUploadCount: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an uncertain remote commit blocks retries, account fallback, and later same-title success', async () => {
|
||||||
|
const calls = [];
|
||||||
|
let markFirstStarted;
|
||||||
|
let releaseUncertain;
|
||||||
|
const firstStarted = new Promise(resolve => {
|
||||||
|
markFirstStarted = resolve;
|
||||||
|
});
|
||||||
|
const uncertainGate = new Promise(resolve => {
|
||||||
|
releaseUncertain = resolve;
|
||||||
|
});
|
||||||
|
loadManager(async (hoster, file, apiKey) => {
|
||||||
|
calls.push({ file, apiKey });
|
||||||
|
if (file === firstPath) {
|
||||||
|
markFirstStarted();
|
||||||
|
await uncertainGate;
|
||||||
|
const error = new Error('Remote commit could not be confirmed');
|
||||||
|
error.remoteCommitUncertain = true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
file_code: 'LATE_REMOTE_CODE',
|
||||||
|
download_url: 'https://byse.sx/d/LATE_REMOTE_CODE'
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const hosterSettings = settings('byse.sx', 2);
|
||||||
|
hosterSettings['byse.sx'].retries = 2;
|
||||||
|
const manager = new UploadManager(hosterSettings);
|
||||||
|
const fallback = { id: 'ACCOUNT_B', apiKey: 'ACCOUNT_KEY_B' };
|
||||||
|
|
||||||
|
const batch = runBatch(manager, [
|
||||||
|
{
|
||||||
|
jobId: 'uncertain-first',
|
||||||
|
file: firstPath,
|
||||||
|
hoster: 'byse.sx',
|
||||||
|
accountId: 'ACCOUNT_A',
|
||||||
|
apiKey: 'ACCOUNT_KEY_A'
|
||||||
|
}
|
||||||
|
], { primeOverrides: [['byse.sx', fallback]] });
|
||||||
|
await waitFor(firstStarted, 500, 'Uncertain predecessor did not start');
|
||||||
|
const added = manager.addJobs([
|
||||||
|
{
|
||||||
|
jobId: 'uncertain-later',
|
||||||
|
file: secondPath,
|
||||||
|
hoster: 'byse.sx',
|
||||||
|
accountId: 'ACCOUNT_A',
|
||||||
|
apiKey: 'ACCOUNT_KEY_A'
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
assert.equal(added.added, 1);
|
||||||
|
releaseUncertain();
|
||||||
|
const summary = await batch;
|
||||||
|
|
||||||
|
assert.equal(summary.succeeded, 0);
|
||||||
|
assert.equal(summary.failed, 2);
|
||||||
|
assert.deepEqual(calls, [{ file: firstPath, apiKey: 'ACCOUNT_KEY_A' }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('recovery claims do not leak into a later batch on the same manager', async () => {
|
||||||
|
loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
|
||||||
|
if (!options.recoveryClaim.reserve('REUSED_BATCH_CODE')) {
|
||||||
|
const error = new Error('Remote recovery candidate already claimed');
|
||||||
|
error.hosterTransient = true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
file_code: 'REUSED_BATCH_CODE',
|
||||||
|
download_url: 'https://byse.sx/d/REUSED_BATCH_CODE'
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const manager = new UploadManager(settings('byse.sx', 1));
|
||||||
|
const task = { jobId: 'batch-one', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' };
|
||||||
|
|
||||||
|
const first = await runBatch(manager, [task]);
|
||||||
|
const second = await runBatch(manager, [{ ...task, jobId: 'batch-two' }]);
|
||||||
|
|
||||||
|
assert.equal(first.succeeded, 1);
|
||||||
|
assert.equal(second.succeeded, 1);
|
||||||
|
});
|
||||||
|
|
||||||
for (const scenario of [
|
for (const scenario of [
|
||||||
{
|
{
|
||||||
label: 'VOE',
|
label: 'VOE',
|
||||||
@@ -263,6 +400,41 @@ for (const scenario of [
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test(`${scenario.label} uploader instances reject one direct remote code across different titles`, async () => {
|
||||||
|
await withUploaderMethods(
|
||||||
|
scenario.Uploader,
|
||||||
|
async function () {
|
||||||
|
await new Promise(resolve => setImmediate(resolve));
|
||||||
|
return scenario.buildResult(this, scenario.sharedCode);
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
loadManager();
|
||||||
|
const manager = new UploadManager(settings(scenario.hoster, 2));
|
||||||
|
const summary = await runBatch(manager, [
|
||||||
|
{
|
||||||
|
jobId: `${scenario.label}-different-title-a`,
|
||||||
|
file: firstPath,
|
||||||
|
hoster: scenario.hoster,
|
||||||
|
accountId: 'LOGIN_ACCOUNT',
|
||||||
|
username: 'account@example.test',
|
||||||
|
password: 'password'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
jobId: `${scenario.label}-different-title-b`,
|
||||||
|
file: distinctPath,
|
||||||
|
hoster: scenario.hoster,
|
||||||
|
accountId: 'LOGIN_ACCOUNT',
|
||||||
|
username: 'account@example.test',
|
||||||
|
password: 'password'
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(summary.succeeded, 1);
|
||||||
|
assert.equal(summary.failed, 1);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => {
|
test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => {
|
||||||
let active = 0;
|
let active = 0;
|
||||||
let maximumActive = 0;
|
let maximumActive = 0;
|
||||||
|
|||||||
Reference in New Issue
Block a user