Harden batch-wide recovery identity claims

Scope remote code ownership to normalized hoster and account identities while retaining title-only recovery serialization and canonical Unicode title matching.

Mark post-upload ambiguity and duplicate identities as uncertain so retries, account fallback, and later same-title jobs fail closed instead of reporting unsafe success.

Acquire recovery title leases before hoster and global semaphores, revalidate failed-account overrides before upload, and clear claim state at batch boundaries.

Add deterministic concurrent coverage for same-code rejection, distinct-code parallel success, uncertainty propagation, semaphore fairness, account isolation, Unicode equivalence, and registry lifetime.
This commit is contained in:
Sucukdeluxe
2026-08-13 23:09:03 +02:00
parent dd14381e43
commit c78160a521
7 changed files with 640 additions and 141 deletions
+122 -34
View File
@@ -575,7 +575,40 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') {
}
function _normalizeFileTitle(s) {
return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, '');
return String(s || '')
.normalize('NFKD')
.toLowerCase()
.replace(/\.[\p{Letter}\p{Number}]+$/u, '')
.replace(/\p{Mark}+/gu, '')
.replace(/[^\p{Letter}\p{Number}]+/gu, '');
}
function _normalizeRecoveryHoster(value) {
return String(value || '').normalize('NFKC').trim().toLowerCase();
}
function _normalizeRecoveryAccount(value) {
return String(value || '').normalize('NFKC').trim();
}
function _createRecoveryUncertainError() {
const error = new Error('Upload-Ergebnis für diesen Titel ist wegen eines möglichen Remote-Commits unsicher');
error.remoteCommitUncertain = true;
error.hosterTransient = true;
return error;
}
function _markRecoveryUncertain(recoveryClaim, error) {
if (!recoveryClaim) return error;
if (typeof recoveryClaim.markUncertain === 'function') {
return recoveryClaim.markUncertain(error);
}
const uncertainError = error && typeof error === 'object'
? error
: _createRecoveryUncertainError();
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
}
function _createAbortError() {
@@ -607,40 +640,69 @@ function _waitForRecoveryTurn(predecessor, signal) {
}
function createRecoveryClaimRegistry() {
const scopes = new Map();
const accounts = new Map();
let nextClaimId = 1;
return {
forUpload(hosterName, apiKey, fileName) {
const identity = crypto.createHash('sha256')
.update(`${String(hosterName || '').toLowerCase()}\0${String(apiKey || '')}\0${_normalizeFileTitle(fileName)}`)
const accountIdentity = crypto.createHash('sha256')
.update(`${_normalizeRecoveryHoster(hosterName)}\0${_normalizeRecoveryAccount(apiKey)}`)
.digest('hex');
let codes = scopes.get(identity);
if (!codes) {
codes = {
values: new Set(),
tail: Promise.resolve()
let account = accounts.get(accountIdentity);
if (!account) {
account = {
codes: new Map(),
titles: new Map()
};
scopes.set(identity, codes);
accounts.set(accountIdentity, account);
}
const titleIdentity = _normalizeFileTitle(fileName);
let title = account.titles.get(titleIdentity);
if (!title) {
title = {
tail: Promise.resolve(),
uncertain: false
};
account.titles.set(titleIdentity, title);
}
const claimId = nextClaimId++;
return {
has(code) {
return codes.values.has(String(code || '').trim());
return account.codes.has(String(code || '').trim());
},
reserve(code) {
const normalized = String(code || '').trim();
if (!normalized || codes.values.has(normalized)) return false;
codes.values.add(normalized);
if (!normalized) return false;
if (account.codes.has(normalized)) {
return account.codes.get(normalized) === claimId;
}
account.codes.set(normalized, claimId);
return true;
},
markUncertain(error) {
title.uncertain = true;
const uncertainError = error && typeof error === 'object'
? error
: _createRecoveryUncertainError();
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
},
isUncertain() {
return title.uncertain;
},
async runExclusive(operation, signal) {
const predecessor = codes.tail;
const predecessor = title.tail;
let release;
const current = new Promise(resolve => {
release = resolve;
});
codes.tail = predecessor.then(() => current);
title.tail = predecessor.then(() => current, () => current);
try {
await _waitForRecoveryTurn(predecessor, signal);
return await operation();
if (title.uncertain) throw _createRecoveryUncertainError();
const result = await operation();
if (title.uncertain) throw _createRecoveryUncertainError();
return result;
} finally {
release();
}
@@ -648,7 +710,7 @@ function createRecoveryClaimRegistry() {
};
},
clear() {
scopes.clear();
accounts.clear();
}
};
}
@@ -805,23 +867,28 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
bodyTimeout: UPLOAD_TIMEOUT
});
} catch (err) {
if (signal && signal.aborted) throw err;
throw createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
const error = signal && signal.aborted ? err : createTransportError(`Upload zu ${hosterName} konnte nicht übertragen werden`, {
phase: 'upload-request',
endpoint: targetUrl,
retryable: true,
transientNetwork: true
});
throw _markRecoveryUncertain(opts && opts.recoveryClaim, error);
}
const { body, statusCode, headers } = uploadResponse;
const rawBody = await body.text();
let rawBody;
try {
rawBody = await body.text();
} catch (err) {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
}
let payload = null;
try {
payload = rawBody ? JSON.parse(rawBody) : {};
} catch {
throw createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload-Antwort von ${hosterName} war kein JSON`, {
phase: 'upload-response',
endpoint: targetUrl,
httpStatus: statusCode,
@@ -829,7 +896,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
body: rawBody,
retryable: statusCode >= 500,
transientNetwork: statusCode >= 500
});
}));
}
// Normalize valid-but-not-object JSON (JSON.parse('null') → null;
// JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this
@@ -843,7 +910,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
}
if (statusCode < 200 || statusCode >= 300) {
throw createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
const error = createTransportError(`Upload zu ${hosterName} fehlgeschlagen`, {
phase: 'upload-response',
endpoint: targetUrl,
httpStatus: statusCode,
@@ -852,10 +919,13 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
retryable: statusCode === 429 || statusCode >= 500,
transientNetwork: statusCode >= 500
});
throw statusCode >= 500
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
: error;
}
if (payload.status && [401, 403, 429, 500].includes(payload.status)) {
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
const error = createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
phase: 'upload-response',
endpoint: targetUrl,
httpStatus: Number(payload.status),
@@ -864,6 +934,9 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
retryable: Number(payload.status) === 429 || Number(payload.status) >= 500,
transientNetwork: Number(payload.status) >= 500
});
throw Number(payload.status) >= 500
? _markRecoveryUncertain(opts && opts.recoveryClaim, error)
: error;
}
let result = null;
@@ -885,7 +958,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
if (result && (result.file_code || result.download_url || result.embed_url)) {
if (result.file_code && opts && opts.recoveryClaim && typeof opts.recoveryClaim.reserve === 'function') {
if (!opts.recoveryClaim.reserve(result.file_code)) {
throw createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
const error = createTransportError(`Upload zu ${hosterName} lieferte eine bereits zugeordnete file_code-Antwort`, {
phase: 'upload-result',
endpoint: targetUrl,
httpStatus: statusCode,
@@ -894,6 +967,8 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
retryable: true,
hosterTransient: true
});
error.remoteIdentityClaimed = true;
throw _markRecoveryUncertain(opts.recoveryClaim, error);
}
}
return result;
@@ -921,8 +996,12 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// even after our uploader gave up.
if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) {
const fileName = path.basename(filePath);
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
try {
const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
} catch (err) {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
}
}
// Doodstream: the doodapi upload POST returned no filecode (the same backend
@@ -930,21 +1009,29 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// the file did register, claim its code instead of failing the upload.
if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) {
const fileName = path.basename(filePath);
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
try {
const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal, opts && opts.recoveryClaim);
if (polled) return polled;
} catch (err) {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, err);
}
}
if (hosterName === 'byse.sx' && byseBaselineError && !explicitlyRejected) {
byseBaselineError.hosterTransient = true;
throw byseBaselineError;
throw _markRecoveryUncertain(opts && opts.recoveryClaim, byseBaselineError);
}
if (hosterName === 'doodstream.com' && doodBaselineError && !explicitlyRejected) {
doodBaselineError.hosterTransient = true;
throw doodBaselineError;
throw _markRecoveryUncertain(opts && opts.recoveryClaim, doodBaselineError);
}
if (parseErr) throw parseErr;
if (parseErr) {
throw explicitlyRejected
? parseErr
: _markRecoveryUncertain(opts && opts.recoveryClaim, parseErr);
}
if (payload.success === false) {
throw createTransportError(`Upload zu ${hosterName} wurde vom Server abgelehnt`, {
@@ -970,7 +1057,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
// so tag it hosterTransient: the upload-manager then fails this file WITHOUT
// blacklisting the account (same protection the web path got in 3.3.29) and
// the account stays usable for the next retry/batch.
throw createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
throw _markRecoveryUncertain(opts && opts.recoveryClaim, createTransportError(`Upload zu ${hosterName} lieferte keine file_code-Antwort`, {
phase: 'upload-result',
endpoint: targetUrl,
httpStatus: statusCode,
@@ -978,7 +1065,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
body: rawBody,
retryable: true,
hosterTransient: true
});
}));
}
throw createTransportError(`Upload zu ${hosterName} wurde abgelehnt`, {
phase: 'upload-result',
@@ -1007,6 +1094,7 @@ module.exports = {
uploadFile,
prefetchBaseline,
createRecoveryClaimRegistry,
normalizeRecoveryTitle: _normalizeFileTitle,
HOSTER_CONFIGS,
__test: {
extractUploadServerUrl,
+157 -73
View File
@@ -71,7 +71,7 @@ class UploadManager extends EventEmitter {
this._suspectGoodAccounts.clear();
this._doodApiKeyCache.clear();
this._baselineCache.clear();
this._recoveryClaims.clear();
if (!this.running) this._recoveryClaims.clear();
}
switchAccount(hoster, fallbackAccount) {
@@ -134,6 +134,41 @@ class UploadManager extends EventEmitter {
return true;
}
_swapFailedAccount(task, jobId, fileName) {
if (!task.accountId || !this._failedAccounts.has(task.hoster + ':' + task.accountId)) return false;
const override = this._accountOverrides.get(task.hoster);
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
this._rotLog('pre-job-swap', {
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
});
task.accountId = override.id;
task.username = override.username;
task.password = override.password;
task.apiKey = override.apiKey;
return true;
}
this._rotLog('pre-job-swap-blocked', {
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
hasOverride: !!override,
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
});
return false;
}
async _coordinateAccountFailure(task, err, signal, jobId) {
if (!task.accountId || (err && err.remoteCommitUncertain === true)) return;
if (!this._shouldSkipRetryOnAccountError(err)) return;
const key = task.hoster + ':' + task.accountId;
if (this._failedAccounts.has(key)) return;
this._failedAccounts.set(key, true);
this._rotLog('mark-failed', {
jobId, hoster: task.hoster, fileName: path.basename(task.file),
accountId: task.accountId, lastError: err && err.message ? err.message : String(err)
});
this.emit('account-failed', { hoster: task.hoster, accountId: task.accountId });
await this._sleep(800, signal);
}
_rotLog(event, data) {
this.emit('rot-log', { ts: Date.now(), event, ...data });
}
@@ -161,6 +196,7 @@ class UploadManager extends EventEmitter {
// which takes priority in _shouldSkipRetryOnAccountError.
_isFileRejectedError(err) {
if (!err) return false;
if (err.remoteCommitUncertain === true) return false;
if (err.transientNetwork === true) return false;
if (err.accountError === true) return false; // explicit account-level wins
if (err.fileRejected === true) return true;
@@ -355,6 +391,7 @@ class UploadManager extends EventEmitter {
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
this._recoveryClaims.clear();
this._recoveryClaims = createRecoveryClaimRegistry();
this.semaphores = {};
this.globalSemaphore = null;
this.globalThrottle = null;
@@ -440,13 +477,12 @@ class UploadManager extends EventEmitter {
files
};
this._recoveryClaims.clear();
this.emit('batch-done', summary);
}
async _runJob(task, results, batchSignal) {
const settings = this._getSettings(task.hoster);
const hosterSemaphore = this._getSemaphore(task.hoster);
const globalSemaphore = this._getGlobalSemaphore();
const uploadId = crypto.randomBytes(8).toString('hex');
const jobId = task.jobId || uploadId;
const fileName = path.basename(task.file);
@@ -466,8 +502,6 @@ class UploadManager extends EventEmitter {
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
this.jobAbortControllers.set(jobId, jobAbortController);
let hosterSlotAcquired = false;
let globalSlotAcquired = false;
let finalResultRecorded = false;
let finalStatus = 'error';
let lastError = null;
@@ -549,9 +583,6 @@ class UploadManager extends EventEmitter {
// queueJobs array; the first event it actually needs from main is the
// 'getting-server' / 'uploading' transition for the jobs that the
// semaphore lets through.
await hosterSemaphore.acquire(signal);
hosterSlotAcquired = true;
let fileProbe = null;
try {
fileProbe = await probeFileHead(task.file, 512);
@@ -566,11 +597,6 @@ class UploadManager extends EventEmitter {
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
});
if (globalSemaphore) {
await globalSemaphore.acquire(signal);
globalSlotAcquired = true;
}
if (settings.timeIntervalSec > 0) {
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal);
}
@@ -580,24 +606,7 @@ class UploadManager extends EventEmitter {
// of burning a guaranteed-to-fail upload attempt. Critical at scale:
// with 500 queued jobs and 1 parallel slot, without this check every
// job still hits the original dead account first.
if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) {
const override = this._accountOverrides.get(task.hoster);
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
this._rotLog('pre-job-swap', {
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
});
task.accountId = override.id;
task.username = override.username;
task.password = override.password;
task.apiKey = override.apiKey;
} else {
this._rotLog('pre-job-swap-blocked', {
jobId, hoster: task.hoster, fileName, accountId: task.accountId,
hasOverride: !!override,
overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false
});
}
}
this._swapFailedAccount(task, jobId, fileName);
// A previous file of at least this size already got a suspect rejection
// on this exact account — skip the guaranteed-to-fail multi-GB upload
@@ -735,7 +744,7 @@ class UploadManager extends EventEmitter {
} catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ }
};
const result = await this._executeUpload(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe);
const result = await this._executeUploadWithAdmission(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe, true, jobId);
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
@@ -784,6 +793,11 @@ class UploadManager extends EventEmitter {
break;
}
if (err && err.remoteCommitUncertain === true) {
lastError = err;
break;
}
if (isSpeedRestart && attempt < maxAttempts) {
lastError = new Error('Geschwindigkeit zu niedrig - Neustart');
await this._sleep(3000, signal);
@@ -1047,7 +1061,7 @@ class UploadManager extends EventEmitter {
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
: hosterThrottle || globalThrottle;
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, true, jobId);
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
this.activeJobs.delete(uploadId);
this.sessionBytes += fileSize;
@@ -1104,9 +1118,6 @@ class UploadManager extends EventEmitter {
this.activeJobs.delete(uploadId);
this.jobAbortControllers.delete(jobId);
cleanupSignals();
// Release in reverse order of acquire (global first, then hoster)
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
if (hosterSlotAcquired) hosterSemaphore.release();
this.emit('job-settled', {
jobId,
sourceCleanupToken: task.sourceCleanupToken || null,
@@ -1190,7 +1201,7 @@ class UploadManager extends EventEmitter {
? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } }
: hosterThrottle || globalThrottle;
try {
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
const result = await this._executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, false, jobId);
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
this.activeJobs.delete(uploadId);
this.sessionBytes += fileSize;
@@ -1210,6 +1221,7 @@ class UploadManager extends EventEmitter {
});
}
if (signal.aborted || this.stopAfterActive) break;
if (err && err.remoteCommitUncertain === true) throw err;
if (err && err.suspectReject === true) {
this._noteSuspectReject(task.hoster, account.id, fileSize);
}
@@ -1241,16 +1253,95 @@ class UploadManager extends EventEmitter {
return null;
}
async _executeUpload(task, progressCb, signal, throttle, fileProbe) {
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe);
return assertUploadConfirmation(result, task.hoster);
async _executeUploadWithAdmission(task, progressCb, signal, throttle, fileProbe, coordinateAccountFailure = true, jobId = task.jobId) {
while (true) {
const context = await this._createRecoveryContext(task);
let retryAdmission = false;
const operation = async () => {
const hosterSemaphore = this._getSemaphore(task.hoster);
const globalSemaphore = this._getGlobalSemaphore();
let hosterSlotAcquired = false;
let globalSlotAcquired = false;
try {
await hosterSemaphore.acquire(signal);
hosterSlotAcquired = true;
if (globalSemaphore) {
await globalSemaphore.acquire(signal);
globalSlotAcquired = true;
}
if (this._swapFailedAccount(task, jobId, path.basename(task.file))) {
retryAdmission = true;
return null;
}
try {
return await this._executeUpload(task, progressCb, signal, throttle, fileProbe, context);
} catch (err) {
if (coordinateAccountFailure) await this._coordinateAccountFailure(task, err, signal, jobId);
if (context.recoveryClaim && err && err.remoteCommitUncertain === true) {
throw context.recoveryClaim.markUncertain(err);
}
throw err;
}
} finally {
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
if (hosterSlotAcquired) hosterSemaphore.release();
}
};
const result = context.recoveryClaim
? await context.recoveryClaim.runExclusive(operation, signal)
: await operation();
if (retryAdmission) continue;
return result;
}
}
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) {
async _executeUpload(task, progressCb, signal, throttle, fileProbe, context) {
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context);
let confirmed;
try {
confirmed = assertUploadConfirmation(result, task.hoster);
} catch (err) {
if (context.recoveryClaim) throw context.recoveryClaim.markUncertain(err);
throw err;
}
return confirmed;
}
async _createRecoveryContext(task) {
const fileName = path.basename(task.file);
if ((task.hoster === 'vidmoly.me' || task.hoster === 'voe.sx') && task.username) {
const accountIdentity = task.accountId !== null && task.accountId !== undefined
? task.accountId
: String(task.username || '').trim().toLowerCase();
return {
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
doodApiKey: null
};
}
if (task.hoster === 'doodstream.com' && task.username) {
const doodApiKey = await this._resolveDoodstreamApiKey(task);
const accountIdentity = doodApiKey || (task.accountId !== null && task.accountId !== undefined
? task.accountId
: String(task.username || '').trim().toLowerCase());
return {
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName),
doodApiKey
};
}
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
return {
recoveryClaim: this._recoveryClaims.forUpload(task.hoster, task.apiKey, fileName),
doodApiKey: null
};
}
return { recoveryClaim: null, doodApiKey: null };
}
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context) {
if (task.hoster === 'vidmoly.me' && task.username) {
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle);
return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle, context.recoveryClaim);
} else if (task.hoster === 'voe.sx' && task.username) {
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle);
return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle, context.recoveryClaim);
} else if (task.hoster === 'doodstream.com' && task.username) {
// Login-path reliability fix: the web-form upload returns the filecode in
// an HTML form that comes back empty for large files (doodstream backend
@@ -1258,54 +1349,47 @@ class UploadManager extends EventEmitter {
// session ONCE per batch and upload via the official API instead — it
// returns result[0].filecode directly and has no empty-form failure mode.
// Falls back to the web-form upload if no valid key can be derived.
const apiKey = await this._resolveDoodstreamApiKey(task);
const apiKey = context.doodApiKey;
if (apiKey) {
this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe);
return this._executeRecoveryAwareApiUpload('doodstream.com', task.file, apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
}
this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
const dood = new DoodstreamUploader();
await dood.login(task.username, task.password);
return dood.upload(task.file, progressCb, signal, throttle);
const result = await dood.upload(task.file, progressCb, signal, throttle);
if (result && result.file_code && !context.recoveryClaim.reserve(result.file_code)) {
const error = new Error('Doodstream Upload lieferte eine bereits zugeordnete Remote-Identität');
error.remoteIdentityClaimed = true;
throw context.recoveryClaim.markUncertain(error);
}
return result;
} else if (task.hoster === 'clouddrop.cc') {
const clouddrop = new ClouddropUploader(task.apiKey);
return clouddrop.upload(task.file, progressCb, signal, throttle);
} else {
if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') {
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe);
return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim);
}
return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {});
}
}
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle) {
const accountIdentity = task.accountId !== null && task.accountId !== undefined
? task.accountId
: String(task.username || '').trim().toLowerCase();
const recoveryClaim = this._recoveryClaims.forUpload(
task.hoster,
accountIdentity,
path.basename(task.file)
);
return recoveryClaim.runExclusive(async () => {
const uploader = new UploaderClass(recoveryClaim);
await uploader.login(task.username, task.password);
return uploader.upload(task.file, progressCb, signal, throttle);
}, signal);
async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle, recoveryClaim) {
const uploader = new UploaderClass(recoveryClaim);
await uploader.login(task.username, task.password);
return uploader.upload(task.file, progressCb, signal, throttle);
}
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
return recoveryClaim.runExclusive(async () => {
const options = { recoveryClaim };
if (hosterName === 'byse.sx') {
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
} else {
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
}
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
}, signal);
async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe, recoveryClaim) {
const options = { recoveryClaim };
if (hosterName === 'byse.sx') {
options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal);
if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true;
} else {
options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal);
}
return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options);
}
_getBaseline(hosterName, apiKey, signal) {
+38 -16
View File
@@ -3,6 +3,7 @@ const path = require('path');
const crypto = require('crypto');
const { request } = require('undici');
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
const { normalizeRecoveryTitle } = require('./hosters');
const BASE_URL = 'https://vidmoly.me';
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
@@ -275,13 +276,13 @@ class VidmolyUploader {
bodyTimeout: UPLOAD_TIMEOUT
});
} catch (err) {
if (signal && signal.aborted) throw err;
throw createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
const error = signal && signal.aborted ? err : createTransportError('Vidmoly Upload konnte nicht übertragen werden', {
phase: 'upload-request',
endpoint: targetUrl,
retryable: true,
transientNetwork: true
});
throw this._markRemoteCommitUncertain(error);
}
const { body, statusCode, headers } = uploadResponse;
@@ -295,17 +296,25 @@ class VidmolyUploader {
// Always drain the original body to prevent connection leak
try { await body.text(); } catch {}
if (location) {
const resultRes = await this._fetch(new URL(location, uploadUrl).href);
resultHtml = await resultRes.text();
try {
const resultRes = await this._fetch(new URL(location, uploadUrl).href);
resultHtml = await resultRes.text();
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
} else {
resultHtml = '';
}
} else {
resultHtml = await body.text();
try {
resultHtml = await body.text();
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
}
if (statusCode >= 400) {
throw createTransportError('Vidmoly Upload fehlgeschlagen', {
const error = createTransportError('Vidmoly Upload fehlgeschlagen', {
phase: 'upload-response',
endpoint: targetUrl,
httpStatus: statusCode,
@@ -314,6 +323,7 @@ class VidmolyUploader {
retryable: statusCode === 429 || statusCode >= 500,
transientNetwork: statusCode >= 500
});
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
}
// Try JSON first. The current transit server returns
@@ -355,23 +365,35 @@ class VidmolyUploader {
} catch (primaryErr) {
if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr;
if (baselineCodes) {
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
if (fallback) return fallback;
try {
const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
if (fallback) return fallback;
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
}
if (baselineError) {
baselineError.hosterTransient = true;
throw baselineError;
throw this._markRemoteCommitUncertain(baselineError);
}
throw primaryErr;
throw this._markRemoteCommitUncertain(primaryErr);
}
}
_normalizeTitle(value) {
return String(value || '')
.toLowerCase()
.normalize('NFKD')
.replace(/\.[a-z0-9]+$/i, '')
.replace(/[^a-z0-9]+/g, '');
return normalizeRecoveryTitle(value);
}
_markRemoteCommitUncertain(error) {
if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
return this.recoveryClaim.markUncertain(error);
}
const uncertainError = error && typeof error === 'object'
? error
: new Error('Vidmoly Upload-Ergebnis ist unsicher');
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
}
_buildUrlsFromCode(fileCode, phase = 'upload-result') {
@@ -387,7 +409,7 @@ class VidmolyUploader {
hosterTransient: true
});
error.remoteIdentityClaimed = true;
throw error;
throw this._markRemoteCommitUncertain(error);
}
return {
+34 -15
View File
@@ -3,6 +3,7 @@ const path = require('path');
const crypto = require('crypto');
const { request } = require('undici');
const { createTransportError, sanitizeRemoteText } = require('./hoster-transport-error');
const { normalizeRecoveryTitle } = require('./hosters');
const BASE_URL = 'https://voe.sx';
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
@@ -352,22 +353,27 @@ class VoeUploader {
bodyTimeout: UPLOAD_TIMEOUT
});
} catch (err) {
if (signal && signal.aborted) throw err;
throw createTransportError('VOE Upload konnte nicht übertragen werden', {
const error = signal && signal.aborted ? err : createTransportError('VOE Upload konnte nicht übertragen werden', {
phase: 'upload-request',
endpoint: uploadServer,
retryable: true,
transientNetwork: true
});
throw this._markRemoteCommitUncertain(error);
}
const { body, headers, statusCode } = uploadResponse;
this._parseCookiesFromHeaders(headers || {});
const rawBody = await body.text();
let rawBody;
try {
rawBody = await body.text();
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
if (statusCode < 200 || statusCode >= 300) {
throw createTransportError('VOE Upload fehlgeschlagen', {
const error = createTransportError('VOE Upload fehlgeschlagen', {
phase: 'upload-response',
endpoint: uploadServer,
httpStatus: statusCode,
@@ -376,6 +382,7 @@ class VoeUploader {
retryable: statusCode === 429 || statusCode >= 500,
transientNetwork: statusCode >= 500
});
throw statusCode >= 500 ? this._markRemoteCommitUncertain(error) : error;
}
// Try JSON response
@@ -407,23 +414,27 @@ class VoeUploader {
// Fallback: poll the file list to find the newly uploaded file
if (baselineCodes) {
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
if (result) return result;
try {
const result = await this._resolveUploadedFile(fileName, baselineCodes, signal);
if (result) return result;
} catch (err) {
throw this._markRemoteCommitUncertain(err);
}
}
if (baselineError) {
baselineError.hosterTransient = true;
throw baselineError;
throw this._markRemoteCommitUncertain(baselineError);
}
throw createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
throw this._markRemoteCommitUncertain(createTransportError('VOE Upload: Kein file_code in der Antwort gefunden', {
phase: 'upload-result',
endpoint: uploadServer,
contentType: headers && headers['content-type'],
body: rawBody,
hosterTransient: true,
retryable: true
});
}));
}
async _resolveUploadedFile(fileName, baselineCodes, signal) {
@@ -478,11 +489,19 @@ class VoeUploader {
}
_normalizeTitle(value) {
return String(value || '')
.toLowerCase()
.normalize('NFKD')
.replace(/\.[a-z0-9]+$/i, '')
.replace(/[^a-z0-9]+/g, '');
return normalizeRecoveryTitle(value);
}
_markRemoteCommitUncertain(error) {
if (this.recoveryClaim && typeof this.recoveryClaim.markUncertain === 'function') {
return this.recoveryClaim.markUncertain(error);
}
const uncertainError = error && typeof error === 'object'
? error
: new Error('VOE Upload-Ergebnis ist unsicher');
uncertainError.remoteCommitUncertain = true;
uncertainError.hosterTransient = true;
return uncertainError;
}
_buildUrls(fileCode, phase = 'upload-result') {
@@ -498,7 +517,7 @@ class VoeUploader {
hosterTransient: true
});
error.remoteIdentityClaimed = true;
throw error;
throw this._markRemoteCommitUncertain(error);
}
return {
download_url: `${BASE_URL}/${code}`,