diff --git a/lib/hosters.js b/lib/hosters.js index 3a45bf5..64e826f 100644 --- a/lib/hosters.js +++ b/lib/hosters.js @@ -575,12 +575,17 @@ async function _fetchByseFileList(apiKey, signal, phase = 'recovery-poll') { } function _normalizeFileTitle(s) { - return String(s || '') + const normalized = String(s || '') .normalize('NFKD') .toLowerCase() .replace(/\.[\p{Letter}\p{Number}]+$/u, '') + .replace(/\p{Variation_Selector}+/gu, ''); + const alphanumeric = normalized .replace(/\p{Mark}+/gu, '') .replace(/[^\p{Letter}\p{Number}]+/gu, ''); + if (alphanumeric) return alphanumeric; + const codePoints = Array.from(normalized, value => value.codePointAt(0).toString(16)).join('-'); + return `symbols:${codePoints}`; } function _normalizeRecoveryHoster(value) { diff --git a/lib/upload-manager.js b/lib/upload-manager.js index c554781..ca82a97 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -1310,9 +1310,7 @@ class UploadManager extends EventEmitter { async _createRecoveryContext(task) { const fileName = path.basename(task.file); if ((task.hoster === 'vidmoly.me' || task.hoster === 'voe.sx') && task.username) { - const accountIdentity = task.accountId !== null && task.accountId !== undefined - ? task.accountId - : String(task.username || '').trim().toLowerCase(); + const accountIdentity = this._recoveryAccountIdentity(task); return { recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName), doodApiKey: null @@ -1320,23 +1318,31 @@ class UploadManager extends EventEmitter { } if (task.hoster === 'doodstream.com' && task.username) { const doodApiKey = await this._resolveDoodstreamApiKey(task); - const accountIdentity = doodApiKey || (task.accountId !== null && task.accountId !== undefined - ? task.accountId - : String(task.username || '').trim().toLowerCase()); + const accountIdentity = this._recoveryAccountIdentity(task, doodApiKey); return { recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName), doodApiKey }; } - if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') { + if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com' || task.hoster === 'voe.sx') { + const accountIdentity = task.hoster === 'byse.sx' + ? task.apiKey + : this._recoveryAccountIdentity(task); return { - recoveryClaim: this._recoveryClaims.forUpload(task.hoster, task.apiKey, fileName), + recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName), doodApiKey: null }; } return { recoveryClaim: null, doodApiKey: null }; } + _recoveryAccountIdentity(task, fallbackIdentity = null) { + for (const value of [task.accountId, task.apiKey, fallbackIdentity]) { + if (value !== null && value !== undefined && String(value).trim()) return value; + } + return String(task.username || '').normalize('NFKC').trim().toLowerCase(); + } + async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context) { if (task.hoster === 'vidmoly.me' && task.username) { return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle, context.recoveryClaim); @@ -1357,7 +1363,15 @@ class UploadManager extends EventEmitter { this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) }); const dood = new DoodstreamUploader(); await dood.login(task.username, task.password); - const result = await dood.upload(task.file, progressCb, signal, throttle); + let result; + try { + result = await dood.upload(task.file, progressCb, signal, throttle); + } catch (err) { + if (context.recoveryClaim && this._isDoodstreamRemoteCommitUncertain(err)) { + throw context.recoveryClaim.markUncertain(err); + } + throw err; + } if (result && result.file_code && !context.recoveryClaim.reserve(result.file_code)) { const error = new Error('Doodstream Upload lieferte eine bereits zugeordnete Remote-Identität'); error.remoteIdentityClaimed = true; @@ -1368,7 +1382,7 @@ class UploadManager extends EventEmitter { const clouddrop = new ClouddropUploader(task.apiKey); return clouddrop.upload(task.file, progressCb, signal, throttle); } else { - if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com') { + if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com' || task.hoster === 'voe.sx') { return this._executeRecoveryAwareApiUpload(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, fileProbe, context.recoveryClaim); } return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, {}); @@ -1386,12 +1400,22 @@ class UploadManager extends EventEmitter { if (hosterName === 'byse.sx') { options.byseBaseline = await this._getBaseline(hosterName, apiKey, signal); if (fileProbe && fileProbe.ok !== false) options.probeIsVideoLike = fileProbe.isVideoLike === true; - } else { + } else if (hosterName === 'doodstream.com') { options.doodBaseline = await this._getBaseline(hosterName, apiKey, signal); } return uploadFile(hosterName, filePath, apiKey, progressCb, signal, throttle, options); } + _isDoodstreamRemoteCommitUncertain(error) { + if (!error || typeof error !== 'object') return false; + if (error.remoteCommitUncertain === true) return true; + if (error.accountError === true || error.fileRejected === true) return false; + const phase = error.diagnostic && error.diagnostic.phase; + if (phase === 'upload-request') return true; + return (phase === 'upload-response' || phase === 'upload-result-submit' || phase === 'upload-result') + && (error.hosterTransient === true || error.transientNetwork === true); + } + _getBaseline(hosterName, apiKey, signal) { if (!apiKey) return Promise.resolve(null); const key = `${hosterName}:${apiKey}`; @@ -1408,19 +1432,27 @@ class UploadManager extends EventEmitter { // so a 40-file batch logs in + derives ONCE, not per file). The empty-string // sentinel distinguishes "tried, none" from "not yet tried" (undefined). async _resolveDoodstreamApiKey(task) { - const cacheKey = task.accountId || task.username; + const accountId = task.accountId !== null && task.accountId !== undefined + ? String(task.accountId).normalize('NFKC').trim() + : ''; + const cacheKey = accountId + ? `account:${accountId}` + : `username:${String(task.username || '').normalize('NFKC').trim().toLowerCase()}`; const cached = this._doodApiKeyCache.get(cacheKey); - if (cached !== undefined) return cached || null; + if (cached !== undefined) return (await cached) || null; - let key = ''; - try { - const probe = new DoodstreamUploader(); - await probe.login(task.username, task.password); - key = (await probe.deriveApiKey()) || ''; - } catch { - key = ''; - } - this._doodApiKeyCache.set(cacheKey, key); + const pending = (async () => { + try { + const probe = new DoodstreamUploader(); + await probe.login(task.username, task.password); + return (await probe.deriveApiKey()) || ''; + } catch { + return ''; + } + })(); + this._doodApiKeyCache.set(cacheKey, pending); + const key = await pending; + if (this._doodApiKeyCache.get(cacheKey) === pending) this._doodApiKeyCache.set(cacheKey, key); return key || null; } diff --git a/tests/hosters.test.js b/tests/hosters.test.js index 5556d9d..8dd318b 100644 --- a/tests/hosters.test.js +++ b/tests/hosters.test.js @@ -1,7 +1,7 @@ const { describe, it } = require('node:test'); const assert = require('node:assert/strict'); -const { __test, createRecoveryClaimRegistry } = require('../lib/hosters'); +const { __test, createRecoveryClaimRegistry, normalizeRecoveryTitle } = require('../lib/hosters'); describe('hosters helpers', () => { it('extracts VOE file_code from nested result payloads', () => { @@ -96,6 +96,19 @@ describe('hosters helpers', () => { }); describe('recovery claim registry', () => { + it('keeps symbol-only titles distinct while matching Unicode-equivalent forms', () => { + const gear = normalizeRecoveryTitle('⚙.mkv'); + const emojiGear = normalizeRecoveryTitle('⚙️.mp4'); + const fire = normalizeRecoveryTitle('🔥.mkv'); + const joined = normalizeRecoveryTitle('👩‍💻.mkv'); + const unjoined = normalizeRecoveryTitle('👩💻.mkv'); + + assert.ok(gear); + assert.equal(emojiGear, gear); + assert.notEqual(fire, gear); + assert.notEqual(joined, unjoined); + }); + it('claims remote codes across every title of one normalized hoster account', () => { const registry = createRecoveryClaimRegistry(); const first = registry.forUpload(' VOE.SX ', 'ACCOUNT', 'First Episode.mkv'); diff --git a/tests/upload-manager-recovery-claims.test.js b/tests/upload-manager-recovery-claims.test.js index 80fa304..d0c6cd6 100644 --- a/tests/upload-manager-recovery-claims.test.js +++ b/tests/upload-manager-recovery-claims.test.js @@ -5,6 +5,7 @@ const os = require('node:os'); const path = require('node:path'); const hosters = require('../lib/hosters'); +const DoodstreamUploader = require('../lib/doodstream-upload'); const VoeUploader = require('../lib/voe-upload'); const VidmolyUploader = require('../lib/vidmoly-upload'); const originalUploadFile = hosters.uploadFile; @@ -78,6 +79,21 @@ async function withUploaderMethods(Uploader, upload, operation) { } } +async function withDoodstreamMethods(methods, operation) { + const originals = {}; + for (const [name, method] of Object.entries(methods)) { + originals[name] = DoodstreamUploader.prototype[name]; + DoodstreamUploader.prototype[name] = method; + } + try { + return await operation(); + } finally { + for (const [name, method] of Object.entries(originals)) { + DoodstreamUploader.prototype[name] = method; + } + } +} + function waitFor(promise, timeoutMs, message) { let timer; return Promise.race([ @@ -309,6 +325,266 @@ test('recovery claims do not leak into a later batch on the same manager', async assert.equal(second.succeeded, 1); }); +test('VOE API and login auth paths share account-wide remote code claims', async () => { + const sharedCode = 'VOEALLAUTH01'; + await withUploaderMethods( + VoeUploader, + async function () { + await new Promise(resolve => setImmediate(resolve)); + return this._buildUrls(sharedCode); + }, + async () => { + loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => { + const claim = options && options.recoveryClaim; + if (claim && !claim.reserve(sharedCode)) { + const error = new Error('Remote identity already claimed'); + error.hosterTransient = true; + throw error; + } + return { + file_code: sharedCode, + download_url: `https://voe.sx/${sharedCode}`, + embed_url: `https://voe.sx/e/${sharedCode}` + }; + }); + const manager = new UploadManager(settings('voe.sx', 2)); + + const summary = await runBatch(manager, [ + { + jobId: 'voe-login-auth', + file: firstPath, + hoster: 'voe.sx', + accountId: 'VOE_SHARED_ACCOUNT', + username: 'account@example.test', + password: 'password' + }, + { + jobId: 'voe-api-auth', + file: distinctPath, + hoster: 'voe.sx', + accountId: 'VOE_SHARED_ACCOUNT', + apiKey: 'VOE_API_KEY' + } + ]); + + assert.equal(summary.succeeded, 1); + assert.equal(summary.failed, 1); + } + ); +}); + +test('an uncertain VOE API upload blocks a later same-title login upload', async () => { + let markApiStarted; + let releaseApi; + let loginUploads = 0; + const apiStarted = new Promise(resolve => { + markApiStarted = resolve; + }); + const apiGate = new Promise(resolve => { + releaseApi = resolve; + }); + await withUploaderMethods( + VoeUploader, + async function () { + loginUploads++; + return this._buildUrls('UNSAFEVOELOGIN'); + }, + async () => { + loadManager(async () => { + markApiStarted(); + await apiGate; + const error = new Error('VOE API result could not be confirmed'); + error.remoteCommitUncertain = true; + throw error; + }); + const manager = new UploadManager(settings('voe.sx', 2)); + const batch = runBatch(manager, [ + { + jobId: 'voe-api-uncertain', + file: firstPath, + hoster: 'voe.sx', + accountId: 'VOE_SHARED_ACCOUNT', + apiKey: 'VOE_API_KEY' + } + ]); + + await waitFor(apiStarted, 500, 'VOE API upload did not start'); + const added = manager.addJobs([ + { + jobId: 'voe-login-later', + file: secondPath, + hoster: 'voe.sx', + accountId: 'VOE_SHARED_ACCOUNT', + username: 'account@example.test', + password: 'password' + } + ]); + assert.equal(added.added, 1); + releaseApi(); + const summary = await batch; + + assert.equal(summary.succeeded, 0); + assert.equal(summary.failed, 2); + assert.equal(loginUploads, 0); + } + ); +}); + +test('a keyless Doodstream web ambiguity blocks a later same-title upload', async () => { + let markUploadStarted; + let releaseUpload; + let uploadCalls = 0; + const uploadStarted = new Promise(resolve => { + markUploadStarted = resolve; + }); + const uploadGate = new Promise(resolve => { + releaseUpload = resolve; + }); + await withDoodstreamMethods({ + login: async function () {}, + deriveApiKey: async function () { + return null; + }, + upload: async function () { + uploadCalls++; + if (uploadCalls === 1) { + markUploadStarted(); + await uploadGate; + const error = new Error('Doodstream returned an empty upload result'); + error.hosterTransient = true; + error.diagnostic = { phase: 'upload-result' }; + throw error; + } + return { + file_code: 'UNSAFE_DOOD_CODE', + download_url: 'https://doodstream.com/d/UNSAFE_DOOD_CODE', + embed_url: 'https://doodstream.com/e/UNSAFE_DOOD_CODE' + }; + } + }, async () => { + loadManager(); + const manager = new UploadManager(settings('doodstream.com', 2)); + const batch = runBatch(manager, [ + { + jobId: 'dood-web-uncertain', + file: firstPath, + hoster: 'doodstream.com', + accountId: 'DOOD_SHARED_ACCOUNT', + username: 'account@example.test', + password: 'password' + } + ]); + + await waitFor(uploadStarted, 500, 'Doodstream web upload did not start'); + const added = manager.addJobs([ + { + jobId: 'dood-web-later', + file: secondPath, + hoster: 'doodstream.com', + accountId: 'DOOD_SHARED_ACCOUNT', + username: 'account@example.test', + password: 'password' + } + ]); + assert.equal(added.added, 1); + releaseUpload(); + const summary = await batch; + + assert.equal(summary.succeeded, 0); + assert.equal(summary.failed, 2); + assert.equal(uploadCalls, 1); + }); +}); + +test('Doodstream key resolution is singleflight per account', async () => { + let releaseLogin; + let loginCalls = 0; + let deriveCalls = 0; + const loginGate = new Promise(resolve => { + releaseLogin = resolve; + }); + await withDoodstreamMethods({ + login: async function () { + loginCalls++; + await loginGate; + }, + deriveApiKey: async function () { + deriveCalls++; + return 'DERIVED_DOOD_KEY'; + } + }, async () => { + loadManager(); + const manager = new UploadManager(settings('doodstream.com', 12)); + const resolutions = Array.from({ length: 12 }, (_, index) => manager._resolveDoodstreamApiKey({ + accountId: 'DOOD_SHARED_ACCOUNT', + username: `account-${index}@example.test`, + password: 'password' + })); + const queuedLoginCalls = loginCalls; + + releaseLogin(); + const keys = await Promise.all(resolutions); + + assert.equal(queuedLoginCalls, 1); + assert.equal(loginCalls, 1); + assert.equal(deriveCalls, 1); + assert.deepEqual(keys, Array(12).fill('DERIVED_DOOD_KEY')); + }); +}); + +test('Doodstream web and API auth paths use one canonical account claim identity', async () => { + const sharedCode = 'DOODALLAUTH01'; + await withDoodstreamMethods({ + login: async function () {}, + deriveApiKey: async function () { + return null; + }, + upload: async function () { + await new Promise(resolve => setImmediate(resolve)); + return { + file_code: sharedCode, + download_url: `https://doodstream.com/d/${sharedCode}`, + embed_url: `https://doodstream.com/e/${sharedCode}` + }; + } + }, async () => { + loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => { + if (!options.recoveryClaim.reserve(sharedCode)) { + const error = new Error('Remote identity already claimed'); + error.hosterTransient = true; + throw error; + } + return { + file_code: sharedCode, + download_url: `https://doodstream.com/d/${sharedCode}`, + embed_url: `https://doodstream.com/e/${sharedCode}` + }; + }); + const manager = new UploadManager(settings('doodstream.com', 2)); + + const summary = await runBatch(manager, [ + { + jobId: 'dood-web-auth', + file: firstPath, + hoster: 'doodstream.com', + accountId: 'DOOD_SHARED_ACCOUNT', + username: 'account@example.test', + password: 'password' + }, + { + jobId: 'dood-api-auth', + file: distinctPath, + hoster: 'doodstream.com', + accountId: 'DOOD_SHARED_ACCOUNT', + apiKey: 'DOOD_API_KEY' + } + ]); + + assert.equal(summary.succeeded, 1); + assert.equal(summary.failed, 1); + }); +}); + for (const scenario of [ { label: 'VOE',