diff --git a/lib/upload-manager.js b/lib/upload-manager.js index 3239121..64f7fb7 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -1248,13 +1248,9 @@ class UploadManager extends EventEmitter { async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) { if (task.hoster === 'vidmoly.me' && task.username) { - const vidmoly = new VidmolyUploader(); - await vidmoly.login(task.username, task.password); - return vidmoly.upload(task.file, progressCb, signal, throttle); + return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle); } else if (task.hoster === 'voe.sx' && task.username) { - const voe = new VoeUploader(); - await voe.login(task.username, task.password); - return voe.upload(task.file, progressCb, signal, throttle); + return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle); } else if (task.hoster === 'doodstream.com' && task.username) { // Login-path reliability fix: the web-form upload returns the filecode in // an HTML form that comes back empty for large files (doodstream backend @@ -1282,6 +1278,22 @@ class UploadManager extends EventEmitter { } } + async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle) { + const accountIdentity = task.accountId !== null && task.accountId !== undefined + ? task.accountId + : String(task.username || '').trim().toLowerCase(); + const recoveryClaim = this._recoveryClaims.forUpload( + task.hoster, + accountIdentity, + path.basename(task.file) + ); + return recoveryClaim.runExclusive(async () => { + const uploader = new UploaderClass(recoveryClaim); + await uploader.login(task.username, task.password); + return uploader.upload(task.file, progressCb, signal, throttle); + }, signal); + } + async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) { const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath)); return recoveryClaim.runExclusive(async () => { diff --git a/lib/vidmoly-upload.js b/lib/vidmoly-upload.js index 5a6db92..9757852 100644 --- a/lib/vidmoly-upload.js +++ b/lib/vidmoly-upload.js @@ -14,8 +14,9 @@ const RESULT_POLL_DELAY_MS = 2000; * XFileSharing-based upload for Vidmoly (login + form upload) */ class VidmolyUploader { - constructor() { + constructor(recoveryClaim = null) { this.cookies = new Map(); + this.recoveryClaim = recoveryClaim; } _cookieHeader() { @@ -352,6 +353,7 @@ class VidmolyUploader { try { return this._parseUploadResult(resultHtml); } catch (primaryErr) { + if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr; if (baselineCodes) { const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal); if (fallback) return fallback; @@ -372,9 +374,21 @@ class VidmolyUploader { .replace(/[^a-z0-9]+/g, ''); } - _buildUrlsFromCode(fileCode) { + _buildUrlsFromCode(fileCode, phase = 'upload-result') { const code = String(fileCode || '').trim(); if (!code) return null; + if (this.recoveryClaim + && typeof this.recoveryClaim.reserve === 'function' + && !this.recoveryClaim.reserve(code)) { + const error = createTransportError('Vidmoly Upload-Ergebnis ist bereits einem anderen Upload zugeordnet', { + phase, + endpoint: BASE_URL, + retryable: true, + hosterTransient: true + }); + error.remoteIdentityClaimed = true; + throw error; + } return { download_url: `${BASE_URL}/w/${code}`, @@ -473,14 +487,18 @@ class VidmolyUploader { const withCode = files.filter((f) => f && typeof f.file_code === 'string' && f.file_code.trim()); const newFiles = withCode.filter((f) => !baselineCodes.has(f.file_code.trim())); - const matches = newFiles.filter((file) => { - const title = this._normalizeTitle(file.full_title || file.title_txt || ''); - return expectedTitle && title === expectedTitle; - }); + const matches = newFiles + .filter((file) => { + const title = this._normalizeTitle(file.full_title || file.title_txt || ''); + return expectedTitle && title === expectedTitle; + }) + .filter((file) => !this.recoveryClaim + || typeof this.recoveryClaim.has !== 'function' + || !this.recoveryClaim.has(file.file_code.trim())); if (matches.length > 1) return null; if (matches.length === 1) { - return this._buildUrlsFromCode(matches[0].file_code); + return this._buildUrlsFromCode(matches[0].file_code, 'recovery-poll'); } if (attempt < RESULT_POLL_ATTEMPTS - 1) { @@ -577,12 +595,10 @@ class VidmolyUploader { if (codeInPage) file_code = codeInPage[1]; } - // Build URLs from file_code - if (file_code && !download_url) { - download_url = `${BASE_URL}/w/${file_code}`; - } - if (file_code && !embed_url) { - embed_url = `${BASE_URL}/embed-${file_code}.html`; + if (file_code) { + const urls = this._buildUrlsFromCode(file_code); + if (!download_url) download_url = urls.download_url; + if (!embed_url) embed_url = urls.embed_url; } if (!download_url && !file_code) { diff --git a/lib/voe-upload.js b/lib/voe-upload.js index f03f5c0..54df019 100644 --- a/lib/voe-upload.js +++ b/lib/voe-upload.js @@ -15,8 +15,9 @@ const RESULT_POLL_DELAY_MS = 2000; * Fallback when API-based upload fails or is unavailable. */ class VoeUploader { - constructor() { + constructor(recoveryClaim = null) { this.cookies = new Map(); + this.recoveryClaim = recoveryClaim; } _cookieHeader() { @@ -449,15 +450,22 @@ class VoeUploader { const withCode = files.filter(f => f && (f.file_code || f.slug)); const newFiles = withCode.filter(f => !baselineCodes.has(String(f.file_code || f.slug || '').trim())); - const matches = newFiles.filter(file => { - const title = this._normalizeTitle(file.title || file.name || ''); - return expectedTitle && title === expectedTitle; - }); + const matches = newFiles + .filter(file => { + const title = this._normalizeTitle(file.title || file.name || ''); + return expectedTitle && title === expectedTitle; + }) + .filter(file => { + const code = String(file.file_code || file.slug || '').trim(); + return !this.recoveryClaim + || typeof this.recoveryClaim.has !== 'function' + || !this.recoveryClaim.has(code); + }); if (matches.length > 1) return null; if (matches.length === 1) { const code = matches[0].file_code || matches[0].slug; - return this._buildUrls(code); + return this._buildUrls(code, 'recovery-poll'); } if (attempt < RESULT_POLL_ATTEMPTS - 1) { @@ -477,9 +485,21 @@ class VoeUploader { .replace(/[^a-z0-9]+/g, ''); } - _buildUrls(fileCode) { + _buildUrls(fileCode, phase = 'upload-result') { const code = String(fileCode || '').trim(); if (!code) return null; + if (this.recoveryClaim + && typeof this.recoveryClaim.reserve === 'function' + && !this.recoveryClaim.reserve(code)) { + const error = createTransportError('VOE Upload-Ergebnis ist bereits einem anderen Upload zugeordnet', { + phase, + endpoint: BASE_URL, + retryable: true, + hosterTransient: true + }); + error.remoteIdentityClaimed = true; + throw error; + } return { download_url: `${BASE_URL}/${code}`, embed_url: `${BASE_URL}/e/${code}`, diff --git a/tests/hoster-recovery-provenance.test.js b/tests/hoster-recovery-provenance.test.js index 7830d88..25521ca 100644 --- a/tests/hoster-recovery-provenance.test.js +++ b/tests/hoster-recovery-provenance.test.js @@ -3,6 +3,7 @@ const assert = require('node:assert/strict'); const VoeUploader = require('../lib/voe-upload'); const VidmolyUploader = require('../lib/vidmoly-upload'); +const { createRecoveryClaimRegistry } = require('../lib/hosters'); function response(body, status = 200, contentType = 'application/json') { return { @@ -116,3 +117,73 @@ test('Vidmoly preserves a failed recovery baseline as a safe structured error', } ); }); + +test('VOE concurrent same-name recovery claims one remote entry only once', async () => { + const registry = createRecoveryClaimRegistry(); + const first = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'Shared Episode.mkv')); + const second = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'shared-episode.mp4')); + const remoteFiles = [{ file_code: 'VOE_SHARED', title: 'shared episode' }]; + first._fetchFileList = async () => remoteFiles; + second._fetchFileList = async () => remoteFiles; + first._sleep = async () => {}; + second._sleep = async () => {}; + + const results = await Promise.all([ + first._resolveUploadedFile('C:\\source-a\\Shared Episode.mkv', new Set(), null), + second._resolveUploadedFile('D:\\source-b\\shared-episode.mp4', new Set(), null) + ]); + + assert.deepEqual(results.filter(Boolean).map(result => result.file_code), ['VOE_SHARED']); +}); + +test('VOE concurrent same-name recovery accepts distinct remote codes', async () => { + const registry = createRecoveryClaimRegistry(); + const first = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'Shared Episode.mkv')); + const second = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'shared-episode.mp4')); + first._fetchFileList = async () => [{ file_code: 'VOE_FIRST', title: 'shared episode' }]; + second._fetchFileList = async () => [{ file_code: 'VOE_SECOND', title: 'shared episode' }]; + first._sleep = async () => {}; + second._sleep = async () => {}; + + const results = await Promise.all([ + first._resolveUploadedFile('C:\\source-a\\Shared Episode.mkv', new Set(), null), + second._resolveUploadedFile('D:\\source-b\\shared-episode.mp4', new Set(), null) + ]); + + assert.deepEqual(results.map(result => result.file_code), ['VOE_FIRST', 'VOE_SECOND']); +}); + +test('Vidmoly concurrent same-name recovery claims one remote entry only once', async () => { + const registry = createRecoveryClaimRegistry(); + const first = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv')); + const second = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4')); + const remoteFiles = [{ file_code: 'VIDSHARED001', full_title: 'shared episode' }]; + first._fetchVmList = async () => remoteFiles; + second._fetchVmList = async () => remoteFiles; + first._sleep = async () => {}; + second._sleep = async () => {}; + + const results = await Promise.all([ + first._resolveUploadedFileFromVmApi('C:\\source-a\\Shared Episode.mkv', new Set(), null), + second._resolveUploadedFileFromVmApi('D:\\source-b\\shared-episode.mp4', new Set(), null) + ]); + + assert.deepEqual(results.filter(Boolean).map(result => result.file_code), ['VIDSHARED001']); +}); + +test('Vidmoly concurrent same-name recovery accepts distinct remote codes', async () => { + const registry = createRecoveryClaimRegistry(); + const first = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv')); + const second = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4')); + first._fetchVmList = async () => [{ file_code: 'VIDFIRST0001', full_title: 'shared episode' }]; + second._fetchVmList = async () => [{ file_code: 'VIDSECOND001', full_title: 'shared episode' }]; + first._sleep = async () => {}; + second._sleep = async () => {}; + + const results = await Promise.all([ + first._resolveUploadedFileFromVmApi('C:\\source-a\\Shared Episode.mkv', new Set(), null), + second._resolveUploadedFileFromVmApi('D:\\source-b\\shared-episode.mp4', new Set(), null) + ]); + + assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']); +}); diff --git a/tests/upload-manager-recovery-claims.test.js b/tests/upload-manager-recovery-claims.test.js index f8e7956..854b473 100644 --- a/tests/upload-manager-recovery-claims.test.js +++ b/tests/upload-manager-recovery-claims.test.js @@ -5,11 +5,14 @@ const os = require('node:os'); const path = require('node:path'); const hosters = require('../lib/hosters'); +const VoeUploader = require('../lib/voe-upload'); +const VidmolyUploader = require('../lib/vidmoly-upload'); const originalUploadFile = hosters.uploadFile; const originalPrefetchBaseline = hosters.prefetchBaseline; let tempRoot; let firstPath; let secondPath; +let distinctPath; let UploadManager; before(() => { @@ -20,8 +23,10 @@ before(() => { fs.mkdirSync(secondDir); firstPath = path.join(firstDir, 'Shared Episode.mkv'); secondPath = path.join(secondDir, 'shared-episode.mp4'); + distinctPath = path.join(secondDir, 'different-title.mkv'); fs.writeFileSync(firstPath, Buffer.alloc(1024, 1)); fs.writeFileSync(secondPath, Buffer.alloc(1024, 2)); + fs.writeFileSync(distinctPath, Buffer.alloc(1024, 3)); }); after(() => { @@ -31,16 +36,16 @@ after(() => { fs.rmSync(tempRoot, { recursive: true, force: true }); }); -function loadManager(uploadFile) { +function loadManager(uploadFile = originalUploadFile) { hosters.uploadFile = uploadFile; hosters.prefetchBaseline = async () => new Set(); delete require.cache[require.resolve('../lib/upload-manager')]; UploadManager = require('../lib/upload-manager'); } -function settings(parallelCount) { +function settings(hoster, parallelCount) { return { - 'byse.sx': { + [hoster]: { retries: 0, parallelCount, maxSpeedKbs: 0, @@ -60,6 +65,29 @@ async function runBatch(manager, tasks) { return summary; } +async function withUploaderMethods(Uploader, upload, operation) { + const originalLogin = Uploader.prototype.login; + const originalUpload = Uploader.prototype.upload; + Uploader.prototype.login = async function () {}; + Uploader.prototype.upload = upload; + try { + return await operation(); + } finally { + Uploader.prototype.login = originalLogin; + Uploader.prototype.upload = originalUpload; + } +} + +function waitFor(promise, timeoutMs, message) { + let timer; + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), timeoutMs); + }) + ]).finally(() => clearTimeout(timer)); +} + test('a batch shares recovery claims across normalized same-name jobs', async () => { let unsafeCalls = 0; loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => { @@ -81,7 +109,7 @@ test('a batch shares recovery claims across normalized same-name jobs', async () error.hosterTransient = true; throw error; }); - const manager = new UploadManager(settings(2)); + const manager = new UploadManager(settings('byse.sx', 2)); const summary = await runBatch(manager, [ { jobId: 'same-name-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }, @@ -107,7 +135,7 @@ test('recovery claims stay isolated between accounts', async () => { download_url: 'https://byse.sx/d/SHARED_REMOTE_CODE' }; }); - const manager = new UploadManager(settings(2)); + const manager = new UploadManager(settings('byse.sx', 2)); const summary = await runBatch(manager, [ { jobId: 'account-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_A' }, @@ -133,7 +161,7 @@ test('normalized same-name recovery sections never overlap', async () => { download_url: `https://byse.sx/d/SERIAL_${sequence}` }; }); - const manager = new UploadManager(settings(2)); + const manager = new UploadManager(settings('byse.sx', 2)); const summary = await runBatch(manager, [ { jobId: 'serialized-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' }, @@ -143,3 +171,147 @@ test('normalized same-name recovery sections never overlap', async () => { assert.equal(summary.succeeded, 2); assert.equal(maximumActive, 1); }); + +for (const scenario of [ + { + label: 'VOE', + hoster: 'voe.sx', + Uploader: VoeUploader, + sharedCode: 'SHAREDVOE01', + distinctCodes: ['VOEDISTINCT1', 'VOEDISTINCT2'], + buildResult(uploader, code) { + return uploader._buildUrls(code); + } + }, + { + label: 'Vidmoly', + hoster: 'vidmoly.me', + Uploader: VidmolyUploader, + sharedCode: 'SHAREDVID001', + distinctCodes: ['VIDDISTINCT1', 'VIDDISTINCT2'], + buildResult(uploader, code) { + return uploader._buildUrlsFromCode(code); + } + } +]) { + test(`${scenario.label} uploader instances reject a duplicate direct remote code for same-name sources`, async () => { + await withUploaderMethods( + scenario.Uploader, + async function () { + await new Promise(resolve => setImmediate(resolve)); + return scenario.buildResult(this, scenario.sharedCode); + }, + async () => { + loadManager(); + const manager = new UploadManager(settings(scenario.hoster, 2)); + const summary = await runBatch(manager, [ + { + jobId: `${scenario.label}-same-name-a`, + file: firstPath, + hoster: scenario.hoster, + accountId: 'LOGIN_ACCOUNT', + username: 'account@example.test', + password: 'password' + }, + { + jobId: `${scenario.label}-same-name-b`, + file: secondPath, + hoster: scenario.hoster, + accountId: 'LOGIN_ACCOUNT', + username: 'account@example.test', + password: 'password' + } + ]); + + assert.equal(summary.succeeded, 1); + assert.equal(summary.failed, 1); + } + ); + }); + + test(`${scenario.label} uploader instances isolate direct remote code claims between accounts`, async () => { + await withUploaderMethods( + scenario.Uploader, + async function () { + return scenario.buildResult(this, scenario.sharedCode); + }, + async () => { + loadManager(); + const manager = new UploadManager(settings(scenario.hoster, 2)); + const summary = await runBatch(manager, [ + { + jobId: `${scenario.label}-account-a`, + file: firstPath, + hoster: scenario.hoster, + accountId: 'LOGIN_ACCOUNT_A', + username: 'account-a@example.test', + password: 'password' + }, + { + jobId: `${scenario.label}-account-b`, + file: secondPath, + hoster: scenario.hoster, + accountId: 'LOGIN_ACCOUNT_B', + username: 'account-b@example.test', + password: 'password' + } + ]); + + assert.equal(summary.succeeded, 2); + assert.equal(summary.failed, 0); + } + ); + }); + + test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => { + let active = 0; + let maximumActive = 0; + let started = 0; + let releaseBoth; + const bothStarted = new Promise(resolve => { + releaseBoth = resolve; + }); + await withUploaderMethods( + scenario.Uploader, + async function (filePath) { + active++; + started++; + maximumActive = Math.max(maximumActive, active); + if (started === 2) releaseBoth(); + try { + await waitFor(bothStarted, 500, 'Distinct uploads did not overlap'); + const code = filePath === firstPath ? scenario.distinctCodes[0] : scenario.distinctCodes[1]; + return scenario.buildResult(this, code); + } finally { + active--; + } + }, + async () => { + loadManager(); + const manager = new UploadManager(settings(scenario.hoster, 2)); + const summary = await runBatch(manager, [ + { + jobId: `${scenario.label}-distinct-a`, + file: firstPath, + hoster: scenario.hoster, + accountId: 'LOGIN_ACCOUNT', + username: 'account@example.test', + password: 'password' + }, + { + jobId: `${scenario.label}-distinct-b`, + file: distinctPath, + hoster: scenario.hoster, + accountId: 'LOGIN_ACCOUNT', + username: 'account@example.test', + password: 'password' + } + ]); + + assert.equal(summary.succeeded, 2); + assert.equal(summary.failed, 0); + assert.equal(maximumActive, 2); + } + ); + }); +}