diff --git a/lib/doodstream-upload.js b/lib/doodstream-upload.js index d6a56a0..4860ba7 100644 --- a/lib/doodstream-upload.js +++ b/lib/doodstream-upload.js @@ -393,7 +393,7 @@ class DoodstreamUploader { let uploadRes; try { - uploadRes = await request(uploadUrl, { + uploadRes = await this._requestUpload(uploadUrl, { method: 'POST', headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}`, @@ -412,7 +412,8 @@ class DoodstreamUploader { phase: 'upload-request', endpoint: uploadUrl, retryable: true, - transientNetwork: true + transientNetwork: true, + remoteCommitUncertain: true }); } @@ -429,7 +430,18 @@ class DoodstreamUploader { } } - const resText = await uploadRes.body.text(); + let resText; + try { + resText = await uploadRes.body.text(); + } catch { + throw createTransportError('Doodstream Upload-Antwort konnte nicht gelesen werden', { + phase: 'upload-response-read', + endpoint: uploadUrl, + retryable: true, + transientNetwork: true, + remoteCommitUncertain: true + }); + } const uploadContentType = uploadRes.headers && uploadRes.headers['content-type']; _debugLog(`Upload response: ${summarizeResponse(resText, uploadContentType)}`); @@ -451,13 +463,34 @@ class DoodstreamUploader { return this._parseUploadResponse(resText); } + _requestUpload(url, options) { + return request(url, options); + } + /** * Follow a redirect URL from upload server and extract filecode */ async _handleUploadResult(url) { _debugLog(`Following upload result URL: ${safeEndpoint(url) || 'unknown endpoint'}`); - const res = await this._fetch(url); - const html = await res.text(); + let res; + try { + res = await this._fetch(url); + } catch (error) { + if (error && typeof error === 'object') error.remoteCommitUncertain = true; + throw error; + } + let html; + try { + html = await res.text(); + } catch { + throw createTransportError('Doodstream Ergebnis-Antwort konnte nicht gelesen werden', { + phase: 'upload-response-read', + endpoint: url, + retryable: true, + transientNetwork: true, + remoteCommitUncertain: true + }); + } const contentType = res.headers && typeof res.headers.get === 'function' ? res.headers.get('content-type') : ''; _debugLog(`Result page: ${summarizeResponse(html, contentType)}`); return this._parseUploadResponse(html); diff --git a/lib/hoster-transport-error.js b/lib/hoster-transport-error.js index 59261b6..e710a94 100644 --- a/lib/hoster-transport-error.js +++ b/lib/hoster-transport-error.js @@ -79,6 +79,7 @@ function createTransportError(message, options = {}) { if (options.hosterTransient === true) error.hosterTransient = true; if (options.accountError === true) error.accountError = true; if (options.fileRejected === true) error.fileRejected = true; + if (options.remoteCommitUncertain === true) error.remoteCommitUncertain = true; return error; } diff --git a/lib/upload-manager.js b/lib/upload-manager.js index ca82a97..76a2a2a 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -3,7 +3,7 @@ const path = require('path'); const { assertUploadConfirmation } = require('./upload-confirmation'); const fs = require('fs'); const crypto = require('crypto'); -const { uploadFile, prefetchBaseline, createRecoveryClaimRegistry } = require('./hosters'); +const { uploadFile, prefetchBaseline, createRecoveryClaimRegistry, normalizeRecoveryTitle } = require('./hosters'); const VidmolyUploader = require('./vidmoly-upload'); const VoeUploader = require('./voe-upload'); const DoodstreamUploader = require('./doodstream-upload'); @@ -53,6 +53,7 @@ class UploadManager extends EventEmitter { this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none) this._baselineCache = new Map(); // hoster:apiKey -> Promise> (one fetch shared across all jobs in batch) this._recoveryClaims = createRecoveryClaimRegistry(); + this._recoveryAuthModes = new Map(); this._batchJobIds = new Set(); this._batchTotal = 0; } @@ -72,6 +73,7 @@ class UploadManager extends EventEmitter { this._doodApiKeyCache.clear(); this._baselineCache.clear(); if (!this.running) this._recoveryClaims.clear(); + if (!this.running) this._recoveryAuthModes.clear(); } switchAccount(hoster, fallbackAccount) { @@ -392,6 +394,7 @@ class UploadManager extends EventEmitter { this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance) this._recoveryClaims.clear(); this._recoveryClaims = createRecoveryClaimRegistry(); + this._recoveryAuthModes.clear(); this.semaphores = {}; this.globalSemaphore = null; this.globalThrottle = null; @@ -478,6 +481,9 @@ class UploadManager extends EventEmitter { }; this._recoveryClaims.clear(); + this._recoveryAuthModes.clear(); + this._doodApiKeyCache.clear(); + this._baselineCache.clear(); this.emit('batch-done', summary); } @@ -597,10 +603,6 @@ class UploadManager extends EventEmitter { headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null }); - if (settings.timeIntervalSec > 0) { - await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal); - } - // Pre-job-swap: if this account was marked failed WHILE this task was // waiting in the semaphore queue, jump straight to the override instead // of burning a guaranteed-to-fail upload attempt. Critical at scale: @@ -1273,6 +1275,10 @@ class UploadManager extends EventEmitter { retryAdmission = true; return null; } + const settings = this._getSettings(task.hoster); + if (settings.timeIntervalSec > 0) { + await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal); + } try { return await this._executeUpload(task, progressCb, signal, throttle, fileProbe, context); } catch (err) { @@ -1312,24 +1318,24 @@ class UploadManager extends EventEmitter { if ((task.hoster === 'vidmoly.me' || task.hoster === 'voe.sx') && task.username) { const accountIdentity = this._recoveryAccountIdentity(task); return { - recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName), + recoveryClaim: this._createRecoveryClaim(task, accountIdentity, fileName), doodApiKey: null }; } if (task.hoster === 'doodstream.com' && task.username) { const doodApiKey = await this._resolveDoodstreamApiKey(task); - const accountIdentity = this._recoveryAccountIdentity(task, doodApiKey); + const accountIdentity = doodApiKey || this._recoveryAccountIdentity(task); return { - recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName), + recoveryClaim: this._createRecoveryClaim(task, accountIdentity, fileName), doodApiKey }; } if (task.hoster === 'byse.sx' || task.hoster === 'doodstream.com' || task.hoster === 'voe.sx') { const accountIdentity = task.hoster === 'byse.sx' ? task.apiKey - : this._recoveryAccountIdentity(task); + : (task.hoster === 'doodstream.com' ? task.apiKey : this._recoveryAccountIdentity(task)); return { - recoveryClaim: this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName), + recoveryClaim: this._createRecoveryClaim(task, accountIdentity, fileName), doodApiKey: null }; } @@ -1337,10 +1343,54 @@ class UploadManager extends EventEmitter { } _recoveryAccountIdentity(task, fallbackIdentity = null) { - for (const value of [task.accountId, task.apiKey, fallbackIdentity]) { + for (const value of [task.accountId, task.apiKey, fallbackIdentity, task.username]) { if (value !== null && value !== undefined && String(value).trim()) return value; } - return String(task.username || '').normalize('NFKC').trim().toLowerCase(); + return ''; + } + + _createRecoveryClaim(task, accountIdentity, fileName) { + const accountClaim = this._recoveryClaims.forUpload(task.hoster, accountIdentity, fileName); + if (task.hoster !== 'doodstream.com' && task.hoster !== 'voe.sx') return accountClaim; + const hosterClaim = this._recoveryClaims.forUpload(task.hoster, 'mixed-auth-recovery-boundary', fileName); + const modeKey = `${task.hoster}\0${normalizeRecoveryTitle(fileName)}`; + let modeState = this._recoveryAuthModes.get(modeKey); + if (!modeState) { + modeState = new Set(); + this._recoveryAuthModes.set(modeKey, modeState); + } + const authMode = task.username ? 'login' : 'api'; + return { + has(code) { + return accountClaim.has(code); + }, + reserve(code) { + return accountClaim.reserve(code); + }, + markUncertain(error) { + hosterClaim.markUncertain(error); + return accountClaim.markUncertain(error); + }, + isUncertain() { + return hosterClaim.isUncertain() || accountClaim.isUncertain(); + }, + runExclusive(operation, signal) { + return hosterClaim.runExclusive( + async () => { + if (Array.from(modeState).some(mode => mode !== authMode)) { + const error = new Error('Gemischte Upload-Anmeldungen für denselben Remote-Titel wurden sicher blockiert'); + error.remoteCommitUncertain = true; + error.hosterTransient = true; + throw error; + } + const result = await accountClaim.runExclusive(operation, signal); + if (result !== null && result !== undefined) modeState.add(authMode); + return result; + }, + signal + ); + } + }; } async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe, context) { diff --git a/tests/doodstream-upload.test.js b/tests/doodstream-upload.test.js index 492cc0b..760cb80 100644 --- a/tests/doodstream-upload.test.js +++ b/tests/doodstream-upload.test.js @@ -1,5 +1,8 @@ const { test } = require('node:test'); const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); const DoodstreamUploader = require('../lib/doodstream-upload'); // The CDN hands back an XFileSharing form. `fn` is the filecode, `st` is the @@ -268,3 +271,61 @@ test('getUploadServer: failures expose safe structured diagnostics without respo } ); }); + +test('upload response read failure is marked as an uncertain remote commit', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mhu-dood-response-')); + const file = path.join(root, 'episode.mkv'); + fs.writeFileSync(file, Buffer.alloc(16, 1)); + const up = new DoodstreamUploader(); + up.sessId = 'SESSION'; + up._getUploadServer = async () => 'https://node.example/upload/01'; + up._requestUpload = async () => ({ + statusCode: 200, + headers: {}, + body: { text: async () => { throw new Error('socket closed'); } } + }); + try { + await assert.rejects( + () => up.upload(file), + (err) => { + assert.equal(err.remoteCommitUncertain, true); + assert.equal(err.diagnostic.phase, 'upload-response-read'); + return true; + } + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('redirect fetch failure after upload is marked as an uncertain remote commit', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mhu-dood-redirect-')); + const file = path.join(root, 'episode.mkv'); + fs.writeFileSync(file, Buffer.alloc(16, 1)); + const up = new DoodstreamUploader(); + up.sessId = 'SESSION'; + up._getUploadServer = async () => 'https://node.example/upload/01'; + up._requestUpload = async () => ({ + statusCode: 302, + headers: { location: 'https://doodstream.com/upload-result' }, + body: { text: async () => '' } + }); + up._fetch = async () => { + const error = new Error('redirect fetch failed'); + error.diagnostic = { phase: 'web-request' }; + error.transientNetwork = true; + throw error; + }; + try { + await assert.rejects( + () => up.upload(file), + (err) => { + assert.equal(err.remoteCommitUncertain, true); + assert.equal(err.diagnostic.phase, 'web-request'); + return true; + } + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/upload-manager-recovery-claims.test.js b/tests/upload-manager-recovery-claims.test.js index d0c6cd6..f9533d8 100644 --- a/tests/upload-manager-recovery-claims.test.js +++ b/tests/upload-manager-recovery-claims.test.js @@ -532,12 +532,12 @@ test('Doodstream key resolution is singleflight per account', async () => { }); }); -test('Doodstream web and API auth paths use one canonical account claim identity', async () => { +test('Doodstream web and API auth paths use one canonical remote account identity across profile IDs', async () => { const sharedCode = 'DOODALLAUTH01'; await withDoodstreamMethods({ login: async function () {}, deriveApiKey: async function () { - return null; + return 'DOOD_API_KEY'; }, upload: async function () { await new Promise(resolve => setImmediate(resolve)); @@ -567,7 +567,7 @@ test('Doodstream web and API auth paths use one canonical account claim identity jobId: 'dood-web-auth', file: firstPath, hoster: 'doodstream.com', - accountId: 'DOOD_SHARED_ACCOUNT', + accountId: 'DOOD_WEB_PROFILE', username: 'account@example.test', password: 'password' }, @@ -575,7 +575,7 @@ test('Doodstream web and API auth paths use one canonical account claim identity jobId: 'dood-api-auth', file: distinctPath, hoster: 'doodstream.com', - accountId: 'DOOD_SHARED_ACCOUNT', + accountId: 'DOOD_API_PROFILE', apiKey: 'DOOD_API_KEY' } ]); @@ -585,6 +585,190 @@ test('Doodstream web and API auth paths use one canonical account claim identity }); }); +test('Doodstream uncertainty blocks a same-title API profile with a different local ID', async () => { + let markWebStarted; + let releaseWeb; + let uploadCalls = 0; + const webStarted = new Promise(resolve => { + markWebStarted = resolve; + }); + const webGate = new Promise(resolve => { + releaseWeb = resolve; + }); + await withDoodstreamMethods({ + login: async function () {}, + deriveApiKey: async function () { + return null; + }, + upload: async function () { + uploadCalls++; + markWebStarted(); + await webGate; + const error = new Error('Doodstream upload result could not be read'); + error.remoteCommitUncertain = true; + throw error; + } + }, async () => { + loadManager(async () => { + uploadCalls++; + return { + file_code: 'UNSAFE_CROSS_AUTH', + download_url: 'https://doodstream.com/d/UNSAFE_CROSS_AUTH', + embed_url: 'https://doodstream.com/e/UNSAFE_CROSS_AUTH' + }; + }); + const manager = new UploadManager(settings('doodstream.com', 2)); + const batch = runBatch(manager, [ + { + jobId: 'dood-web-uncertain-profile', + file: firstPath, + hoster: 'doodstream.com', + accountId: 'DOOD_WEB_PROFILE', + username: 'account@example.test', + password: 'password' + } + ]); + await waitFor(webStarted, 500, 'Doodstream web upload did not start'); + const added = manager.addJobs([ + { + jobId: 'dood-api-after-uncertain-profile', + file: secondPath, + hoster: 'doodstream.com', + accountId: 'DOOD_API_PROFILE', + apiKey: 'DOOD_API_KEY' + } + ]); + assert.equal(added.added, 1); + releaseWeb(); + const summary = await batch; + + assert.equal(summary.succeeded, 0); + assert.equal(summary.failed, 2); + assert.equal(uploadCalls, 1); + }); +}); + +test('VOE mixed auth profiles share a fail-closed recovery boundary', async () => { + const sharedCode = 'VOECROSSAUTH1'; + await withUploaderMethods( + VoeUploader, + async function () { + await new Promise(resolve => setImmediate(resolve)); + return this._buildUrls(sharedCode); + }, + async () => { + loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => { + if (!options.recoveryClaim.reserve(sharedCode)) { + const error = new Error('Remote identity already claimed'); + error.hosterTransient = true; + throw error; + } + return { + file_code: sharedCode, + download_url: `https://voe.sx/${sharedCode}`, + embed_url: `https://voe.sx/e/${sharedCode}` + }; + }); + const manager = new UploadManager(settings('voe.sx', 2)); + const summary = await runBatch(manager, [ + { + jobId: 'voe-web-auth-profile', + file: firstPath, + hoster: 'voe.sx', + accountId: 'VOE_WEB_PROFILE', + username: 'account@example.test', + password: 'password' + }, + { + jobId: 'voe-api-auth-profile', + file: secondPath, + hoster: 'voe.sx', + accountId: 'VOE_API_PROFILE', + apiKey: 'VOE_API_KEY' + } + ]); + + assert.equal(summary.succeeded, 1); + assert.equal(summary.failed, 1); + } + ); +}); + +test('batch completion clears Doodstream key and baseline caches', async () => { + await withDoodstreamMethods({ + login: async function () {}, + deriveApiKey: async function () { + return 'DOOD_BATCH_KEY'; + } + }, async () => { + loadManager(async () => ({ + file_code: 'DOODBATCHDONE1', + download_url: 'https://doodstream.com/d/DOODBATCHDONE1', + embed_url: 'https://doodstream.com/e/DOODBATCHDONE1' + })); + const manager = new UploadManager(settings('doodstream.com', 1)); + const summary = await runBatch(manager, [{ + jobId: 'dood-cache-cleanup', + file: firstPath, + hoster: 'doodstream.com', + accountId: 'DOOD_CACHE_PROFILE', + username: 'account@example.test', + password: 'password' + }]); + + assert.equal(summary.succeeded, 1); + assert.equal(manager._doodApiKeyCache.size, 0); + assert.equal(manager._baselineCache.size, 0); + }); +}); + +test('upload interval is enforced at the admitted upload start', async () => { + const events = []; + let releaseFirst; + let markFirstStarted; + const firstGate = new Promise(resolve => { + releaseFirst = resolve; + }); + const firstStarted = new Promise(resolve => { + markFirstStarted = resolve; + }); + let uploadCount = 0; + loadManager(async (hoster, file) => { + uploadCount++; + events.push(`start:${path.basename(file)}`); + if (uploadCount === 1) { + markFirstStarted(); + await firstGate; + } + return { + file_code: `INTERVAL${events.length}`, + download_url: `https://byse.sx/d/INTERVAL${events.length}`, + embed_url: `https://byse.sx/e/INTERVAL${events.length}` + }; + }); + const hosterSettings = settings('byse.sx', 1); + hosterSettings['byse.sx'].timeIntervalSec = 1; + const manager = new UploadManager(hosterSettings); + manager._waitForInterval = async () => { + events.push('interval'); + }; + const batch = runBatch(manager, [ + { jobId: 'interval-first', file: firstPath, hoster: 'byse.sx', accountId: 'BYSE_ACCOUNT', apiKey: 'BYSE_KEY' }, + { jobId: 'interval-second', file: secondPath, hoster: 'byse.sx', accountId: 'BYSE_ACCOUNT', apiKey: 'BYSE_KEY' }, + { jobId: 'interval-third', file: distinctPath, hoster: 'byse.sx', accountId: 'BYSE_ACCOUNT', apiKey: 'BYSE_KEY' } + ]); + + await waitFor(firstStarted, 500, 'First admitted upload did not start'); + await new Promise(resolve => setImmediate(resolve)); + const intervalsBeforeRelease = events.filter(event => event === 'interval').length; + releaseFirst(); + const summary = await batch; + + assert.equal(intervalsBeforeRelease, 1, JSON.stringify(events)); + assert.equal(summary.succeeded, 3); + assert.deepEqual(events.filter(event => event === 'interval'), ['interval', 'interval', 'interval']); +}); + for (const scenario of [ { label: 'VOE',