Fix duplicate VOE and Vidmoly recovery claims

Share account-scoped recovery claims with login uploader instances and serialize same-title recovery windows. Reserve direct and recovered file codes consistently, reject reused identities, and cover concurrent singleton recovery plus distinct-code and parallel-success cases.
2026-08-13 22:30:27 +02:00
parent acb43d982e
commit c209abdbbc
5 changed files with 323 additions and 32 deletions
@@ -1248,13 +1248,9 @@ class UploadManager extends EventEmitter {

  async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) {
    if (task.hoster === 'vidmoly.me' && task.username) {
-      const vidmoly = new VidmolyUploader();
-      await vidmoly.login(task.username, task.password);
-      return vidmoly.upload(task.file, progressCb, signal, throttle);
+      return this._executeRecoveryAwareLoginUpload(task, VidmolyUploader, progressCb, signal, throttle);
    } else if (task.hoster === 'voe.sx' && task.username) {
-      const voe = new VoeUploader();
-      await voe.login(task.username, task.password);
-      return voe.upload(task.file, progressCb, signal, throttle);
+      return this._executeRecoveryAwareLoginUpload(task, VoeUploader, progressCb, signal, throttle);
    } else if (task.hoster === 'doodstream.com' && task.username) {
      // Login-path reliability fix: the web-form upload returns the filecode in
      // an HTML form that comes back empty for large files (doodstream backend
@@ -1282,6 +1278,22 @@ class UploadManager extends EventEmitter {
    }
  }

+  async _executeRecoveryAwareLoginUpload(task, UploaderClass, progressCb, signal, throttle) {
+    const accountIdentity = task.accountId !== null && task.accountId !== undefined
+      ? task.accountId
+      : String(task.username || '').trim().toLowerCase();
+    const recoveryClaim = this._recoveryClaims.forUpload(
+      task.hoster,
+      accountIdentity,
+      path.basename(task.file)
+    );
+    return recoveryClaim.runExclusive(async () => {
+      const uploader = new UploaderClass(recoveryClaim);
+      await uploader.login(task.username, task.password);
+      return uploader.upload(task.file, progressCb, signal, throttle);
+    }, signal);
+  }
+
  async _executeRecoveryAwareApiUpload(hosterName, filePath, apiKey, progressCb, signal, throttle, fileProbe) {
    const recoveryClaim = this._recoveryClaims.forUpload(hosterName, apiKey, path.basename(filePath));
    return recoveryClaim.runExclusive(async () => {
@@ -14,8 +14,9 @@ const RESULT_POLL_DELAY_MS = 2000;
 * XFileSharing-based upload for Vidmoly (login + form upload)
 */
 class VidmolyUploader {
-  constructor() {
+  constructor(recoveryClaim = null) {
    this.cookies = new Map();
+    this.recoveryClaim = recoveryClaim;
  }

  _cookieHeader() {
@@ -352,6 +353,7 @@ class VidmolyUploader {
    try {
      return this._parseUploadResult(resultHtml);
    } catch (primaryErr) {
+      if (primaryErr && primaryErr.remoteIdentityClaimed === true) throw primaryErr;
      if (baselineCodes) {
        const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal);
        if (fallback) return fallback;
@@ -372,9 +374,21 @@ class VidmolyUploader {
      .replace(/[^a-z0-9]+/g, '');
  }

-  _buildUrlsFromCode(fileCode) {
+  _buildUrlsFromCode(fileCode, phase = 'upload-result') {
    const code = String(fileCode || '').trim();
    if (!code) return null;
+    if (this.recoveryClaim
+        && typeof this.recoveryClaim.reserve === 'function'
+        && !this.recoveryClaim.reserve(code)) {
+      const error = createTransportError('Vidmoly Upload-Ergebnis ist bereits einem anderen Upload zugeordnet', {
+        phase,
+        endpoint: BASE_URL,
+        retryable: true,
+        hosterTransient: true
+      });
+      error.remoteIdentityClaimed = true;
+      throw error;
+    }

    return {
      download_url: `${BASE_URL}/w/${code}`,
@@ -473,14 +487,18 @@ class VidmolyUploader {

      const withCode = files.filter((f) => f && typeof f.file_code === 'string' && f.file_code.trim());
      const newFiles = withCode.filter((f) => !baselineCodes.has(f.file_code.trim()));
-      const matches = newFiles.filter((file) => {
-        const title = this._normalizeTitle(file.full_title || file.title_txt || '');
-        return expectedTitle && title === expectedTitle;
-      });
+      const matches = newFiles
+        .filter((file) => {
+          const title = this._normalizeTitle(file.full_title || file.title_txt || '');
+          return expectedTitle && title === expectedTitle;
+        })
+        .filter((file) => !this.recoveryClaim
+          || typeof this.recoveryClaim.has !== 'function'
+          || !this.recoveryClaim.has(file.file_code.trim()));

      if (matches.length > 1) return null;
      if (matches.length === 1) {
-        return this._buildUrlsFromCode(matches[0].file_code);
+        return this._buildUrlsFromCode(matches[0].file_code, 'recovery-poll');
      }

      if (attempt < RESULT_POLL_ATTEMPTS - 1) {
@@ -577,12 +595,10 @@ class VidmolyUploader {
      if (codeInPage) file_code = codeInPage[1];
    }

-    // Build URLs from file_code
-    if (file_code && !download_url) {
-      download_url = `${BASE_URL}/w/${file_code}`;
-    }
-    if (file_code && !embed_url) {
-      embed_url = `${BASE_URL}/embed-${file_code}.html`;
+    if (file_code) {
+      const urls = this._buildUrlsFromCode(file_code);
+      if (!download_url) download_url = urls.download_url;
+      if (!embed_url) embed_url = urls.embed_url;
    }

    if (!download_url && !file_code) {
@@ -15,8 +15,9 @@ const RESULT_POLL_DELAY_MS = 2000;
 * Fallback when API-based upload fails or is unavailable.
 */
 class VoeUploader {
-  constructor() {
+  constructor(recoveryClaim = null) {
    this.cookies = new Map();
+    this.recoveryClaim = recoveryClaim;
  }

  _cookieHeader() {
@@ -449,15 +450,22 @@ class VoeUploader {

      const withCode = files.filter(f => f && (f.file_code || f.slug));
      const newFiles = withCode.filter(f => !baselineCodes.has(String(f.file_code || f.slug || '').trim()));
-      const matches = newFiles.filter(file => {
-        const title = this._normalizeTitle(file.title || file.name || '');
-        return expectedTitle && title === expectedTitle;
-      });
+      const matches = newFiles
+        .filter(file => {
+          const title = this._normalizeTitle(file.title || file.name || '');
+          return expectedTitle && title === expectedTitle;
+        })
+        .filter(file => {
+          const code = String(file.file_code || file.slug || '').trim();
+          return !this.recoveryClaim
+            || typeof this.recoveryClaim.has !== 'function'
+            || !this.recoveryClaim.has(code);
+        });

      if (matches.length > 1) return null;
      if (matches.length === 1) {
        const code = matches[0].file_code || matches[0].slug;
-        return this._buildUrls(code);
+        return this._buildUrls(code, 'recovery-poll');
      }

      if (attempt < RESULT_POLL_ATTEMPTS - 1) {
@@ -477,9 +485,21 @@ class VoeUploader {
      .replace(/[^a-z0-9]+/g, '');
  }

-  _buildUrls(fileCode) {
+  _buildUrls(fileCode, phase = 'upload-result') {
    const code = String(fileCode || '').trim();
    if (!code) return null;
+    if (this.recoveryClaim
+        && typeof this.recoveryClaim.reserve === 'function'
+        && !this.recoveryClaim.reserve(code)) {
+      const error = createTransportError('VOE Upload-Ergebnis ist bereits einem anderen Upload zugeordnet', {
+        phase,
+        endpoint: BASE_URL,
+        retryable: true,
+        hosterTransient: true
+      });
+      error.remoteIdentityClaimed = true;
+      throw error;
+    }
    return {
      download_url: `${BASE_URL}/${code}`,
      embed_url: `${BASE_URL}/e/${code}`,
@@ -3,6 +3,7 @@ const assert = require('node:assert/strict');

 const VoeUploader = require('../lib/voe-upload');
 const VidmolyUploader = require('../lib/vidmoly-upload');
+const { createRecoveryClaimRegistry } = require('../lib/hosters');

 function response(body, status = 200, contentType = 'application/json') {
  return {
@@ -116,3 +117,73 @@ test('Vidmoly preserves a failed recovery baseline as a safe structured error',
    }
  );
 });
+
+test('VOE concurrent same-name recovery claims one remote entry only once', async () => {
+  const registry = createRecoveryClaimRegistry();
+  const first = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'Shared Episode.mkv'));
+  const second = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'shared-episode.mp4'));
+  const remoteFiles = [{ file_code: 'VOE_SHARED', title: 'shared episode' }];
+  first._fetchFileList = async () => remoteFiles;
+  second._fetchFileList = async () => remoteFiles;
+  first._sleep = async () => {};
+  second._sleep = async () => {};
+
+  const results = await Promise.all([
+    first._resolveUploadedFile('C:\\source-a\\Shared Episode.mkv', new Set(), null),
+    second._resolveUploadedFile('D:\\source-b\\shared-episode.mp4', new Set(), null)
+  ]);
+
+  assert.deepEqual(results.filter(Boolean).map(result => result.file_code), ['VOE_SHARED']);
+});
+
+test('VOE concurrent same-name recovery accepts distinct remote codes', async () => {
+  const registry = createRecoveryClaimRegistry();
+  const first = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'Shared Episode.mkv'));
+  const second = new VoeUploader(registry.forUpload('voe.sx', 'ACCOUNT', 'shared-episode.mp4'));
+  first._fetchFileList = async () => [{ file_code: 'VOE_FIRST', title: 'shared episode' }];
+  second._fetchFileList = async () => [{ file_code: 'VOE_SECOND', title: 'shared episode' }];
+  first._sleep = async () => {};
+  second._sleep = async () => {};
+
+  const results = await Promise.all([
+    first._resolveUploadedFile('C:\\source-a\\Shared Episode.mkv', new Set(), null),
+    second._resolveUploadedFile('D:\\source-b\\shared-episode.mp4', new Set(), null)
+  ]);
+
+  assert.deepEqual(results.map(result => result.file_code), ['VOE_FIRST', 'VOE_SECOND']);
+});
+
+test('Vidmoly concurrent same-name recovery claims one remote entry only once', async () => {
+  const registry = createRecoveryClaimRegistry();
+  const first = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv'));
+  const second = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4'));
+  const remoteFiles = [{ file_code: 'VIDSHARED001', full_title: 'shared episode' }];
+  first._fetchVmList = async () => remoteFiles;
+  second._fetchVmList = async () => remoteFiles;
+  first._sleep = async () => {};
+  second._sleep = async () => {};
+
+  const results = await Promise.all([
+    first._resolveUploadedFileFromVmApi('C:\\source-a\\Shared Episode.mkv', new Set(), null),
+    second._resolveUploadedFileFromVmApi('D:\\source-b\\shared-episode.mp4', new Set(), null)
+  ]);
+
+  assert.deepEqual(results.filter(Boolean).map(result => result.file_code), ['VIDSHARED001']);
+});
+
+test('Vidmoly concurrent same-name recovery accepts distinct remote codes', async () => {
+  const registry = createRecoveryClaimRegistry();
+  const first = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'Shared Episode.mkv'));
+  const second = new VidmolyUploader(registry.forUpload('vidmoly.me', 'ACCOUNT', 'shared-episode.mp4'));
+  first._fetchVmList = async () => [{ file_code: 'VIDFIRST0001', full_title: 'shared episode' }];
+  second._fetchVmList = async () => [{ file_code: 'VIDSECOND001', full_title: 'shared episode' }];
+  first._sleep = async () => {};
+  second._sleep = async () => {};
+
+  const results = await Promise.all([
+    first._resolveUploadedFileFromVmApi('C:\\source-a\\Shared Episode.mkv', new Set(), null),
+    second._resolveUploadedFileFromVmApi('D:\\source-b\\shared-episode.mp4', new Set(), null)
+  ]);
+
+  assert.deepEqual(results.map(result => result.file_code), ['VIDFIRST0001', 'VIDSECOND001']);
+});
@@ -5,11 +5,14 @@ const os = require('node:os');
 const path = require('node:path');

 const hosters = require('../lib/hosters');
+const VoeUploader = require('../lib/voe-upload');
+const VidmolyUploader = require('../lib/vidmoly-upload');
 const originalUploadFile = hosters.uploadFile;
 const originalPrefetchBaseline = hosters.prefetchBaseline;
 let tempRoot;
 let firstPath;
 let secondPath;
+let distinctPath;
 let UploadManager;

 before(() => {
@@ -20,8 +23,10 @@ before(() => {
  fs.mkdirSync(secondDir);
  firstPath = path.join(firstDir, 'Shared Episode.mkv');
  secondPath = path.join(secondDir, 'shared-episode.mp4');
+  distinctPath = path.join(secondDir, 'different-title.mkv');
  fs.writeFileSync(firstPath, Buffer.alloc(1024, 1));
  fs.writeFileSync(secondPath, Buffer.alloc(1024, 2));
+  fs.writeFileSync(distinctPath, Buffer.alloc(1024, 3));
 });

 after(() => {
@@ -31,16 +36,16 @@ after(() => {
  fs.rmSync(tempRoot, { recursive: true, force: true });
 });

-function loadManager(uploadFile) {
+function loadManager(uploadFile = originalUploadFile) {
  hosters.uploadFile = uploadFile;
  hosters.prefetchBaseline = async () => new Set();
  delete require.cache[require.resolve('../lib/upload-manager')];
  UploadManager = require('../lib/upload-manager');
 }

-function settings(parallelCount) {
+function settings(hoster, parallelCount) {
  return {
-    'byse.sx': {
+    [hoster]: {
      retries: 0,
      parallelCount,
      maxSpeedKbs: 0,
@@ -60,6 +65,29 @@ async function runBatch(manager, tasks) {
  return summary;
 }

+async function withUploaderMethods(Uploader, upload, operation) {
+  const originalLogin = Uploader.prototype.login;
+  const originalUpload = Uploader.prototype.upload;
+  Uploader.prototype.login = async function () {};
+  Uploader.prototype.upload = upload;
+  try {
+    return await operation();
+  } finally {
+    Uploader.prototype.login = originalLogin;
+    Uploader.prototype.upload = originalUpload;
+  }
+}
+
+function waitFor(promise, timeoutMs, message) {
+  let timer;
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+    })
+  ]).finally(() => clearTimeout(timer));
+}
+
 test('a batch shares recovery claims across normalized same-name jobs', async () => {
  let unsafeCalls = 0;
  loadManager(async (hoster, file, apiKey, onProgress, signal, throttle, options) => {
@@ -81,7 +109,7 @@ test('a batch shares recovery claims across normalized same-name jobs', async ()
    error.hosterTransient = true;
    throw error;
  });
-  const manager = new UploadManager(settings(2));
+  const manager = new UploadManager(settings('byse.sx', 2));

  const summary = await runBatch(manager, [
    { jobId: 'same-name-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
@@ -107,7 +135,7 @@ test('recovery claims stay isolated between accounts', async () => {
      download_url: 'https://byse.sx/d/SHARED_REMOTE_CODE'
    };
  });
-  const manager = new UploadManager(settings(2));
+  const manager = new UploadManager(settings('byse.sx', 2));

  const summary = await runBatch(manager, [
    { jobId: 'account-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_A' },
@@ -133,7 +161,7 @@ test('normalized same-name recovery sections never overlap', async () => {
      download_url: `https://byse.sx/d/SERIAL_${sequence}`
    };
  });
-  const manager = new UploadManager(settings(2));
+  const manager = new UploadManager(settings('byse.sx', 2));

  const summary = await runBatch(manager, [
    { jobId: 'serialized-a', file: firstPath, hoster: 'byse.sx', apiKey: 'ACCOUNT_KEY' },
@@ -143,3 +171,147 @@ test('normalized same-name recovery sections never overlap', async () => {
  assert.equal(summary.succeeded, 2);
  assert.equal(maximumActive, 1);
 });
+
+for (const scenario of [
+  {
+    label: 'VOE',
+    hoster: 'voe.sx',
+    Uploader: VoeUploader,
+    sharedCode: 'SHAREDVOE01',
+    distinctCodes: ['VOEDISTINCT1', 'VOEDISTINCT2'],
+    buildResult(uploader, code) {
+      return uploader._buildUrls(code);
+    }
+  },
+  {
+    label: 'Vidmoly',
+    hoster: 'vidmoly.me',
+    Uploader: VidmolyUploader,
+    sharedCode: 'SHAREDVID001',
+    distinctCodes: ['VIDDISTINCT1', 'VIDDISTINCT2'],
+    buildResult(uploader, code) {
+      return uploader._buildUrlsFromCode(code);
+    }
+  }
+]) {
+  test(`${scenario.label} uploader instances reject a duplicate direct remote code for same-name sources`, async () => {
+    await withUploaderMethods(
+      scenario.Uploader,
+      async function () {
+        await new Promise(resolve => setImmediate(resolve));
+        return scenario.buildResult(this, scenario.sharedCode);
+      },
+      async () => {
+        loadManager();
+        const manager = new UploadManager(settings(scenario.hoster, 2));
+        const summary = await runBatch(manager, [
+          {
+            jobId: `${scenario.label}-same-name-a`,
+            file: firstPath,
+            hoster: scenario.hoster,
+            accountId: 'LOGIN_ACCOUNT',
+            username: 'account@example.test',
+            password: 'password'
+          },
+          {
+            jobId: `${scenario.label}-same-name-b`,
+            file: secondPath,
+            hoster: scenario.hoster,
+            accountId: 'LOGIN_ACCOUNT',
+            username: 'account@example.test',
+            password: 'password'
+          }
+        ]);
+
+        assert.equal(summary.succeeded, 1);
+        assert.equal(summary.failed, 1);
+      }
+    );
+  });
+
+  test(`${scenario.label} uploader instances isolate direct remote code claims between accounts`, async () => {
+    await withUploaderMethods(
+      scenario.Uploader,
+      async function () {
+        return scenario.buildResult(this, scenario.sharedCode);
+      },
+      async () => {
+        loadManager();
+        const manager = new UploadManager(settings(scenario.hoster, 2));
+        const summary = await runBatch(manager, [
+          {
+            jobId: `${scenario.label}-account-a`,
+            file: firstPath,
+            hoster: scenario.hoster,
+            accountId: 'LOGIN_ACCOUNT_A',
+            username: 'account-a@example.test',
+            password: 'password'
+          },
+          {
+            jobId: `${scenario.label}-account-b`,
+            file: secondPath,
+            hoster: scenario.hoster,
+            accountId: 'LOGIN_ACCOUNT_B',
+            username: 'account-b@example.test',
+            password: 'password'
+          }
+        ]);
+
+        assert.equal(summary.succeeded, 2);
+        assert.equal(summary.failed, 0);
+      }
+    );
+  });
+
+  test(`${scenario.label} uploader instances preserve parallel success for distinct remote identities`, async () => {
+    let active = 0;
+    let maximumActive = 0;
+    let started = 0;
+    let releaseBoth;
+    const bothStarted = new Promise(resolve => {
+      releaseBoth = resolve;
+    });
+    await withUploaderMethods(
+      scenario.Uploader,
+      async function (filePath) {
+        active++;
+        started++;
+        maximumActive = Math.max(maximumActive, active);
+        if (started === 2) releaseBoth();
+        try {
+          await waitFor(bothStarted, 500, 'Distinct uploads did not overlap');
+          const code = filePath === firstPath ? scenario.distinctCodes[0] : scenario.distinctCodes[1];
+          return scenario.buildResult(this, code);
+        } finally {
+          active--;
+        }
+      },
+      async () => {
+        loadManager();
+        const manager = new UploadManager(settings(scenario.hoster, 2));
+        const summary = await runBatch(manager, [
+          {
+            jobId: `${scenario.label}-distinct-a`,
+            file: firstPath,
+            hoster: scenario.hoster,
+            accountId: 'LOGIN_ACCOUNT',
+            username: 'account@example.test',
+            password: 'password'
+          },
+          {
+            jobId: `${scenario.label}-distinct-b`,
+            file: distinctPath,
+            hoster: scenario.hoster,
+            accountId: 'LOGIN_ACCOUNT',
+            username: 'account@example.test',
+            password: 'password'
+          }
+        ]);
+
+        assert.equal(summary.succeeded, 2);
+        assert.equal(summary.failed, 0);
+        assert.equal(maximumActive, 2);
+      }
+    );
+  });
+}