From 87f976faf0fc3461629f9bb97c6690931b8d87d0 Mon Sep 17 00:00:00 2001 From: Administrator Date: Tue, 16 Jun 2026 00:44:03 +0200 Subject: [PATCH] fix(rotation): soften the byse suspect size-memo so one blip cannot poison a batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-batch size memo previously armed on a SINGLE confirmed suspect rejection and blocked every same-or-larger file on that account (fileSize >= memoSize). A single spurious byse "Not video file format" at e.g. 1.3GB therefore pre-failed the whole series of ~1.3GB files in that batch with "Bekanntes Größen-Limit auf diesem Account", even when byse could actually take that size. Now the memo: - arms only after the 2nd confirmed rejection on the same account (count >= 2), so a lone byse aussetzer no longer short-circuits anything; and - blocks only STRICTLY LARGER files (fileSize > memoSize), so a same-size file always still gets one real attempt. _suspectSizeMemo now stores { size: smallest-rejected, count } instead of a bare number. Still per-batch (cleared at batch start, never re-primed across batches). Updated the two memo tests to the new semantics (added a size-aware statSync mock so a strictly-larger file can be exercised). Full suite 284 green. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/upload-manager.js | 9 +++++---- tests/suspect-reject-alternates.test.js | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/upload-manager.js b/lib/upload-manager.js index ff85688..4963340 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -42,7 +42,7 @@ class UploadManager extends EventEmitter { this.globalThrottle = null; this._failedAccounts = new Map(); // hoster -> Set of failed accountIds this._accountOverrides = new Map(); // hoster -> fallback account object - this._suspectSizeMemo = new Map(); // 'hoster:accountId' -> smallest fileSize that got a suspect rejection + this._suspectSizeMemo = new Map(); // 'hoster:accountId' -> { size: smallest suspect-rejected fileSize, count: confirmed rejections }; blocks only after 2nd rejection this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none) this._baselineCache = new Map(); // hoster:apiKey -> Promise> (one fetch shared across all jobs in batch) @@ -111,12 +111,13 @@ class UploadManager extends EventEmitter { if (!accountId || !Number.isFinite(fileSize) || fileSize <= 0) return; const key = hoster + ':' + accountId; const prev = this._suspectSizeMemo.get(key); - if (prev === undefined || fileSize < prev) this._suspectSizeMemo.set(key, fileSize); + if (prev === undefined) this._suspectSizeMemo.set(key, { size: fileSize, count: 1 }); + else this._suspectSizeMemo.set(key, { size: Math.min(prev.size, fileSize), count: prev.count + 1 }); } _suspectMemoBlocks(hoster, accountId, fileSize) { - const memoSize = this._suspectSizeMemo.get(hoster + ':' + accountId); - return memoSize !== undefined && fileSize >= memoSize; + const memo = this._suspectSizeMemo.get(hoster + ':' + accountId); + return !!memo && memo.count >= 2 && fileSize > memo.size; } // File-specific rejections from the hoster: the same file will get rejected diff --git a/tests/suspect-reject-alternates.test.js b/tests/suspect-reject-alternates.test.js index f9b02b7..d962566 100644 --- a/tests/suspect-reject-alternates.test.js +++ b/tests/suspect-reject-alternates.test.js @@ -28,7 +28,10 @@ describe('suspect-reject alternate accounts', () => { const fs = require('fs'); const origStatSync = fs.statSync; fs.statSync = function (p) { - if (typeof p === 'string' && p.startsWith('/test/')) return { size: 3 * 1024 * 1024 * 1024 }; + if (typeof p === 'string' && p.startsWith('/test/')) { + const m = /-(\d+)gb/i.exec(p); + return { size: (m ? parseInt(m[1], 10) : 3) * 1024 * 1024 * 1024 }; + } return origStatSync.call(this, p); }; @@ -172,10 +175,10 @@ describe('suspect-reject alternate accounts', () => { assert.equal(summary.succeeded, 2); assert.ok(mgr.getFailedAccountKeys().includes('byse.sx:acc2'), 'dead alternate must be remembered'); const keys = mockUploadFile.mock.calls.map(c => c.arguments[2]); - assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key3'], 'second file must skip the memoized primary AND the dead alternate'); + assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key1', 'key3'], 'second same-size file still gets one real attempt on the primary (memo arms only on the 2nd rejection), then skips the dead alternate and lands on the good account'); }); - it('size memo short-circuits later oversized files straight to the known-good account', async () => { + it('size memo short-circuits a later LARGER file once the account has two confirmed rejections', async () => { const mgr = poolMgr([ { id: 'acc1', apiKey: 'key1' }, { id: 'acc2', apiKey: 'key2' }, @@ -191,14 +194,15 @@ describe('suspect-reject alternate accounts', () => { mgr.on('batch-done', (s) => { summary = s; }); await mgr.startBatch([ - { file: '/test/big1.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' }, - { file: '/test/big2.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' } + { file: '/test/a-1gb.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' }, + { file: '/test/b-1gb.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' }, + { file: '/test/c-2gb.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' } ]); - assert.equal(summary.succeeded, 2); + assert.equal(summary.succeeded, 3); const keys = mockUploadFile.mock.calls.map(c => c.arguments[2]); - assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key3'], 'no repeat multi-GB upload to size-limited accounts for the second file'); - assert.ok(rotEvents.includes('suspect-memo-skip'), 'second file must skip its primary via the size memo'); + assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key1', 'key3', 'key3'], 'files 1+2 each get a real attempt on the 1GB-rejecting primary (arming the memo at count 2); the larger 3rd file then short-circuits the primary straight to the good account'); + assert.ok(rotEvents.includes('suspect-memo-skip'), 'the larger third file must skip its primary via the armed size memo'); }); it('plain fileRejected without suspect flag keeps the old fast-fail behavior', async () => {