Compare commits
2 Commits
5d04ddced3
...
b032dd6b3f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b032dd6b3f | ||
|
|
87f976faf0 |
@ -42,7 +42,7 @@ class UploadManager extends EventEmitter {
|
|||||||
this.globalThrottle = null;
|
this.globalThrottle = null;
|
||||||
this._failedAccounts = new Map(); // hoster -> Set of failed accountIds
|
this._failedAccounts = new Map(); // hoster -> Set of failed accountIds
|
||||||
this._accountOverrides = new Map(); // hoster -> fallback account object
|
this._accountOverrides = new Map(); // hoster -> fallback account object
|
||||||
this._suspectSizeMemo = new Map(); // 'hoster:accountId' -> smallest fileSize that got a suspect rejection
|
this._suspectSizeMemo = new Map(); // 'hoster:accountId' -> { size: smallest suspect-rejected fileSize, count: confirmed rejections }; blocks only after 2nd rejection
|
||||||
this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file
|
this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file
|
||||||
this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
|
this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
|
||||||
this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
|
this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
|
||||||
@ -111,12 +111,13 @@ class UploadManager extends EventEmitter {
|
|||||||
if (!accountId || !Number.isFinite(fileSize) || fileSize <= 0) return;
|
if (!accountId || !Number.isFinite(fileSize) || fileSize <= 0) return;
|
||||||
const key = hoster + ':' + accountId;
|
const key = hoster + ':' + accountId;
|
||||||
const prev = this._suspectSizeMemo.get(key);
|
const prev = this._suspectSizeMemo.get(key);
|
||||||
if (prev === undefined || fileSize < prev) this._suspectSizeMemo.set(key, fileSize);
|
if (prev === undefined) this._suspectSizeMemo.set(key, { size: fileSize, count: 1 });
|
||||||
|
else this._suspectSizeMemo.set(key, { size: Math.min(prev.size, fileSize), count: prev.count + 1 });
|
||||||
}
|
}
|
||||||
|
|
||||||
_suspectMemoBlocks(hoster, accountId, fileSize) {
|
_suspectMemoBlocks(hoster, accountId, fileSize) {
|
||||||
const memoSize = this._suspectSizeMemo.get(hoster + ':' + accountId);
|
const memo = this._suspectSizeMemo.get(hoster + ':' + accountId);
|
||||||
return memoSize !== undefined && fileSize >= memoSize;
|
return !!memo && memo.count >= 2 && fileSize > memo.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
// File-specific rejections from the hoster: the same file will get rejected
|
// File-specific rejections from the hoster: the same file will get rejected
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "multi-hoster-uploader",
|
"name": "multi-hoster-uploader",
|
||||||
"version": "3.3.72",
|
"version": "3.3.73",
|
||||||
"description": "Upload files to doodstream, voe, vidmoly, byse simultaneously",
|
"description": "Upload files to doodstream, voe, vidmoly, byse simultaneously",
|
||||||
"main": "main.js",
|
"main": "main.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@ -28,7 +28,10 @@ describe('suspect-reject alternate accounts', () => {
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const origStatSync = fs.statSync;
|
const origStatSync = fs.statSync;
|
||||||
fs.statSync = function (p) {
|
fs.statSync = function (p) {
|
||||||
if (typeof p === 'string' && p.startsWith('/test/')) return { size: 3 * 1024 * 1024 * 1024 };
|
if (typeof p === 'string' && p.startsWith('/test/')) {
|
||||||
|
const m = /-(\d+)gb/i.exec(p);
|
||||||
|
return { size: (m ? parseInt(m[1], 10) : 3) * 1024 * 1024 * 1024 };
|
||||||
|
}
|
||||||
return origStatSync.call(this, p);
|
return origStatSync.call(this, p);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -172,10 +175,10 @@ describe('suspect-reject alternate accounts', () => {
|
|||||||
assert.equal(summary.succeeded, 2);
|
assert.equal(summary.succeeded, 2);
|
||||||
assert.ok(mgr.getFailedAccountKeys().includes('byse.sx:acc2'), 'dead alternate must be remembered');
|
assert.ok(mgr.getFailedAccountKeys().includes('byse.sx:acc2'), 'dead alternate must be remembered');
|
||||||
const keys = mockUploadFile.mock.calls.map(c => c.arguments[2]);
|
const keys = mockUploadFile.mock.calls.map(c => c.arguments[2]);
|
||||||
assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key3'], 'second file must skip the memoized primary AND the dead alternate');
|
assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key1', 'key3'], 'second same-size file still gets one real attempt on the primary (memo arms only on the 2nd rejection), then skips the dead alternate and lands on the good account');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('size memo short-circuits later oversized files straight to the known-good account', async () => {
|
it('size memo short-circuits a later LARGER file once the account has two confirmed rejections', async () => {
|
||||||
const mgr = poolMgr([
|
const mgr = poolMgr([
|
||||||
{ id: 'acc1', apiKey: 'key1' },
|
{ id: 'acc1', apiKey: 'key1' },
|
||||||
{ id: 'acc2', apiKey: 'key2' },
|
{ id: 'acc2', apiKey: 'key2' },
|
||||||
@ -191,14 +194,15 @@ describe('suspect-reject alternate accounts', () => {
|
|||||||
mgr.on('batch-done', (s) => { summary = s; });
|
mgr.on('batch-done', (s) => { summary = s; });
|
||||||
|
|
||||||
await mgr.startBatch([
|
await mgr.startBatch([
|
||||||
{ file: '/test/big1.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' },
|
{ file: '/test/a-1gb.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' },
|
||||||
{ file: '/test/big2.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' }
|
{ file: '/test/b-1gb.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' },
|
||||||
|
{ file: '/test/c-2gb.mkv', hoster: 'byse.sx', apiKey: 'key1', accountId: 'acc1' }
|
||||||
]);
|
]);
|
||||||
|
|
||||||
assert.equal(summary.succeeded, 2);
|
assert.equal(summary.succeeded, 3);
|
||||||
const keys = mockUploadFile.mock.calls.map(c => c.arguments[2]);
|
const keys = mockUploadFile.mock.calls.map(c => c.arguments[2]);
|
||||||
assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key3'], 'no repeat multi-GB upload to size-limited accounts for the second file');
|
assert.deepEqual(keys, ['key1', 'key2', 'key3', 'key1', 'key3', 'key3'], 'files 1+2 each get a real attempt on the 1GB-rejecting primary (arming the memo at count 2); the larger 3rd file then short-circuits the primary straight to the good account');
|
||||||
assert.ok(rotEvents.includes('suspect-memo-skip'), 'second file must skip its primary via the size memo');
|
assert.ok(rotEvents.includes('suspect-memo-skip'), 'the larger third file must skip its primary via the armed size memo');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('plain fileRejected without suspect flag keeps the old fast-fail behavior', async () => {
|
it('plain fileRejected without suspect flag keeps the old fast-fail behavior', async () => {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user