Multi-Hoster-Upload/tests/upload-manager.test.js
Administrator 17e9a419b2 fix(rotation): treat byse "disk space" as account-level, not file-rejected
Byse rejects uploads with status like "not enough disk space on your
account" when the account's storage is exhausted. The parser was
flagging every non-OK status as err.fileRejected=true, and the upload-
manager classifier additionally matched the generic "lehnte Datei ab"
prefix as file-rejected. Result: rotation was skipped on a full account
and every subsequent file failed on the same dead account.

- hosters.js: byse parser now distinguishes account-level phrases
  (disk space / storage / quota / insufficient / account full) and sets
  err.accountError=true for those. File-specific failures (Duplicate,
  wrong format, size) keep err.fileRejected=true.
- upload-manager.js: _isFileRejectedError no longer matches the generic
  "lehnte Datei ab" prefix and short-circuits when err.accountError is
  true. _shouldSkipRetryOnAccountError honors the flag and has added
  regex patterns as a safety net.
- Tests: 5 new unit tests covering disk-space/account-level/duplicate
  and the accountError-wins-over-fileRejected precedence.
2026-04-21 16:42:56 +02:00

558 lines
21 KiB
JavaScript

const { describe, it, mock, beforeEach } = require('node:test');
const assert = require('node:assert/strict');
const path = require('path');
const { EventEmitter } = require('events');
// We need to mock fs.statSync and the hoster upload functions before requiring upload-manager
// Use node:test mock.module (available in Node 22+)
describe('UploadManager', () => {
let UploadManager;
let mockUploadFile;
let fakeFileSize;
beforeEach(() => {
fakeFileSize = 1024 * 1024; // 1 MB default
// Clear module cache for fresh mocks each test
delete require.cache[require.resolve('../lib/upload-manager')];
// Mock the hosters module
mockUploadFile = mock.fn(async (hoster, filePath, apiKey, onProgress, signal, throttle) => {
// Simulate upload progress
if (onProgress) {
onProgress(fakeFileSize / 2, fakeFileSize);
onProgress(fakeFileSize, fakeFileSize);
}
return { download_url: `https://${hoster}/test123`, embed_url: null, file_code: 'test123' };
});
// Override require for hosters
const origRequire = module.constructor.prototype.require;
const hosters = require('../lib/hosters');
hosters.uploadFile = mockUploadFile;
// Mock fs.statSync for test file paths
const fs = require('fs');
const origStatSync = fs.statSync;
fs.statSync = function(p) {
if (typeof p === 'string' && p.startsWith('/test/')) {
return { size: fakeFileSize };
}
return origStatSync.call(this, p);
};
UploadManager = require('../lib/upload-manager');
});
it('emits progress events for each task', async () => {
const mgr = new UploadManager({});
const events = [];
mgr.on('progress', (data) => events.push(data));
await mgr.startBatch([
{ file: '/test/video1.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
const statuses = events.map(e => e.status);
assert.ok(statuses.includes('queued'), 'should have queued status');
assert.ok(statuses.includes('done'), 'should have done status');
});
it('emits batch-done with correct summary', async () => {
const mgr = new UploadManager({});
let summary = null;
mgr.on('batch-done', (s) => { summary = s; });
await mgr.startBatch([
{ file: '/test/video1.mp4', hoster: 'doodstream.com', apiKey: 'key1' },
{ file: '/test/video2.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.ok(summary);
assert.equal(summary.total, 2);
assert.equal(summary.succeeded, 2);
assert.equal(summary.failed, 0);
assert.equal(summary.files.length, 2);
});
it('retries on failure then succeeds', async () => {
let callCount = 0;
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress) => {
callCount++;
if (callCount <= 2) throw new Error('network error');
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'https://test/ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({ 'doodstream.com': { retries: 3, parallelCount: 1, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 } });
const statuses = [];
mgr.on('progress', (d) => statuses.push(d.status));
await mgr.startBatch([
{ file: '/test/video.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.ok(statuses.includes('retrying'), 'should show retrying status');
assert.ok(statuses.includes('done'), 'should eventually succeed');
assert.equal(callCount, 3);
});
it('exhausted retries result in error', async () => {
mockUploadFile.mock.mockImplementation(async () => {
throw new Error('permanent failure');
});
const mgr = new UploadManager({ 'doodstream.com': { retries: 1, parallelCount: 1, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 } });
let summary = null;
mgr.on('batch-done', (s) => { summary = s; });
await mgr.startBatch([
{ file: '/test/video.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.ok(summary);
assert.equal(summary.failed, 1);
assert.equal(summary.succeeded, 0);
});
it('cancel aborts running uploads', async () => {
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress, signal) => {
// Simulate a slow upload
await new Promise((resolve, reject) => {
const timer = setTimeout(() => resolve({ download_url: 'x', embed_url: null, file_code: 'x' }), 10000);
if (signal) signal.addEventListener('abort', () => { clearTimeout(timer); reject(new Error('Aborted')); });
});
});
const mgr = new UploadManager({});
let batchDone = false;
mgr.on('batch-done', () => { batchDone = true; });
const batchPromise = mgr.startBatch([
{ file: '/test/video.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
// Wait a bit then cancel
await new Promise(r => setTimeout(r, 100));
mgr.cancel();
await batchPromise;
assert.equal(mgr.running, false);
assert.ok(batchDone, 'batch-done should be emitted even after cancel');
});
it('maxSizeMb filter skips oversized files', async () => {
fakeFileSize = 5 * 1024 * 1024; // 5 MB
const mgr = new UploadManager({ 'doodstream.com': { retries: 0, parallelCount: 1, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 1 } });
const statuses = [];
mgr.on('progress', (d) => statuses.push(d.status));
await mgr.startBatch([
{ file: '/test/big.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.ok(statuses.includes('skipped'), 'oversized file should be skipped');
assert.ok(!statuses.includes('uploading'), 'should not attempt upload');
});
it('per-hoster semaphore limits concurrency', async () => {
let concurrent = 0;
let maxConcurrent = 0;
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress, signal) => {
concurrent++;
maxConcurrent = Math.max(maxConcurrent, concurrent);
await new Promise(r => setTimeout(r, 50));
concurrent--;
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'https://test/ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({ 'doodstream.com': { retries: 0, parallelCount: 1, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 } });
await mgr.startBatch([
{ file: '/test/a.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ file: '/test/b.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ file: '/test/c.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
assert.equal(maxConcurrent, 1, 'should only run 1 upload at a time');
});
it('global parallel limit caps concurrency across hosters', async () => {
let concurrent = 0;
let maxConcurrent = 0;
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress) => {
concurrent++;
maxConcurrent = Math.max(maxConcurrent, concurrent);
await new Promise((resolve) => setTimeout(resolve, 40));
concurrent--;
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: `https://${hoster}/ok`, embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({
'doodstream.com': { retries: 0, parallelCount: 5, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 },
'voe.sx': { retries: 0, parallelCount: 5, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 },
'vidmoly.me': { retries: 0, parallelCount: 5, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 }
}, {
parallelUploadCount: 2,
scaleParallelUploads: true
});
await mgr.startBatch([
{ jobId: 'job-1', file: '/test/a.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ jobId: 'job-2', file: '/test/b.mp4', hoster: 'voe.sx', apiKey: 'k' },
{ jobId: 'job-3', file: '/test/c.mp4', hoster: 'vidmoly.me', apiKey: 'k' }
]);
assert.equal(maxConcurrent, 2, 'should only run 2 uploads globally at once');
});
it('cancelJobs aborts a selected running upload', async () => {
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress, signal) => {
await new Promise((resolve, reject) => {
const timer = setTimeout(() => resolve(), 250);
if (signal) {
signal.addEventListener('abort', () => {
clearTimeout(timer);
reject(new Error('Aborted'));
}, { once: true });
}
});
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'https://test/ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({});
const statuses = [];
mgr.on('progress', (data) => statuses.push({ jobId: data.jobId, status: data.status }));
const batchPromise = mgr.startBatch([
{ jobId: 'selected-job', file: '/test/video.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
await new Promise((resolve) => setTimeout(resolve, 50));
mgr.cancelJobs(['selected-job']);
await batchPromise;
assert.ok(statuses.some((entry) => entry.jobId === 'selected-job' && entry.status === 'aborted'));
});
it('addJobs returns duplicate info and still runs newly queued jobs', async () => {
let releaseFirst = null;
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress, signal) => {
if (filePath.endsWith('/first.mp4')) {
await new Promise((resolve, reject) => {
releaseFirst = resolve;
if (signal) {
signal.addEventListener('abort', () => reject(new Error('Aborted')), { once: true });
}
});
} else {
await new Promise((resolve) => setTimeout(resolve, 20));
}
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'https://test/ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({
'doodstream.com': { retries: 0, parallelCount: 1, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 }
});
const statuses = [];
mgr.on('progress', (data) => statuses.push({ jobId: data.jobId, status: data.status }));
const batchPromise = mgr.startBatch([
{ jobId: 'job-first', file: '/test/first.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
for (let i = 0; i < 50 && !releaseFirst; i++) {
await new Promise((resolve) => setTimeout(resolve, 10));
}
assert.equal(typeof releaseFirst, 'function', 'first job should be running before addJobs');
const addResult = mgr.addJobs([
{ jobId: 'job-first', file: '/test/first.mp4', hoster: 'doodstream.com', apiKey: 'key1' },
{ jobId: 'job-second', file: '/test/second.mp4', hoster: 'doodstream.com', apiKey: 'key1' },
{ jobId: 'job-third', file: '/test/third.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.equal(addResult.added, 2);
assert.deepEqual(addResult.alreadyInBatchJobIds, ['job-first']);
releaseFirst();
await batchPromise;
assert.ok(statuses.some((entry) => entry.jobId === 'job-second' && entry.status === 'done'));
assert.ok(statuses.some((entry) => entry.jobId === 'job-third' && entry.status === 'done'));
});
it('_combineSignals propagates abort from either source', () => {
const mgr = new UploadManager({});
const ac1 = new AbortController();
const ac2 = new AbortController();
const { signal, cleanup } = mgr._combineSignals(ac1.signal, ac2.signal);
assert.equal(signal.aborted, false);
ac2.abort();
assert.equal(signal.aborted, true);
cleanup();
});
it('_combineSignals returns aborted signal if input already aborted', () => {
const mgr = new UploadManager({});
const ac1 = new AbortController();
ac1.abort();
const ac2 = new AbortController();
const { signal, cleanup } = mgr._combineSignals(ac1.signal, ac2.signal);
assert.equal(signal.aborted, true);
cleanup();
});
it('_sleep resolves after delay', async () => {
const mgr = new UploadManager({});
const start = Date.now();
await mgr._sleep(100);
assert.ok(Date.now() - start >= 90);
});
it('_sleep rejects on abort', async () => {
const mgr = new UploadManager({});
const ac = new AbortController();
setTimeout(() => ac.abort(), 10);
await assert.rejects(mgr._sleep(5000, ac.signal), /Aborted/);
});
it('file not found produces descriptive error', async () => {
// Override fs.statSync to throw ENOENT for a specific path
const fs = require('fs');
const origStat = fs.statSync;
fs.statSync = function(p) {
if (p === '/test/deleted.mp4') throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
return origStat.call(this, p);
};
const mgr = new UploadManager({});
const errors = [];
mgr.on('progress', (d) => { if (d.error) errors.push(d.error); });
await mgr.startBatch([
{ file: '/test/deleted.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
fs.statSync = origStat;
assert.ok(errors.some(e => e.includes('nicht gefunden')), `expected "nicht gefunden" error, got: ${errors.join(', ')}`);
});
it('zero-byte file produces descriptive error', async () => {
fakeFileSize = 0;
const mgr = new UploadManager({});
const errors = [];
mgr.on('progress', (d) => { if (d.error) errors.push(d.error); });
await mgr.startBatch([
{ file: '/test/empty.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.ok(errors.some(e => e.includes('0 Bytes')), `expected "0 Bytes" error, got: ${errors.join(', ')}`);
});
it('empty batch completes immediately with zero counts', async () => {
const mgr = new UploadManager({});
let summary = null;
mgr.on('batch-done', (s) => { summary = s; });
const start = Date.now();
await mgr.startBatch([]);
const elapsed = Date.now() - start;
assert.ok(summary, 'batch-done should be emitted');
assert.equal(summary.total, 0);
assert.equal(summary.succeeded, 0);
assert.equal(summary.failed, 0);
assert.ok(elapsed < 200, `empty batch should complete fast, took ${elapsed}ms`);
});
it('scaleParallelUploads limits per-hoster count to global limit', async () => {
let concurrent = 0;
let maxConcurrent = 0;
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress) => {
concurrent++;
maxConcurrent = Math.max(maxConcurrent, concurrent);
await new Promise(r => setTimeout(r, 40));
concurrent--;
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager(
{ 'doodstream.com': { retries: 0, parallelCount: 10, maxSpeedKbs: 0, restartBelowKbs: 0, timeIntervalSec: 0, maxSizeMb: 0 } },
{ parallelUploadCount: 2, scaleParallelUploads: true }
);
await mgr.startBatch([
{ file: '/test/a.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ file: '/test/b.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ file: '/test/c.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ file: '/test/d.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ file: '/test/e.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
assert.ok(maxConcurrent <= 2, `scaleParallelUploads should cap at 2, was ${maxConcurrent}`);
});
it('addJobs injects new tasks into running batch', async () => {
let started = 0;
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress) => {
started++;
await new Promise(r => setTimeout(r, 100));
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({});
let summary = null;
mgr.on('batch-done', (s) => { summary = s; });
// Start batch with 2 tasks
const batchPromise = mgr.startBatch([
{ jobId: 'job-1', file: '/test/a.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ jobId: 'job-2', file: '/test/b.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
// After 30ms (during upload), inject 2 more tasks
await new Promise(r => setTimeout(r, 30));
const result = mgr.addJobs([
{ jobId: 'job-3', file: '/test/c.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ jobId: 'job-4', file: '/test/d.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
assert.equal(result.added, 2, 'should add 2 new jobs');
assert.equal(result.alreadyInBatchJobIds.length, 0);
await batchPromise;
assert.ok(summary);
assert.equal(started, 4, 'all 4 jobs should have run');
});
it('addJobs rejects duplicates already in running batch', async () => {
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress, signal) => {
// Slow upload so we can add jobs while it's running
await new Promise((resolve, reject) => {
const timer = setTimeout(resolve, 200);
if (signal) signal.addEventListener('abort', () => { clearTimeout(timer); reject(new Error('Aborted')); });
});
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({});
const batchPromise = mgr.startBatch([
{ jobId: 'job-A', file: '/test/x.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
// Try to add the SAME jobId while it's running
await new Promise(r => setTimeout(r, 50));
const result = mgr.addJobs([
{ jobId: 'job-A', file: '/test/x.mp4', hoster: 'doodstream.com', apiKey: 'k' },
{ jobId: 'job-B', file: '/test/y.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
assert.equal(result.added, 1, 'should skip duplicate jobId, add only the new one');
assert.deepEqual(result.alreadyInBatchJobIds, ['job-A']);
await batchPromise;
});
it('addJobs returns added=0 when not running', () => {
const mgr = new UploadManager({});
const result = mgr.addJobs([
{ jobId: 'job-1', file: '/test/a.mp4', hoster: 'doodstream.com', apiKey: 'k' }
]);
assert.equal(result.added, 0);
});
it('stats event contains expected fields', async () => {
// Make upload take long enough for stats interval to fire
mockUploadFile.mock.mockImplementation(async (hoster, filePath, apiKey, onProgress, signal) => {
await new Promise(r => setTimeout(r, 1500));
if (onProgress) onProgress(fakeFileSize, fakeFileSize);
return { download_url: 'https://test/ok', embed_url: null, file_code: 'ok' };
});
const mgr = new UploadManager({});
const statsEvents = [];
mgr.on('stats', (d) => statsEvents.push(d));
await mgr.startBatch([
{ file: '/test/video.mp4', hoster: 'doodstream.com', apiKey: 'key1' }
]);
assert.ok(statsEvents.length > 0, 'should have received stats events');
const stat = statsEvents[0];
assert.ok('globalSpeedKbs' in stat);
assert.ok('totalBytes' in stat);
assert.ok('elapsed' in stat);
assert.ok('activeJobs' in stat);
});
describe('error classification', () => {
it('treats "not enough disk space" as account-level, not file-rejected', () => {
const mgr = new UploadManager({});
// Shape matches what lib/hosters.js attaches for byse account-storage-full
const err = new Error('Byse lehnte Datei ab: 0:0:0:not enough disk space on your account');
err.accountError = true;
assert.equal(mgr._isFileRejectedError(err), false,
'account-level error must NOT be classified as file-rejected');
assert.equal(mgr._shouldSkipRetryOnAccountError(err), true,
'account-storage-full must trigger account rotation');
});
it('classifies disk-space errors by message alone (safety net)', () => {
const mgr = new UploadManager({});
const err = new Error('Byse lehnte Datei ab: not enough disk space');
// No flag set — regex alone must catch it.
assert.equal(mgr._shouldSkipRetryOnAccountError(err), true);
assert.equal(mgr._isFileRejectedError(err), false,
'must not match generic "lehnte Datei ab" as file-rejected');
});
it('keeps true file rejections as file-rejected', () => {
const mgr = new UploadManager({});
const err = new Error('Byse lehnte Datei ab: Duplicate');
err.fileRejected = true;
assert.equal(mgr._isFileRejectedError(err), true);
assert.equal(mgr._shouldSkipRetryOnAccountError(err), false);
});
it('file-rejected regex still matches known phrases without flag', () => {
const mgr = new UploadManager({});
for (const msg of [
'Not video file format',
'Duplicate',
'Datei zu klein',
'File too large',
'Invalid file'
]) {
assert.equal(mgr._isFileRejectedError(new Error(msg)), true, `should match: ${msg}`);
}
});
it('accountError flag beats fileRejected if both set (defensive)', () => {
const mgr = new UploadManager({});
const err = new Error('weird');
err.fileRejected = true;
err.accountError = true;
assert.equal(mgr._isFileRejectedError(err), false,
'account-level always wins — rotation must happen');
assert.equal(mgr._shouldSkipRetryOnAccountError(err), true);
});
});
});