Administrator/Multi-Hoster-Upload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 207 Wiki Activity

fix(perf): chunked startBatch + async rotLog — kill remaining 30s freeze on 5k+ jobs

Browse Source
This commit is contained in:
Administrator 2026-06-08 01:29:31 +02:00
parent ba4642e09a
commit d9199f8aaf
2 changed files with 22 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/upload-manager.js
View File

@ -310,18 +310,30 @@ class UploadManager extends EventEmitter {
this._batchResults = results; this._batchResults = results;
this._additionalPromises = []; // Track jobs added mid-batch via addJobs() this._additionalPromises = []; // Track jobs added mid-batch via addJobs()
for (const task of tasks) { const DEDUP_CHUNK = 200;
const fileName = path.basename(task.file); for (let i = 0; i < tasks.length; i += DEDUP_CHUNK) {
if (!results.has(task.file)) { const end = Math.min(i + DEDUP_CHUNK, tasks.length);
let size = 0; for (let j = i; j < end; j++) {
try { size = fs.statSync(task.file).size; } catch {} const task = tasks[j];
results.set(task.file, { name: fileName, size, results: [] }); if (!results.has(task.file)) {
const fileName = path.basename(task.file);
let size = 0;
try { size = fs.statSync(task.file).size; } catch {}
results.set(task.file, { name: fileName, size, results: [] });
}
} }
if (end < tasks.length) await new Promise(setImmediate);
} }
this._startStatsTimer(); this._startStatsTimer();
const promises = tasks.map((task) => this._runJob(task, results, signal)); const SPAWN_CHUNK = 100;
const promises = [];
for (let i = 0; i < tasks.length; i += SPAWN_CHUNK) {
const end = Math.min(i + SPAWN_CHUNK, tasks.length);
for (let j = i; j < end; j++) promises.push(this._runJob(tasks[j], results, signal));
if (end < tasks.length) await new Promise(setImmediate);
}
await Promise.allSettled(promises); await Promise.allSettled(promises);
// Wait for any jobs added mid-batch via addJobs() // Wait for any jobs added mid-batch via addJobs()
while (this._additionalPromises.length > 0) { while (this._additionalPromises.length > 0) {

19
main.js
View File

@ -216,23 +216,10 @@ function rotLog(msg, ts) {
try { try {
const iso = new Date(ts || Date.now()).toISOString(); const iso = new Date(ts || Date.now()).toISOString();
const line = `[${iso}] ${msg}\n`; const line = `[${iso}] ${msg}\n`;
// Write synchronously. Rotation events are rare (a handful per batch) so _rotLogBuffer.push(line);
// the batching optimization from debugLog doesn't buy us anything, and if (!_rotLogFlushTimer) {
// syncing guarantees the user can refresh the file and see fresh entries _rotLogFlushTimer = setTimeout(() => { _rotLogFlushTimer = null; _flushRotLog(); }, 500);
// without waiting on a flush timer.
const candidates = [
getRotLogPath(),
path.join(app.getPath('desktop') || app.getPath('userData'), 'account-rotation.log'),
path.join(app.getPath('userData'), 'account-rotation.log')
];
for (const target of candidates) {
try {
fs.mkdirSync(path.dirname(target), { recursive: true });
fs.appendFileSync(target, line, 'utf-8');
break;
} catch {}
} }
// Mirror into the main debug log for single-file-grep convenience.
_debugLogBuffer.push(`[${iso}] [ROT] ${msg}\n`); _debugLogBuffer.push(`[${iso}] [ROT] ${msg}\n`);
if (!_debugLogFlushTimer) { if (!_debugLogFlushTimer) {
_debugLogFlushTimer = setTimeout(() => { _debugLogFlushTimer = null; _flushDebugLog(); }, 500); _debugLogFlushTimer = setTimeout(() => { _debugLogFlushTimer = null; _flushDebugLog(); }, 500);