fix: eliminate upload queue stalls under automation load
Move atomic configuration writes off the Electron main event loop and preserve concurrent queue and folder monitor state. Make finish-and-pause interrupt queued admission waits without aborting active uploads, then resume only persistently marked automation jobs across active, idle, restart, and finalization races. Reuse bounded automation evidence snapshots, refresh them during long drains, and cover high-load persistence, pause, resume, and hidden Electron behavior with regression tests.
This commit is contained in:
+62
-4
@@ -33,6 +33,9 @@ class UploadManager extends EventEmitter {
|
||||
this.semaphores = {};
|
||||
this.globalSemaphore = null;
|
||||
this.abortController = new AbortController();
|
||||
this.queueAdmissionAbortController = new AbortController();
|
||||
this.queueAdmissionPending = 0;
|
||||
this.queueAdmissionWaiters = [];
|
||||
this.running = false;
|
||||
this.stopAfterActive = false;
|
||||
this.statsInterval = null;
|
||||
@@ -344,6 +347,9 @@ class UploadManager extends EventEmitter {
|
||||
this.running = true;
|
||||
this.stopAfterActive = pendingCancelAll;
|
||||
this.abortController = new AbortController();
|
||||
this.queueAdmissionAbortController = new AbortController();
|
||||
this.queueAdmissionPending = 0;
|
||||
this.queueAdmissionWaiters = [];
|
||||
if (pendingCancelAll) this.abortController.abort();
|
||||
this.startTime = Date.now();
|
||||
this.sessionBytes = 0;
|
||||
@@ -462,6 +468,11 @@ class UploadManager extends EventEmitter {
|
||||
const jobAbortController = new AbortController();
|
||||
if (this.cancelledJobIds.has(jobId)) jobAbortController.abort();
|
||||
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
|
||||
const { signal: admissionSignal, cleanup: cleanupAdmissionSignals } = this._combineSignals(
|
||||
signal,
|
||||
this.queueAdmissionAbortController.signal
|
||||
);
|
||||
const leaveQueueAdmission = this._enterQueueAdmission();
|
||||
this.jobAbortControllers.set(jobId, jobAbortController);
|
||||
|
||||
let hosterSlotAcquired = false;
|
||||
@@ -546,9 +557,16 @@ class UploadManager extends EventEmitter {
|
||||
// queueJobs array; the first event it actually needs from main is the
|
||||
// 'getting-server' / 'uploading' transition for the jobs that the
|
||||
// semaphore lets through.
|
||||
await hosterSemaphore.acquire(signal);
|
||||
await hosterSemaphore.acquire(admissionSignal);
|
||||
hosterSlotAcquired = true;
|
||||
|
||||
if (this.stopAfterActive) {
|
||||
const error = 'Warteschlange angehalten';
|
||||
emitFinalStatus('aborted', { error, attempt: 0 });
|
||||
recordFinalResult('aborted', { error });
|
||||
return;
|
||||
}
|
||||
|
||||
let fileProbe = null;
|
||||
try {
|
||||
fileProbe = await probeFileHead(task.file, 512);
|
||||
@@ -564,14 +582,20 @@ class UploadManager extends EventEmitter {
|
||||
});
|
||||
|
||||
if (globalSemaphore) {
|
||||
await globalSemaphore.acquire(signal);
|
||||
await globalSemaphore.acquire(admissionSignal);
|
||||
globalSlotAcquired = true;
|
||||
}
|
||||
|
||||
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
|
||||
|
||||
if (settings.timeIntervalSec > 0) {
|
||||
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal);
|
||||
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal);
|
||||
}
|
||||
|
||||
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
|
||||
|
||||
leaveQueueAdmission();
|
||||
|
||||
// Pre-job-swap: if this account was marked failed WHILE this task was
|
||||
// waiting in the semaphore queue, jump straight to the override instead
|
||||
// of burning a guaranteed-to-fail upload attempt. Critical at scale:
|
||||
@@ -1115,6 +1139,8 @@ class UploadManager extends EventEmitter {
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.jobAbortControllers.delete(jobId);
|
||||
cleanupSignals();
|
||||
cleanupAdmissionSignals();
|
||||
leaveQueueAdmission();
|
||||
// Release in reverse order of acquire (global first, then hoster)
|
||||
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
|
||||
if (hosterSlotAcquired) hosterSemaphore.release();
|
||||
@@ -1467,7 +1493,7 @@ class UploadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
addJobs(tasks) {
|
||||
if (!this.running || !tasks || tasks.length === 0) {
|
||||
if (!this.running || this.stopAfterActive || !tasks || tasks.length === 0) {
|
||||
return { added: 0, alreadyInBatchJobIds: [] };
|
||||
}
|
||||
const { signal } = this.abortController;
|
||||
@@ -1505,6 +1531,38 @@ class UploadManager extends EventEmitter {
|
||||
|
||||
finishAfterActive() {
|
||||
this.stopAfterActive = true;
|
||||
if (!this.queueAdmissionAbortController.signal.aborted) this.queueAdmissionAbortController.abort();
|
||||
}
|
||||
|
||||
async resumeAfterActive() {
|
||||
const stoppedController = this.queueAdmissionAbortController;
|
||||
await this._waitForQueueAdmissionIdle();
|
||||
if (this.queueAdmissionAbortController === stoppedController && stoppedController.signal.aborted) {
|
||||
this.queueAdmissionAbortController = new AbortController();
|
||||
}
|
||||
this.stopAfterActive = false;
|
||||
}
|
||||
|
||||
isStoppingAfterActive() {
|
||||
return this.stopAfterActive;
|
||||
}
|
||||
|
||||
_enterQueueAdmission() {
|
||||
this.queueAdmissionPending++;
|
||||
let active = true;
|
||||
return () => {
|
||||
if (!active) return;
|
||||
active = false;
|
||||
this.queueAdmissionPending--;
|
||||
if (this.queueAdmissionPending !== 0) return;
|
||||
const waiters = this.queueAdmissionWaiters.splice(0);
|
||||
for (const resolve of waiters) resolve();
|
||||
};
|
||||
}
|
||||
|
||||
_waitForQueueAdmissionIdle() {
|
||||
if (this.queueAdmissionPending === 0) return Promise.resolve();
|
||||
return new Promise(resolve => this.queueAdmissionWaiters.push(resolve));
|
||||
}
|
||||
|
||||
cancel() {
|
||||
|
||||
Reference in New Issue
Block a user