Release v2.1.65: safe MCP updates and measured upload scheduling
CI / verify (push) Canceled after 0s

This commit is contained in:
Sucukdeluxe committed 2026-10-16 05:36:42 +02:00
1 parent e3c03d195e
commit 1ff552fc40
26 files changed
+552 -92

No files matched your search

+41 -22
View File
@@ -10,6 +10,7 @@ const DoodstreamUploader = require('./doodstream-upload');
const ClouddropUploader = require('./clouddrop-upload');
const Semaphore = require('./semaphore');
const Throttle = require('./throttle');
const { runtimePerformance } = require('./performance-monitor');
const { probeFileHead } = require('./file-probe');
const { normalizeFailureDetails } = require('./upload-diagnostics');
const { classifyAccountFailure, selectFallbackAccount } = require('./account-rotation');
@@ -47,6 +48,7 @@ class UploadManager extends EventEmitter {
this.pendingCancelledJobIds = new Set();
this.pendingCancelAll = false;
this.sessionBytes = 0;
this.transferredBytes = 0;
this._transientErrorTotal = 0;
this.lastStartTime = {}; // hoster -> timestamp of last upload start
this.intervalLocks = {}; // hoster -> Promise chain for serialized interval waits
@@ -115,13 +117,28 @@ class UploadManager extends EventEmitter {
getDiagnostics() {
const activeByHoster = {};
const phases = {};
for (const v of this.activeJobs.values()) {
const h = v && v.hoster ? v.hoster : 'unknown';
activeByHoster[h] = (activeByHoster[h] || 0) + 1;
const phase = v.phase || 'getting-server';
phases[phase] = (phases[phase] || 0) + 1;
}
let pending = 0;
for (const sem of Object.values(this.semaphores)) pending += (sem && sem.pending) || 0;
return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size };
return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size,
globalSlots: this.globalSemaphore ? { active: this.globalSemaphore.active, pending: this.globalSemaphore.pending, limit: this.globalSemaphore.limit } : null,
hosterSlots: Object.fromEntries(Object.entries(this.semaphores).map(([hoster, sem]) => [hoster, { active: sem.active, pending: sem.pending, limit: sem.limit }])),
sessionBytes: this.sessionBytes, transferredBytes: this.transferredBytes, batchStartedAt: this.startTime,
speedKbs: [...this.activeJobs.values()].reduce((sum, job) => sum + (job.speedKbs || 0), 0), phases };
}
_recordTransferred(entry, bytes, total) {
if (!Number.isFinite(bytes) || bytes < 0) return;
const next = Math.min(bytes, total);
this.transferredBytes += Math.max(0, next - entry.bytesUploaded);
entry.bytesUploaded = next;
entry.phase = next >= total ? 'awaiting-confirmation' : 'uploading';
}
clearFailedAccount(hoster, accountId) {
@@ -368,6 +385,7 @@ class UploadManager extends EventEmitter {
if (pendingCancelAll) this.abortController.abort();
this.startTime = Date.now();
this.sessionBytes = 0;
this.transferredBytes = 0;
this.activeJobs.clear();
this.jobAbortControllers.clear();
this.cancelledJobIds.clear();
@@ -478,6 +496,7 @@ class UploadManager extends EventEmitter {
} else {
try { fileSize = (await fs.promises.stat(task.file)).size; } catch { fileNotFound = true; }
}
if (cachedResult && fileSize > 0) cachedResult.size = fileSize;
const maxAttempts = Math.max(1, (settings.retries || 0) + 1);
const jobAbortController = new AbortController();
@@ -502,6 +521,7 @@ class UploadManager extends EventEmitter {
if (finalResultRecorded) return;
finalResultRecorded = true;
finalStatus = status;
if (status === 'error') runtimePerformance.event('upload-error', { hoster: task.hoster });
const result = {
jobId,
@@ -573,7 +593,7 @@ class UploadManager extends EventEmitter {
// queueJobs array; the first event it actually needs from main is the
// 'getting-server' / 'uploading' transition for the jobs that the
// semaphore lets through.
await hosterSemaphore.acquire(admissionSignal);
await runtimePerformance.measure('slots.hoster', () => hosterSemaphore.acquire(admissionSignal));
hosterSlotAcquired = true;
if (this.stopAfterActive) {
@@ -585,7 +605,7 @@ class UploadManager extends EventEmitter {
let fileProbe = null;
try {
fileProbe = await probeFileHead(task.file, 512);
fileProbe = await runtimePerformance.measure('file.probe', () => probeFileHead(task.file, 512));
} catch (err) {
fileProbe = { ok: false, error: err && err.message, kind: 'unreadable' };
}
@@ -597,16 +617,15 @@ class UploadManager extends EventEmitter {
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
});
if (globalSemaphore) {
await globalSemaphore.acquire(admissionSignal);
globalSlotAcquired = true;
}
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
const acquireGlobal = async () => {
if (globalSemaphore) {
await runtimePerformance.measure('slots.global', () => globalSemaphore.acquire(admissionSignal));
globalSlotAcquired = true;
}
};
if (settings.timeIntervalSec > 0) {
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal);
}
await runtimePerformance.measure('slots.interval', () => this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal, acquireGlobal));
} else await acquireGlobal();
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
@@ -746,7 +765,7 @@ class UploadManager extends EventEmitter {
}
activeEntry.speedKbs = currentSpeedKbs;
activeEntry.bytesUploaded = bytesUploaded;
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
lastEmitTime = now;
@@ -1080,7 +1099,7 @@ class UploadManager extends EventEmitter {
lastSpeedTime = now;
}
activeEntry.speedKbs = currentSpeedKbs;
activeEntry.bytesUploaded = bytesUploaded;
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
lastEmitTime = now;
const elapsed = Math.round((now - jobStart) / 1000);
@@ -1234,7 +1253,7 @@ class UploadManager extends EventEmitter {
lastSpeedTime = now;
}
activeEntry.speedKbs = currentSpeedKbs;
activeEntry.bytesUploaded = bytesUploaded;
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
lastEmitTime = now;
const elapsed = Math.round((now - jobStart) / 1000);
@@ -1490,18 +1509,20 @@ class UploadManager extends EventEmitter {
});
}
_waitForInterval(hoster, intervalMs, signal) {
_waitForInterval(hoster, intervalMs, signal, admit = async () => {}) {
// Serialize interval waits per hoster so concurrent jobs queue up properly
const prev = this.intervalLocks[hoster] || Promise.resolve();
const next = prev.then(async () => {
const prev = this.intervalLocks[hoster];
const waitAndAdmit = async () => {
const now = Date.now();
const last = this.lastStartTime[hoster] || 0;
const elapsed = now - last;
if (elapsed < intervalMs) {
await this._sleep(intervalMs - elapsed, signal);
}
await admit();
this.lastStartTime[hoster] = Date.now();
});
};
const next = prev ? prev.then(waitAndAdmit) : waitAndAdmit();
this.intervalLocks[hoster] = next.catch(() => {});
return next;
}
@@ -1521,9 +1542,7 @@ class UploadManager extends EventEmitter {
}
const fileName = path.basename(task.file);
if (!results.has(task.file)) {
let size = 0;
try { size = fs.statSync(task.file).size; } catch {}
results.set(task.file, { name: fileName, size, results: [] });
results.set(task.file, { name: fileName, size: 0, results: [] });
}
this._additionalPromises.push(this._runJob(task, results, signal));
addResult.added++;