Release v2.1.65: safe MCP updates and measured upload scheduling
CI / verify (push) Canceled after 0s
CI / verify (push) Canceled after 0s
This commit is contained in:
1 parent
e3c03d195e
commit
1ff552fc40
26 files changed
+552
-92
No files matched your search
+41
-22
@@ -10,6 +10,7 @@ const DoodstreamUploader = require('./doodstream-upload');
|
||||
const ClouddropUploader = require('./clouddrop-upload');
|
||||
const Semaphore = require('./semaphore');
|
||||
const Throttle = require('./throttle');
|
||||
const { runtimePerformance } = require('./performance-monitor');
|
||||
const { probeFileHead } = require('./file-probe');
|
||||
const { normalizeFailureDetails } = require('./upload-diagnostics');
|
||||
const { classifyAccountFailure, selectFallbackAccount } = require('./account-rotation');
|
||||
@@ -47,6 +48,7 @@ class UploadManager extends EventEmitter {
|
||||
this.pendingCancelledJobIds = new Set();
|
||||
this.pendingCancelAll = false;
|
||||
this.sessionBytes = 0;
|
||||
this.transferredBytes = 0;
|
||||
this._transientErrorTotal = 0;
|
||||
this.lastStartTime = {}; // hoster -> timestamp of last upload start
|
||||
this.intervalLocks = {}; // hoster -> Promise chain for serialized interval waits
|
||||
@@ -115,13 +117,28 @@ class UploadManager extends EventEmitter {
|
||||
|
||||
getDiagnostics() {
|
||||
const activeByHoster = {};
|
||||
const phases = {};
|
||||
for (const v of this.activeJobs.values()) {
|
||||
const h = v && v.hoster ? v.hoster : 'unknown';
|
||||
activeByHoster[h] = (activeByHoster[h] || 0) + 1;
|
||||
const phase = v.phase || 'getting-server';
|
||||
phases[phase] = (phases[phase] || 0) + 1;
|
||||
}
|
||||
let pending = 0;
|
||||
for (const sem of Object.values(this.semaphores)) pending += (sem && sem.pending) || 0;
|
||||
return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size };
|
||||
return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size,
|
||||
globalSlots: this.globalSemaphore ? { active: this.globalSemaphore.active, pending: this.globalSemaphore.pending, limit: this.globalSemaphore.limit } : null,
|
||||
hosterSlots: Object.fromEntries(Object.entries(this.semaphores).map(([hoster, sem]) => [hoster, { active: sem.active, pending: sem.pending, limit: sem.limit }])),
|
||||
sessionBytes: this.sessionBytes, transferredBytes: this.transferredBytes, batchStartedAt: this.startTime,
|
||||
speedKbs: [...this.activeJobs.values()].reduce((sum, job) => sum + (job.speedKbs || 0), 0), phases };
|
||||
}
|
||||
|
||||
_recordTransferred(entry, bytes, total) {
|
||||
if (!Number.isFinite(bytes) || bytes < 0) return;
|
||||
const next = Math.min(bytes, total);
|
||||
this.transferredBytes += Math.max(0, next - entry.bytesUploaded);
|
||||
entry.bytesUploaded = next;
|
||||
entry.phase = next >= total ? 'awaiting-confirmation' : 'uploading';
|
||||
}
|
||||
|
||||
clearFailedAccount(hoster, accountId) {
|
||||
@@ -368,6 +385,7 @@ class UploadManager extends EventEmitter {
|
||||
if (pendingCancelAll) this.abortController.abort();
|
||||
this.startTime = Date.now();
|
||||
this.sessionBytes = 0;
|
||||
this.transferredBytes = 0;
|
||||
this.activeJobs.clear();
|
||||
this.jobAbortControllers.clear();
|
||||
this.cancelledJobIds.clear();
|
||||
@@ -478,6 +496,7 @@ class UploadManager extends EventEmitter {
|
||||
} else {
|
||||
try { fileSize = (await fs.promises.stat(task.file)).size; } catch { fileNotFound = true; }
|
||||
}
|
||||
if (cachedResult && fileSize > 0) cachedResult.size = fileSize;
|
||||
|
||||
const maxAttempts = Math.max(1, (settings.retries || 0) + 1);
|
||||
const jobAbortController = new AbortController();
|
||||
@@ -502,6 +521,7 @@ class UploadManager extends EventEmitter {
|
||||
if (finalResultRecorded) return;
|
||||
finalResultRecorded = true;
|
||||
finalStatus = status;
|
||||
if (status === 'error') runtimePerformance.event('upload-error', { hoster: task.hoster });
|
||||
|
||||
const result = {
|
||||
jobId,
|
||||
@@ -573,7 +593,7 @@ class UploadManager extends EventEmitter {
|
||||
// queueJobs array; the first event it actually needs from main is the
|
||||
// 'getting-server' / 'uploading' transition for the jobs that the
|
||||
// semaphore lets through.
|
||||
await hosterSemaphore.acquire(admissionSignal);
|
||||
await runtimePerformance.measure('slots.hoster', () => hosterSemaphore.acquire(admissionSignal));
|
||||
hosterSlotAcquired = true;
|
||||
|
||||
if (this.stopAfterActive) {
|
||||
@@ -585,7 +605,7 @@ class UploadManager extends EventEmitter {
|
||||
|
||||
let fileProbe = null;
|
||||
try {
|
||||
fileProbe = await probeFileHead(task.file, 512);
|
||||
fileProbe = await runtimePerformance.measure('file.probe', () => probeFileHead(task.file, 512));
|
||||
} catch (err) {
|
||||
fileProbe = { ok: false, error: err && err.message, kind: 'unreadable' };
|
||||
}
|
||||
@@ -597,16 +617,15 @@ class UploadManager extends EventEmitter {
|
||||
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
|
||||
});
|
||||
|
||||
if (globalSemaphore) {
|
||||
await globalSemaphore.acquire(admissionSignal);
|
||||
globalSlotAcquired = true;
|
||||
}
|
||||
|
||||
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
|
||||
|
||||
const acquireGlobal = async () => {
|
||||
if (globalSemaphore) {
|
||||
await runtimePerformance.measure('slots.global', () => globalSemaphore.acquire(admissionSignal));
|
||||
globalSlotAcquired = true;
|
||||
}
|
||||
};
|
||||
if (settings.timeIntervalSec > 0) {
|
||||
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal);
|
||||
}
|
||||
await runtimePerformance.measure('slots.interval', () => this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal, acquireGlobal));
|
||||
} else await acquireGlobal();
|
||||
|
||||
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
|
||||
|
||||
@@ -746,7 +765,7 @@ class UploadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
activeEntry.speedKbs = currentSpeedKbs;
|
||||
activeEntry.bytesUploaded = bytesUploaded;
|
||||
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
|
||||
|
||||
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
|
||||
lastEmitTime = now;
|
||||
@@ -1080,7 +1099,7 @@ class UploadManager extends EventEmitter {
|
||||
lastSpeedTime = now;
|
||||
}
|
||||
activeEntry.speedKbs = currentSpeedKbs;
|
||||
activeEntry.bytesUploaded = bytesUploaded;
|
||||
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
|
||||
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
|
||||
lastEmitTime = now;
|
||||
const elapsed = Math.round((now - jobStart) / 1000);
|
||||
@@ -1234,7 +1253,7 @@ class UploadManager extends EventEmitter {
|
||||
lastSpeedTime = now;
|
||||
}
|
||||
activeEntry.speedKbs = currentSpeedKbs;
|
||||
activeEntry.bytesUploaded = bytesUploaded;
|
||||
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
|
||||
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
|
||||
lastEmitTime = now;
|
||||
const elapsed = Math.round((now - jobStart) / 1000);
|
||||
@@ -1490,18 +1509,20 @@ class UploadManager extends EventEmitter {
|
||||
});
|
||||
}
|
||||
|
||||
_waitForInterval(hoster, intervalMs, signal) {
|
||||
_waitForInterval(hoster, intervalMs, signal, admit = async () => {}) {
|
||||
// Serialize interval waits per hoster so concurrent jobs queue up properly
|
||||
const prev = this.intervalLocks[hoster] || Promise.resolve();
|
||||
const next = prev.then(async () => {
|
||||
const prev = this.intervalLocks[hoster];
|
||||
const waitAndAdmit = async () => {
|
||||
const now = Date.now();
|
||||
const last = this.lastStartTime[hoster] || 0;
|
||||
const elapsed = now - last;
|
||||
if (elapsed < intervalMs) {
|
||||
await this._sleep(intervalMs - elapsed, signal);
|
||||
}
|
||||
await admit();
|
||||
this.lastStartTime[hoster] = Date.now();
|
||||
});
|
||||
};
|
||||
const next = prev ? prev.then(waitAndAdmit) : waitAndAdmit();
|
||||
this.intervalLocks[hoster] = next.catch(() => {});
|
||||
return next;
|
||||
}
|
||||
@@ -1521,9 +1542,7 @@ class UploadManager extends EventEmitter {
|
||||
}
|
||||
const fileName = path.basename(task.file);
|
||||
if (!results.has(task.file)) {
|
||||
let size = 0;
|
||||
try { size = fs.statSync(task.file).size; } catch {}
|
||||
results.set(task.file, { name: fileName, size, results: [] });
|
||||
results.set(task.file, { name: fileName, size: 0, results: [] });
|
||||
}
|
||||
this._additionalPromises.push(this._runJob(task, results, signal));
|
||||
addResult.added++;
|
||||
|
||||
Reference in new issue
Block a user