Add persistent upload statistics alongside history
CI / verify (push) Canceled after 0s

This commit is contained in:
Sucukdeluxe committed 2026-10-24 13:21:04 +02:00
1 parent 54a6bcbb6f
commit dc1d042d73
10 files changed
+784 -8

No files matched your search

+7 -2
View File
@@ -137,7 +137,9 @@ class UploadManager extends EventEmitter {
_recordTransferred(entry, bytes, total) {
if (!Number.isFinite(bytes) || bytes < 0) return;
const next = Math.min(bytes, total);
this.transferredBytes += Math.max(0, next - entry.bytesUploaded);
const delta = Math.max(0, next - entry.bytesUploaded);
this.transferredBytes += delta;
if (delta > 0) this.emit('statistics-transfer', { hoster: entry.hoster, bytes: delta });
entry.bytesUploaded = next;
entry.phase = next >= total ? 'awaiting-confirmation' : 'uploading';
}
@@ -431,6 +433,7 @@ class UploadManager extends EventEmitter {
const batchId = `batch-${Date.now()}`;
const results = new Map(); // filePath -> { name, size, results: [] }
this._batchResults = results;
this._batchTaskCount = tasks.length;
this._additionalPromises = []; // Track jobs added mid-batch via addJobs()
const DEDUP_CHUNK = 200;
@@ -472,7 +475,7 @@ class UploadManager extends EventEmitter {
this._emitStats();
const files = Array.from(results.values());
const total = tasks.length;
const total = this._batchTaskCount;
const succeeded = files.reduce((count, file) => count + file.results.filter((result) => result.status === 'done').length, 0);
const skipped = files.reduce((count, file) => count + file.results.filter((result) => result.status === 'skipped').length, 0);
@@ -549,6 +552,7 @@ class UploadManager extends EventEmitter {
};
results.get(task.file).results.push(result);
this.emit('statistics-outcome', { id: uploadId, hoster: task.hoster, status });
};
const emitFinalStatus = (status, payload = {}) => {
@@ -1557,6 +1561,7 @@ class UploadManager extends EventEmitter {
}
this._additionalPromises.push(this._runJob(task, results, signal));
addResult.added++;
this._batchTaskCount++;
}
return addResult;
}
+186
View File
@@ -0,0 +1,186 @@
const fs = require('node:fs/promises');
const path = require('node:path');
const HOSTERS = Object.freeze({ 'doodstream.com': 'Doodstream', 'voe.sx': 'VOE', 'vidmoly.me': 'Vidmoly', 'byse.sx': 'Byse', 'clouddrop.cc': 'CloudDrop' });
const RANGES = new Set(['session', 'today', 'last24', 'week', 'month', 'all']);
const MINUTE = 60000;
const DAY = 86400000;
const empty = () => ({ bytes: 0, trackedBytes: 0, completed: 0, failed: 0, activeMs: 0, providers: {} });
const count = value => Number.isFinite(value) && value >= 0 ? value : 0;
const dayKey = time => {
const date = new Date(time);
return [date.getFullYear(), String(date.getMonth() + 1).padStart(2, '0'), String(date.getDate()).padStart(2, '0')].join('-');
};
function add(target, delta, hoster) {
for (const key of ['bytes', 'trackedBytes', 'completed', 'failed', 'activeMs']) target[key] = count(target[key]) + count(delta[key]);
if (Object.hasOwn(HOSTERS, hoster)) {
const provider = target.providers[hoster] ||= { bytes: 0, completed: 0, failed: 0 };
for (const key of ['bytes', 'completed', 'failed']) provider[key] += count(delta[key]);
}
}
function merge(target, source) {
add(target, source);
for (const [hoster, bucket] of Object.entries(source.providers || {})) {
if (!Object.hasOwn(HOSTERS, hoster)) continue;
const provider = target.providers[hoster] ||= { bytes: 0, completed: 0, failed: 0 };
for (const key of ['bytes', 'completed', 'failed']) provider[key] += count(bucket[key]);
}
}
function validState(value) {
return value?.version === 1 && value.days && value.minutes && value.total?.providers && Number.isFinite(value.startedAt);
}
class UploadStatistics {
constructor(filePath, { now = Date.now } = {}) {
this.filePath = filePath;
this.now = now;
this.session = empty();
this.sessionStartedAt = now();
this.seen = new Set();
this.activeSince = null;
this.writeChain = Promise.resolve();
this.revision = 0;
this.savedRevision = -1;
this.ready = false;
this.error = '';
}
async init(history = []) {
let present = false;
for (const name of [this.filePath, this.filePath + '.bak']) {
try {
const value = JSON.parse(await fs.readFile(name, 'utf8'));
if (!validState(value)) throw new Error('Ungültiges Statistikformat');
this.state = value;
this.savedRevision = 0;
break;
} catch (error) {
if (error.code !== 'ENOENT') present = true;
}
}
if (!this.state && present) throw new Error('Statistikdatei und Sicherung sind nicht lesbar');
if (!this.state) {
this.state = { version: 1, startedAt: this.now(), historyImported: false, total: empty(), days: {}, minutes: {} };
for (const batch of Array.isArray(history) ? history : []) {
const time = Date.parse(batch?.timestamp);
if (!Number.isFinite(time) || time > this.now()) continue;
for (const file of batch.files || []) {
for (const result of file.results || []) {
if (!Object.hasOwn(HOSTERS, result.hoster) || !['done', 'error'].includes(result.status)) continue;
this.record(result.hoster, { bytes: result.status === 'done' ? count(file.size) : 0, completed: Number(result.status === 'done'), failed: Number(result.status === 'error') }, time, false);
this.state.historyImported = true;
this.state.startedAt = Math.min(this.state.startedAt, time);
}
}
}
}
this.ready = true;
await this.flush();
}
record(hoster, delta, time = this.now(), session = true) {
if (session) {
delta = { ...delta, trackedBytes: count(delta.bytes) };
add(this.session, delta, hoster);
}
add(this.state.total, delta, hoster);
add(this.state.days[dayKey(time)] ||= empty(), delta, hoster);
if (time >= this.now() - 2 * DAY) add(this.state.minutes[Math.floor(time / MINUTE)] ||= empty(), delta, hoster);
this.revision++;
}
transfer(hoster, bytes) {
if (!this.ready || !Object.hasOwn(HOSTERS, hoster) || !Number.isFinite(bytes) || bytes <= 0) return;
this.record(hoster, { bytes });
}
outcome({ id, hoster, status }) {
if (!this.ready || !id || !Object.hasOwn(HOSTERS, hoster) || !['done', 'error'].includes(status) || this.seen.has(id)) return;
this.seen.add(id);
this.record(hoster, { completed: Number(status === 'done'), failed: Number(status === 'error') });
}
activity(active) {
if (!this.ready) return;
const now = this.now();
if (this.activeSince !== null) {
let start = this.activeSince;
while (start < now) {
const end = Math.min(now, (Math.floor(start / MINUTE) + 1) * MINUTE);
this.record(null, { activeMs: end - start }, start);
start = end;
}
}
this.activeSince = active ? now : null;
}
finishBatch() {
this.activity(false);
this.seen.clear();
}
snapshot(range = 'session') {
if (!RANGES.has(range)) throw new Error('Ungültiger Statistikzeitraum');
if (!this.ready) throw new Error('Statistik noch nicht verfügbar');
this.activity(this.activeSince !== null);
const total = empty();
if (range === 'session') merge(total, this.session);
else if (range === 'all') merge(total, this.state.total);
else if (range === 'last24') {
const cutoff = Math.floor((this.now() - DAY) / MINUTE);
for (const [minute, bucket] of Object.entries(this.state.minutes)) if (Number(minute) > cutoff && Number(minute) <= Math.floor(this.now() / MINUTE)) merge(total, bucket);
} else {
const date = new Date(this.now());
date.setDate(date.getDate() - (range === 'month' ? 29 : range === 'week' ? 6 : 0));
const start = dayKey(date.getTime());
const end = dayKey(this.now());
for (const [day, bucket] of Object.entries(this.state.days)) if (day >= start && day <= end) merge(total, bucket);
}
const outcomes = total.completed + total.failed;
return { ...total, range, startedAt: range === 'session' ? this.sessionStartedAt : this.state.startedAt,
historyImported: range !== 'session' && this.state.historyImported,
successRate: outcomes ? total.completed / outcomes * 100 : null,
averageSpeedBps: total.activeMs > 0 ? total.trackedBytes * 1000 / total.activeMs : null,
persistenceError: this.error,
providers: Object.entries(HOSTERS).map(([id, label]) => ({ id, label, ...(total.providers[id] || { bytes: 0, completed: 0, failed: 0 }) })) };
}
async reset(scope) {
if (!['session', 'all', 'errors'].includes(scope)) throw new Error('Ungültiger Statistikreset');
if (!this.ready) throw new Error('Statistik noch nicht verfügbar');
this.activity(this.activeSince !== null);
if (scope === 'session' || scope === 'all') { this.session = empty(); this.sessionStartedAt = this.now(); }
if (scope === 'all') this.state = { version: 1, startedAt: this.now(), historyImported: false, total: empty(), days: {}, minutes: {} };
if (scope === 'errors') {
for (const bucket of [this.session, this.state.total, ...Object.values(this.state.days), ...Object.values(this.state.minutes)]) {
bucket.failed = 0;
for (const provider of Object.values(bucket.providers)) provider.failed = 0;
}
}
this.revision++;
await this.flush();
}
flush() {
const task = this.writeChain.catch(() => {}).then(async () => {
if (!this.ready || this.savedRevision === this.revision) return;
const cutoff = Math.floor((this.now() - 2 * DAY) / MINUTE);
for (const minute of Object.keys(this.state.minutes)) if (Number(minute) < cutoff) delete this.state.minutes[minute];
const revision = this.revision;
const data = JSON.stringify(this.state);
await fs.mkdir(path.dirname(this.filePath), { recursive: true });
const handle = await fs.open(this.filePath + '.tmp', 'w');
try { await handle.writeFile(data, 'utf8'); await handle.sync(); } finally { await handle.close(); }
try {
const previous = await fs.readFile(this.filePath, 'utf8');
if (validState(JSON.parse(previous))) await fs.writeFile(this.filePath + '.bak', previous, 'utf8');
} catch (error) { if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error; }
await fs.rename(this.filePath + '.tmp', this.filePath);
this.savedRevision = revision;
this.error = '';
});
this.writeChain = task;
return task.catch(error => { this.error = 'Statistik konnte nicht gespeichert werden'; throw error; });
}
}
module.exports = { UploadStatistics, HOSTERS };