feat(statistics): record rolling account traffic

This commit is contained in:
Sucukdeluxe
2026-08-21 08:25:32 +02:00
parent 54bcd82cb6
commit 7121a334d2
5 changed files with 415 additions and 7 deletions
+266 -4
View File
@@ -4,9 +4,13 @@ import path from "node:path";
import { getProviderUsageDayKey } from "../shared/provider-daily-limits";
import type {
DebridProvider,
StatisticsAccountMinuteUsage,
StatisticsAccountUsage,
StatisticsDayBucket,
StatisticsLedger,
StatisticsProviderBucket
StatisticsMinuteBucket,
StatisticsProviderBucket,
StatisticsRolling24Hours
} from "../shared/types";
export { aggregateStatisticsRange } from "../shared/statistics-aggregation";
@@ -24,6 +28,24 @@ const providers = new Set<DebridProvider>([
]);
const renameRetryDelaysMs = [15, 40, 90];
const minuteMs = 60_000;
const rollingWindowMs = 24 * 60 * minuteMs;
const retainedMinuteCount = 48 * 60;
const maximumAccountIdLength = 128;
const maximumAccountLabelLength = 96;
const providerLabels: Record<DebridProvider, string> = {
realdebrid: "Real-Debrid",
megadebrid: "Mega-Debrid",
"megadebrid-api": "Mega-Debrid API",
"megadebrid-web": "Mega-Debrid Web",
bestdebrid: "BestDebrid",
alldebrid: "AllDebrid",
ddownload: "DDownload",
onefichier: "1Fichier",
debridlink: "Debrid-Link",
linksnappy: "LinkSnappy"
};
function finiteNonNegative(value: unknown): number {
const number = Number(value);
@@ -36,6 +58,97 @@ function asRecord(value: unknown): Record<string, unknown> | null {
: null;
}
function minuteStart(epochMs: number): number {
return Math.floor(finiteNonNegative(epochMs) / minuteMs) * minuteMs;
}
function validAccountId(value: unknown): string | null {
const id = String(value || "").trim();
return id.length > 0
&& id.length <= maximumAccountIdLength
&& /^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(id)
? id
: null;
}
function maskEmailLikeLabel(value: string): string {
const match = /^([^@\s]+)@([^@\s]+)$/.exec(value);
if (!match) {
return value;
}
const local = match[1];
const hidden = "*".repeat(Math.max(3, Math.min(8, local.length - 1)));
return `${local.slice(0, 1)}${hidden}@${match[2]}`;
}
function safeAccountLabel(value: unknown, provider: DebridProvider): string {
const clean = String(value || "")
.replace(/[\u0000-\u001f\u007f]/g, " ")
.replace(/\s+/g, " ")
.trim()
.slice(0, maximumAccountLabelLength);
return maskEmailLikeLabel(clean || providerLabels[provider]);
}
function fallbackAccountId(provider: DebridProvider): string {
return `provider:${provider}`;
}
function normalizeMinuteAccounts(value: unknown): Record<string, StatisticsAccountMinuteUsage> {
const accounts: Record<string, StatisticsAccountMinuteUsage> = {};
for (const [rawId, rawUsage] of Object.entries(asRecord(value) ?? {})) {
const id = validAccountId(rawId);
const usage = asRecord(rawUsage);
const provider = String(usage?.provider || "") as DebridProvider;
const bytes = finiteNonNegative(usage?.bytes);
if (!id || !providers.has(provider) || bytes <= 0) {
continue;
}
accounts[id] = {
provider,
label: safeAccountLabel(usage?.label, provider),
bytes
};
}
return accounts;
}
function minimumRetainedMinute(now: number): number {
return minuteStart(now) - ((retainedMinuteCount - 1) * minuteMs);
}
function normalizeMinutes(value: unknown, now: number): StatisticsMinuteBucket[] {
const currentMinute = minuteStart(now);
const minimumMinute = minimumRetainedMinute(now);
const buckets = new Map<number, StatisticsMinuteBucket>();
for (const rawValue of Array.isArray(value) ? value : []) {
const record = asRecord(rawValue);
const minute = minuteStart(Number(record?.minute));
if (!record || minute < minimumMinute || minute > currentMinute) {
continue;
}
const accounts = normalizeMinuteAccounts(record.accounts);
if (Object.keys(accounts).length === 0) {
continue;
}
const target = buckets.get(minute) ?? { minute, downloadedBytes: 0, accounts: {} };
for (const [id, usage] of Object.entries(accounts)) {
const existing = target.accounts[id];
if (existing && existing.provider === usage.provider) {
existing.bytes += usage.bytes;
existing.label = usage.label;
} else {
target.accounts[id] = { ...usage };
}
}
target.downloadedBytes = Object.values(target.accounts).reduce((total, usage) => total + usage.bytes, 0);
buckets.set(minute, target);
}
return [...buckets.values()]
.sort((left, right) => left.minute - right.minute)
.slice(-retainedMinuteCount);
}
function emptyProviderBucket(): StatisticsProviderBucket {
return { bytes: 0, completed: 0, failed: 0 };
}
@@ -86,7 +199,7 @@ function normalizeDay(value: unknown): StatisticsDayBucket | null {
}
export function createStatisticsLedger(now = Date.now()): StatisticsLedger {
return { version: 1, startedAt: now, days: [] };
return { version: 2, startedAt: now, days: [], minutes: [] };
}
export function normalizeStatisticsLedger(value: unknown, now = Date.now()): StatisticsLedger {
@@ -102,12 +215,161 @@ export function normalizeStatisticsLedger(value: unknown, now = Date.now()): Sta
}
}
return {
version: 1,
version: 2,
startedAt: finiteNonNegative(record.startedAt) || now,
days: [...byDay.values()].sort((left, right) => left.day.localeCompare(right.day))
days: [...byDay.values()].sort((left, right) => left.day.localeCompare(right.day)),
minutes: normalizeMinutes(record.minutes, now)
};
}
export function projectStatisticsLedger(ledger: StatisticsLedger, now = Date.now()): StatisticsLedger {
return normalizeStatisticsLedger({ ...ledger, minutes: [] }, now);
}
export function addStatisticsAccountBytesInPlace(
ledger: StatisticsLedger,
provider: DebridProvider,
byteDelta: number,
accountId?: string,
accountLabel?: string,
epochMs = Date.now()
): StatisticsAccountUsage | null {
const bytes = finiteNonNegative(byteDelta);
if (bytes <= 0 || !providers.has(provider)) {
return null;
}
const minute = minuteStart(epochMs);
const id = validAccountId(accountId) ?? fallbackAccountId(provider);
const label = safeAccountLabel(accountLabel, provider);
let bucket: StatisticsMinuteBucket | undefined = ledger.minutes[ledger.minutes.length - 1];
if (!bucket || bucket.minute !== minute) {
ledger.minutes = ledger.minutes.filter((entry) => entry.minute >= minimumRetainedMinute(epochMs) && entry.minute <= minute);
bucket = ledger.minutes.find((entry) => entry.minute === minute);
if (!bucket) {
bucket = { minute, downloadedBytes: 0, accounts: {} };
ledger.minutes.push(bucket);
ledger.minutes.sort((left, right) => left.minute - right.minute);
if (ledger.minutes.length > retainedMinuteCount) {
ledger.minutes.splice(0, ledger.minutes.length - retainedMinuteCount);
}
}
}
if (!bucket) {
return null;
}
const existing = bucket.accounts[id];
if (existing && existing.provider === provider) {
existing.bytes += bytes;
existing.label = label;
} else {
bucket.accounts[id] = { provider, label, bytes };
}
bucket.downloadedBytes = Object.values(bucket.accounts).reduce((total, usage) => total + usage.bytes, 0);
return { id, provider, label, bytes };
}
function aggregateRollingAccountStatistics(ledger: StatisticsLedger, now: number): StatisticsRolling24Hours {
const currentMinute = minuteStart(now);
const from = minuteStart(now - rollingWindowMs);
const accounts = new Map<string, StatisticsAccountUsage>();
for (const bucket of ledger.minutes) {
if (bucket.minute < from || bucket.minute > currentMinute) {
continue;
}
for (const [id, usage] of Object.entries(bucket.accounts)) {
const existing = accounts.get(id);
if (existing && existing.provider === usage.provider) {
existing.bytes += usage.bytes;
existing.label = usage.label;
} else {
accounts.set(id, { id, provider: usage.provider, label: usage.label, bytes: usage.bytes });
}
}
}
const rows = [...accounts.values()].sort((left, right) =>
right.bytes - left.bytes
|| left.label.localeCompare(right.label, "de-DE", { sensitivity: "base" })
|| left.id.localeCompare(right.id)
);
return {
from,
to: now,
downloadedBytes: rows.reduce((total, account) => total + account.bytes, 0),
accounts: rows
};
}
export class RollingAccountStatisticsAccumulator {
private ledger: StatisticsLedger;
private minute = -1;
private aggregate: StatisticsRolling24Hours;
public constructor(ledger: StatisticsLedger, now = Date.now()) {
this.ledger = ledger;
this.aggregate = aggregateRollingAccountStatistics(ledger, now);
this.minute = minuteStart(now);
}
public record(
provider: DebridProvider,
byteDelta: number,
accountId?: string,
accountLabel?: string,
epochMs = Date.now()
): void {
this.refresh(epochMs);
const recorded = addStatisticsAccountBytesInPlace(
this.ledger,
provider,
byteDelta,
accountId,
accountLabel,
epochMs
);
if (!recorded || minuteStart(epochMs) < this.aggregate.from || minuteStart(epochMs) > this.minute) {
return;
}
const existing = this.aggregate.accounts.find((account) => account.id === recorded.id && account.provider === recorded.provider);
if (existing) {
existing.bytes += recorded.bytes;
existing.label = recorded.label;
} else {
this.aggregate.accounts.push({ ...recorded });
}
this.aggregate.downloadedBytes += recorded.bytes;
this.aggregate.accounts.sort((left, right) =>
right.bytes - left.bytes
|| left.label.localeCompare(right.label, "de-DE", { sensitivity: "base" })
|| left.id.localeCompare(right.id)
);
this.aggregate.to = epochMs;
}
public snapshot(now = Date.now()): StatisticsRolling24Hours {
this.refresh(now);
return {
...this.aggregate,
to: now,
accounts: this.aggregate.accounts.map((account) => ({ ...account }))
};
}
public reset(ledger: StatisticsLedger, now = Date.now()): void {
this.ledger = ledger;
this.minute = minuteStart(now);
this.aggregate = aggregateRollingAccountStatistics(ledger, now);
}
private refresh(now: number): void {
const currentMinute = minuteStart(now);
if (currentMinute === this.minute) {
return;
}
this.minute = currentMinute;
this.aggregate = aggregateRollingAccountStatistics(this.ledger, now);
}
}
function updateDay(
ledger: StatisticsLedger,
epochMs: number,
+29 -1
View File
@@ -57,10 +57,37 @@ export interface StatisticsDayBucket {
providers: Partial<Record<DebridProvider, StatisticsProviderBucket>>;
}
export interface StatisticsAccountMinuteUsage {
provider: DebridProvider;
label: string;
bytes: number;
}
export interface StatisticsMinuteBucket {
minute: number;
downloadedBytes: number;
accounts: Record<string, StatisticsAccountMinuteUsage>;
}
export interface StatisticsAccountUsage {
id: string;
provider: DebridProvider;
label: string;
bytes: number;
}
export interface StatisticsRolling24Hours {
from: number;
to: number;
downloadedBytes: number;
accounts: StatisticsAccountUsage[];
}
export interface StatisticsLedger {
version: 1;
version: 2;
startedAt: number;
days: StatisticsDayBucket[];
minutes: StatisticsMinuteBucket[];
}
export interface DownloadStats {
@@ -76,6 +103,7 @@ export interface DownloadStats {
totalRuntimeMs: number;
runtimeMeasuredAt: number;
statistics?: StatisticsLedger;
rolling24Hours?: StatisticsRolling24Hours;
}
export interface DebridAccountStatus {
+1 -1
View File
@@ -12,7 +12,7 @@ const session: SessionState = {
reconnectReason: "", paused: false, running: true, updatedAt: 0
};
const history: HistoryEntry[] = [{ id: "h1" } as unknown as HistoryEntry];
const statistics: StatisticsLedger = { version: 1, startedAt: 1, days: [] };
const statistics: StatisticsLedger = { version: 2, startedAt: 1, days: [], minutes: [] };
const baseInput = { appVersion: "1.7.183", exportedAt: "2026-06-07T00:00:00Z", session, history, statistics };
+117
View File
@@ -6,9 +6,13 @@ import { defaultSettings } from "../src/main/constants";
import { DownloadManager } from "../src/main/download-manager";
import { createStoragePaths, emptySession } from "../src/main/storage";
import {
RollingAccountStatisticsAccumulator,
addStatisticsAccountBytesInPlace,
aggregateStatisticsRange,
createStatisticsLedger,
loadStatisticsLedger,
normalizeStatisticsLedger,
projectStatisticsLedger,
recordStatisticsActiveInterval,
recordStatisticsBytes,
recordStatisticsOutcome,
@@ -28,6 +32,119 @@ function localTime(day: number, hour = 12): number {
}
describe("statistics ledger", () => {
it("migrates version one ledgers without inventing minute history", () => {
const now = localTime(10);
const legacy = {
version: 1,
startedAt: now - 1_000,
days: [{
day: "2026-08-10",
downloadedBytes: 1_024,
measuredBytes: 1_024,
completedFiles: 1,
failedFiles: 0,
activeDownloadMs: 100,
providers: { realdebrid: { bytes: 1_024, completed: 1, failed: 0 } }
}]
};
const migrated = normalizeStatisticsLedger(legacy, now);
expect(migrated.version).toBe(2);
expect(migrated.days).toEqual(legacy.days);
expect(migrated.minutes).toEqual([]);
});
it("normalizes, merges, and bounds sparse account minute history", () => {
const now = localTime(10);
const currentMinute = Math.floor(now / 60_000) * 60_000;
const keptMinute = currentMinute - (47 * 60 * 60 * 1_000);
const raw = {
version: 2,
startedAt: now - 1_000,
days: [],
minutes: [
{
minute: keptMinute,
downloadedBytes: 1,
accounts: {
rdw_one: { provider: "realdebrid", label: "secret@example.test", bytes: 20 }
}
},
{
minute: keptMinute,
downloadedBytes: 1,
accounts: {
rdw_one: { provider: "realdebrid", label: "New label", bytes: 30 },
"https://unsafe": { provider: "realdebrid", label: "Unsafe", bytes: 99 }
}
},
{
minute: currentMinute - (49 * 60 * 60 * 1_000),
accounts: { rdw_old: { provider: "realdebrid", label: "Old", bytes: 500 } }
},
{
minute: currentMinute + 120_000,
accounts: { rdw_future: { provider: "realdebrid", label: "Future", bytes: 500 } }
}
]
};
const normalized = normalizeStatisticsLedger(raw, now);
expect(normalized.minutes).toHaveLength(1);
expect(normalized.minutes[0].minute).toBe(keptMinute);
expect(normalized.minutes[0].downloadedBytes).toBe(50);
expect(normalized.minutes[0].accounts.rdw_one.bytes).toBe(50);
expect(normalized.minutes[0].accounts.rdw_one.label).not.toContain("secret@example.test");
expect(normalized.minutes[0].accounts).not.toHaveProperty("https://unsafe");
});
it("records separate accounts and projects no minute history into renderer snapshots", () => {
const now = localTime(10);
const ledger = createStatisticsLedger(now);
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 50, "rdw_one", "Primary", now);
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 75, "rdw_two", "Secondary", now);
addStatisticsAccountBytesInPlace(ledger, "debridlink", 25, undefined, undefined, now);
expect(ledger.minutes).toHaveLength(1);
expect(ledger.minutes[0].downloadedBytes).toBe(150);
expect(ledger.minutes[0].accounts).toMatchObject({
rdw_one: { provider: "realdebrid", label: "Primary", bytes: 50 },
rdw_two: { provider: "realdebrid", label: "Secondary", bytes: 75 },
"provider:debridlink": { provider: "debridlink", label: "Debrid-Link", bytes: 25 }
});
expect(projectStatisticsLedger(ledger, now).minutes).toEqual([]);
expect(ledger.minutes).toHaveLength(1);
});
it("maintains rolling account totals incrementally and expires the boundary minute", () => {
const now = localTime(10);
const oldMinute = now - (24 * 60 * 60 * 1_000) - 60_000;
const boundaryMinute = Math.floor((now - (24 * 60 * 60 * 1_000)) / 60_000) * 60_000;
const ledger = createStatisticsLedger(now);
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 500, "rdw_old", "Old", oldMinute);
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 100, "rdw_boundary", "Boundary", boundaryMinute);
const accumulator = new RollingAccountStatisticsAccumulator(ledger, now);
accumulator.record("realdebrid", 50, "rdw_one", "Primary", now);
accumulator.record("realdebrid", 75, "rdw_two", "Secondary", now);
const current = accumulator.snapshot(now);
expect(current.downloadedBytes).toBe(225);
expect(current.accounts.map((account) => [account.id, account.bytes])).toEqual([
["rdw_boundary", 100],
["rdw_two", 75],
["rdw_one", 50]
]);
expect(current.accounts).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: "rdw_old" })]));
const expired = accumulator.snapshot(now + 60_000);
expect(expired.downloadedBytes).toBe(125);
expect(expired.accounts).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: "rdw_boundary" })]));
});
it("aggregates every available day inside a rolling seven-day window without requiring seven complete days", () => {
let ledger = createStatisticsLedger(localTime(10));
ledger = recordStatisticsBytes(ledger, "realdebrid", 100, localTime(7));
+2 -1
View File
@@ -467,8 +467,9 @@ function createDenseSnapshot(): UiSnapshot {
totalRuntimeMs: 172800000,
runtimeMeasuredAt: 1786312800000,
statistics: {
version: 1,
version: 2,
startedAt: 1786053600000,
minutes: [],
days: [
{
day: "2026-08-08",