feat(statistics): record rolling account traffic
This commit is contained in:
@@ -4,9 +4,13 @@ import path from "node:path";
|
||||
import { getProviderUsageDayKey } from "../shared/provider-daily-limits";
|
||||
import type {
|
||||
DebridProvider,
|
||||
StatisticsAccountMinuteUsage,
|
||||
StatisticsAccountUsage,
|
||||
StatisticsDayBucket,
|
||||
StatisticsLedger,
|
||||
StatisticsProviderBucket
|
||||
StatisticsMinuteBucket,
|
||||
StatisticsProviderBucket,
|
||||
StatisticsRolling24Hours
|
||||
} from "../shared/types";
|
||||
export { aggregateStatisticsRange } from "../shared/statistics-aggregation";
|
||||
|
||||
@@ -24,6 +28,24 @@ const providers = new Set<DebridProvider>([
|
||||
]);
|
||||
|
||||
const renameRetryDelaysMs = [15, 40, 90];
|
||||
const minuteMs = 60_000;
|
||||
const rollingWindowMs = 24 * 60 * minuteMs;
|
||||
const retainedMinuteCount = 48 * 60;
|
||||
const maximumAccountIdLength = 128;
|
||||
const maximumAccountLabelLength = 96;
|
||||
|
||||
const providerLabels: Record<DebridProvider, string> = {
|
||||
realdebrid: "Real-Debrid",
|
||||
megadebrid: "Mega-Debrid",
|
||||
"megadebrid-api": "Mega-Debrid API",
|
||||
"megadebrid-web": "Mega-Debrid Web",
|
||||
bestdebrid: "BestDebrid",
|
||||
alldebrid: "AllDebrid",
|
||||
ddownload: "DDownload",
|
||||
onefichier: "1Fichier",
|
||||
debridlink: "Debrid-Link",
|
||||
linksnappy: "LinkSnappy"
|
||||
};
|
||||
|
||||
function finiteNonNegative(value: unknown): number {
|
||||
const number = Number(value);
|
||||
@@ -36,6 +58,97 @@ function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
: null;
|
||||
}
|
||||
|
||||
function minuteStart(epochMs: number): number {
|
||||
return Math.floor(finiteNonNegative(epochMs) / minuteMs) * minuteMs;
|
||||
}
|
||||
|
||||
function validAccountId(value: unknown): string | null {
|
||||
const id = String(value || "").trim();
|
||||
return id.length > 0
|
||||
&& id.length <= maximumAccountIdLength
|
||||
&& /^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(id)
|
||||
? id
|
||||
: null;
|
||||
}
|
||||
|
||||
function maskEmailLikeLabel(value: string): string {
|
||||
const match = /^([^@\s]+)@([^@\s]+)$/.exec(value);
|
||||
if (!match) {
|
||||
return value;
|
||||
}
|
||||
const local = match[1];
|
||||
const hidden = "*".repeat(Math.max(3, Math.min(8, local.length - 1)));
|
||||
return `${local.slice(0, 1)}${hidden}@${match[2]}`;
|
||||
}
|
||||
|
||||
function safeAccountLabel(value: unknown, provider: DebridProvider): string {
|
||||
const clean = String(value || "")
|
||||
.replace(/[\u0000-\u001f\u007f]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, maximumAccountLabelLength);
|
||||
return maskEmailLikeLabel(clean || providerLabels[provider]);
|
||||
}
|
||||
|
||||
function fallbackAccountId(provider: DebridProvider): string {
|
||||
return `provider:${provider}`;
|
||||
}
|
||||
|
||||
function normalizeMinuteAccounts(value: unknown): Record<string, StatisticsAccountMinuteUsage> {
|
||||
const accounts: Record<string, StatisticsAccountMinuteUsage> = {};
|
||||
for (const [rawId, rawUsage] of Object.entries(asRecord(value) ?? {})) {
|
||||
const id = validAccountId(rawId);
|
||||
const usage = asRecord(rawUsage);
|
||||
const provider = String(usage?.provider || "") as DebridProvider;
|
||||
const bytes = finiteNonNegative(usage?.bytes);
|
||||
if (!id || !providers.has(provider) || bytes <= 0) {
|
||||
continue;
|
||||
}
|
||||
accounts[id] = {
|
||||
provider,
|
||||
label: safeAccountLabel(usage?.label, provider),
|
||||
bytes
|
||||
};
|
||||
}
|
||||
return accounts;
|
||||
}
|
||||
|
||||
function minimumRetainedMinute(now: number): number {
|
||||
return minuteStart(now) - ((retainedMinuteCount - 1) * minuteMs);
|
||||
}
|
||||
|
||||
function normalizeMinutes(value: unknown, now: number): StatisticsMinuteBucket[] {
|
||||
const currentMinute = minuteStart(now);
|
||||
const minimumMinute = minimumRetainedMinute(now);
|
||||
const buckets = new Map<number, StatisticsMinuteBucket>();
|
||||
for (const rawValue of Array.isArray(value) ? value : []) {
|
||||
const record = asRecord(rawValue);
|
||||
const minute = minuteStart(Number(record?.minute));
|
||||
if (!record || minute < minimumMinute || minute > currentMinute) {
|
||||
continue;
|
||||
}
|
||||
const accounts = normalizeMinuteAccounts(record.accounts);
|
||||
if (Object.keys(accounts).length === 0) {
|
||||
continue;
|
||||
}
|
||||
const target = buckets.get(minute) ?? { minute, downloadedBytes: 0, accounts: {} };
|
||||
for (const [id, usage] of Object.entries(accounts)) {
|
||||
const existing = target.accounts[id];
|
||||
if (existing && existing.provider === usage.provider) {
|
||||
existing.bytes += usage.bytes;
|
||||
existing.label = usage.label;
|
||||
} else {
|
||||
target.accounts[id] = { ...usage };
|
||||
}
|
||||
}
|
||||
target.downloadedBytes = Object.values(target.accounts).reduce((total, usage) => total + usage.bytes, 0);
|
||||
buckets.set(minute, target);
|
||||
}
|
||||
return [...buckets.values()]
|
||||
.sort((left, right) => left.minute - right.minute)
|
||||
.slice(-retainedMinuteCount);
|
||||
}
|
||||
|
||||
function emptyProviderBucket(): StatisticsProviderBucket {
|
||||
return { bytes: 0, completed: 0, failed: 0 };
|
||||
}
|
||||
@@ -86,7 +199,7 @@ function normalizeDay(value: unknown): StatisticsDayBucket | null {
|
||||
}
|
||||
|
||||
export function createStatisticsLedger(now = Date.now()): StatisticsLedger {
|
||||
return { version: 1, startedAt: now, days: [] };
|
||||
return { version: 2, startedAt: now, days: [], minutes: [] };
|
||||
}
|
||||
|
||||
export function normalizeStatisticsLedger(value: unknown, now = Date.now()): StatisticsLedger {
|
||||
@@ -102,12 +215,161 @@ export function normalizeStatisticsLedger(value: unknown, now = Date.now()): Sta
|
||||
}
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
version: 2,
|
||||
startedAt: finiteNonNegative(record.startedAt) || now,
|
||||
days: [...byDay.values()].sort((left, right) => left.day.localeCompare(right.day))
|
||||
days: [...byDay.values()].sort((left, right) => left.day.localeCompare(right.day)),
|
||||
minutes: normalizeMinutes(record.minutes, now)
|
||||
};
|
||||
}
|
||||
|
||||
export function projectStatisticsLedger(ledger: StatisticsLedger, now = Date.now()): StatisticsLedger {
|
||||
return normalizeStatisticsLedger({ ...ledger, minutes: [] }, now);
|
||||
}
|
||||
|
||||
export function addStatisticsAccountBytesInPlace(
|
||||
ledger: StatisticsLedger,
|
||||
provider: DebridProvider,
|
||||
byteDelta: number,
|
||||
accountId?: string,
|
||||
accountLabel?: string,
|
||||
epochMs = Date.now()
|
||||
): StatisticsAccountUsage | null {
|
||||
const bytes = finiteNonNegative(byteDelta);
|
||||
if (bytes <= 0 || !providers.has(provider)) {
|
||||
return null;
|
||||
}
|
||||
const minute = minuteStart(epochMs);
|
||||
const id = validAccountId(accountId) ?? fallbackAccountId(provider);
|
||||
const label = safeAccountLabel(accountLabel, provider);
|
||||
let bucket: StatisticsMinuteBucket | undefined = ledger.minutes[ledger.minutes.length - 1];
|
||||
if (!bucket || bucket.minute !== minute) {
|
||||
ledger.minutes = ledger.minutes.filter((entry) => entry.minute >= minimumRetainedMinute(epochMs) && entry.minute <= minute);
|
||||
bucket = ledger.minutes.find((entry) => entry.minute === minute);
|
||||
if (!bucket) {
|
||||
bucket = { minute, downloadedBytes: 0, accounts: {} };
|
||||
ledger.minutes.push(bucket);
|
||||
ledger.minutes.sort((left, right) => left.minute - right.minute);
|
||||
if (ledger.minutes.length > retainedMinuteCount) {
|
||||
ledger.minutes.splice(0, ledger.minutes.length - retainedMinuteCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!bucket) {
|
||||
return null;
|
||||
}
|
||||
const existing = bucket.accounts[id];
|
||||
if (existing && existing.provider === provider) {
|
||||
existing.bytes += bytes;
|
||||
existing.label = label;
|
||||
} else {
|
||||
bucket.accounts[id] = { provider, label, bytes };
|
||||
}
|
||||
bucket.downloadedBytes = Object.values(bucket.accounts).reduce((total, usage) => total + usage.bytes, 0);
|
||||
return { id, provider, label, bytes };
|
||||
}
|
||||
|
||||
function aggregateRollingAccountStatistics(ledger: StatisticsLedger, now: number): StatisticsRolling24Hours {
|
||||
const currentMinute = minuteStart(now);
|
||||
const from = minuteStart(now - rollingWindowMs);
|
||||
const accounts = new Map<string, StatisticsAccountUsage>();
|
||||
for (const bucket of ledger.minutes) {
|
||||
if (bucket.minute < from || bucket.minute > currentMinute) {
|
||||
continue;
|
||||
}
|
||||
for (const [id, usage] of Object.entries(bucket.accounts)) {
|
||||
const existing = accounts.get(id);
|
||||
if (existing && existing.provider === usage.provider) {
|
||||
existing.bytes += usage.bytes;
|
||||
existing.label = usage.label;
|
||||
} else {
|
||||
accounts.set(id, { id, provider: usage.provider, label: usage.label, bytes: usage.bytes });
|
||||
}
|
||||
}
|
||||
}
|
||||
const rows = [...accounts.values()].sort((left, right) =>
|
||||
right.bytes - left.bytes
|
||||
|| left.label.localeCompare(right.label, "de-DE", { sensitivity: "base" })
|
||||
|| left.id.localeCompare(right.id)
|
||||
);
|
||||
return {
|
||||
from,
|
||||
to: now,
|
||||
downloadedBytes: rows.reduce((total, account) => total + account.bytes, 0),
|
||||
accounts: rows
|
||||
};
|
||||
}
|
||||
|
||||
export class RollingAccountStatisticsAccumulator {
|
||||
private ledger: StatisticsLedger;
|
||||
private minute = -1;
|
||||
private aggregate: StatisticsRolling24Hours;
|
||||
|
||||
public constructor(ledger: StatisticsLedger, now = Date.now()) {
|
||||
this.ledger = ledger;
|
||||
this.aggregate = aggregateRollingAccountStatistics(ledger, now);
|
||||
this.minute = minuteStart(now);
|
||||
}
|
||||
|
||||
public record(
|
||||
provider: DebridProvider,
|
||||
byteDelta: number,
|
||||
accountId?: string,
|
||||
accountLabel?: string,
|
||||
epochMs = Date.now()
|
||||
): void {
|
||||
this.refresh(epochMs);
|
||||
const recorded = addStatisticsAccountBytesInPlace(
|
||||
this.ledger,
|
||||
provider,
|
||||
byteDelta,
|
||||
accountId,
|
||||
accountLabel,
|
||||
epochMs
|
||||
);
|
||||
if (!recorded || minuteStart(epochMs) < this.aggregate.from || minuteStart(epochMs) > this.minute) {
|
||||
return;
|
||||
}
|
||||
const existing = this.aggregate.accounts.find((account) => account.id === recorded.id && account.provider === recorded.provider);
|
||||
if (existing) {
|
||||
existing.bytes += recorded.bytes;
|
||||
existing.label = recorded.label;
|
||||
} else {
|
||||
this.aggregate.accounts.push({ ...recorded });
|
||||
}
|
||||
this.aggregate.downloadedBytes += recorded.bytes;
|
||||
this.aggregate.accounts.sort((left, right) =>
|
||||
right.bytes - left.bytes
|
||||
|| left.label.localeCompare(right.label, "de-DE", { sensitivity: "base" })
|
||||
|| left.id.localeCompare(right.id)
|
||||
);
|
||||
this.aggregate.to = epochMs;
|
||||
}
|
||||
|
||||
public snapshot(now = Date.now()): StatisticsRolling24Hours {
|
||||
this.refresh(now);
|
||||
return {
|
||||
...this.aggregate,
|
||||
to: now,
|
||||
accounts: this.aggregate.accounts.map((account) => ({ ...account }))
|
||||
};
|
||||
}
|
||||
|
||||
public reset(ledger: StatisticsLedger, now = Date.now()): void {
|
||||
this.ledger = ledger;
|
||||
this.minute = minuteStart(now);
|
||||
this.aggregate = aggregateRollingAccountStatistics(ledger, now);
|
||||
}
|
||||
|
||||
private refresh(now: number): void {
|
||||
const currentMinute = minuteStart(now);
|
||||
if (currentMinute === this.minute) {
|
||||
return;
|
||||
}
|
||||
this.minute = currentMinute;
|
||||
this.aggregate = aggregateRollingAccountStatistics(this.ledger, now);
|
||||
}
|
||||
}
|
||||
|
||||
function updateDay(
|
||||
ledger: StatisticsLedger,
|
||||
epochMs: number,
|
||||
|
||||
+29
-1
@@ -57,10 +57,37 @@ export interface StatisticsDayBucket {
|
||||
providers: Partial<Record<DebridProvider, StatisticsProviderBucket>>;
|
||||
}
|
||||
|
||||
export interface StatisticsAccountMinuteUsage {
|
||||
provider: DebridProvider;
|
||||
label: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface StatisticsMinuteBucket {
|
||||
minute: number;
|
||||
downloadedBytes: number;
|
||||
accounts: Record<string, StatisticsAccountMinuteUsage>;
|
||||
}
|
||||
|
||||
export interface StatisticsAccountUsage {
|
||||
id: string;
|
||||
provider: DebridProvider;
|
||||
label: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface StatisticsRolling24Hours {
|
||||
from: number;
|
||||
to: number;
|
||||
downloadedBytes: number;
|
||||
accounts: StatisticsAccountUsage[];
|
||||
}
|
||||
|
||||
export interface StatisticsLedger {
|
||||
version: 1;
|
||||
version: 2;
|
||||
startedAt: number;
|
||||
days: StatisticsDayBucket[];
|
||||
minutes: StatisticsMinuteBucket[];
|
||||
}
|
||||
|
||||
export interface DownloadStats {
|
||||
@@ -76,6 +103,7 @@ export interface DownloadStats {
|
||||
totalRuntimeMs: number;
|
||||
runtimeMeasuredAt: number;
|
||||
statistics?: StatisticsLedger;
|
||||
rolling24Hours?: StatisticsRolling24Hours;
|
||||
}
|
||||
|
||||
export interface DebridAccountStatus {
|
||||
|
||||
@@ -12,7 +12,7 @@ const session: SessionState = {
|
||||
reconnectReason: "", paused: false, running: true, updatedAt: 0
|
||||
};
|
||||
const history: HistoryEntry[] = [{ id: "h1" } as unknown as HistoryEntry];
|
||||
const statistics: StatisticsLedger = { version: 1, startedAt: 1, days: [] };
|
||||
const statistics: StatisticsLedger = { version: 2, startedAt: 1, days: [], minutes: [] };
|
||||
|
||||
const baseInput = { appVersion: "1.7.183", exportedAt: "2026-06-07T00:00:00Z", session, history, statistics };
|
||||
|
||||
|
||||
@@ -6,9 +6,13 @@ import { defaultSettings } from "../src/main/constants";
|
||||
import { DownloadManager } from "../src/main/download-manager";
|
||||
import { createStoragePaths, emptySession } from "../src/main/storage";
|
||||
import {
|
||||
RollingAccountStatisticsAccumulator,
|
||||
addStatisticsAccountBytesInPlace,
|
||||
aggregateStatisticsRange,
|
||||
createStatisticsLedger,
|
||||
loadStatisticsLedger,
|
||||
normalizeStatisticsLedger,
|
||||
projectStatisticsLedger,
|
||||
recordStatisticsActiveInterval,
|
||||
recordStatisticsBytes,
|
||||
recordStatisticsOutcome,
|
||||
@@ -28,6 +32,119 @@ function localTime(day: number, hour = 12): number {
|
||||
}
|
||||
|
||||
describe("statistics ledger", () => {
|
||||
it("migrates version one ledgers without inventing minute history", () => {
|
||||
const now = localTime(10);
|
||||
const legacy = {
|
||||
version: 1,
|
||||
startedAt: now - 1_000,
|
||||
days: [{
|
||||
day: "2026-08-10",
|
||||
downloadedBytes: 1_024,
|
||||
measuredBytes: 1_024,
|
||||
completedFiles: 1,
|
||||
failedFiles: 0,
|
||||
activeDownloadMs: 100,
|
||||
providers: { realdebrid: { bytes: 1_024, completed: 1, failed: 0 } }
|
||||
}]
|
||||
};
|
||||
|
||||
const migrated = normalizeStatisticsLedger(legacy, now);
|
||||
|
||||
expect(migrated.version).toBe(2);
|
||||
expect(migrated.days).toEqual(legacy.days);
|
||||
expect(migrated.minutes).toEqual([]);
|
||||
});
|
||||
|
||||
it("normalizes, merges, and bounds sparse account minute history", () => {
|
||||
const now = localTime(10);
|
||||
const currentMinute = Math.floor(now / 60_000) * 60_000;
|
||||
const keptMinute = currentMinute - (47 * 60 * 60 * 1_000);
|
||||
const raw = {
|
||||
version: 2,
|
||||
startedAt: now - 1_000,
|
||||
days: [],
|
||||
minutes: [
|
||||
{
|
||||
minute: keptMinute,
|
||||
downloadedBytes: 1,
|
||||
accounts: {
|
||||
rdw_one: { provider: "realdebrid", label: "secret@example.test", bytes: 20 }
|
||||
}
|
||||
},
|
||||
{
|
||||
minute: keptMinute,
|
||||
downloadedBytes: 1,
|
||||
accounts: {
|
||||
rdw_one: { provider: "realdebrid", label: "New label", bytes: 30 },
|
||||
"https://unsafe": { provider: "realdebrid", label: "Unsafe", bytes: 99 }
|
||||
}
|
||||
},
|
||||
{
|
||||
minute: currentMinute - (49 * 60 * 60 * 1_000),
|
||||
accounts: { rdw_old: { provider: "realdebrid", label: "Old", bytes: 500 } }
|
||||
},
|
||||
{
|
||||
minute: currentMinute + 120_000,
|
||||
accounts: { rdw_future: { provider: "realdebrid", label: "Future", bytes: 500 } }
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
const normalized = normalizeStatisticsLedger(raw, now);
|
||||
|
||||
expect(normalized.minutes).toHaveLength(1);
|
||||
expect(normalized.minutes[0].minute).toBe(keptMinute);
|
||||
expect(normalized.minutes[0].downloadedBytes).toBe(50);
|
||||
expect(normalized.minutes[0].accounts.rdw_one.bytes).toBe(50);
|
||||
expect(normalized.minutes[0].accounts.rdw_one.label).not.toContain("secret@example.test");
|
||||
expect(normalized.minutes[0].accounts).not.toHaveProperty("https://unsafe");
|
||||
});
|
||||
|
||||
it("records separate accounts and projects no minute history into renderer snapshots", () => {
|
||||
const now = localTime(10);
|
||||
const ledger = createStatisticsLedger(now);
|
||||
|
||||
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 50, "rdw_one", "Primary", now);
|
||||
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 75, "rdw_two", "Secondary", now);
|
||||
addStatisticsAccountBytesInPlace(ledger, "debridlink", 25, undefined, undefined, now);
|
||||
|
||||
expect(ledger.minutes).toHaveLength(1);
|
||||
expect(ledger.minutes[0].downloadedBytes).toBe(150);
|
||||
expect(ledger.minutes[0].accounts).toMatchObject({
|
||||
rdw_one: { provider: "realdebrid", label: "Primary", bytes: 50 },
|
||||
rdw_two: { provider: "realdebrid", label: "Secondary", bytes: 75 },
|
||||
"provider:debridlink": { provider: "debridlink", label: "Debrid-Link", bytes: 25 }
|
||||
});
|
||||
expect(projectStatisticsLedger(ledger, now).minutes).toEqual([]);
|
||||
expect(ledger.minutes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("maintains rolling account totals incrementally and expires the boundary minute", () => {
|
||||
const now = localTime(10);
|
||||
const oldMinute = now - (24 * 60 * 60 * 1_000) - 60_000;
|
||||
const boundaryMinute = Math.floor((now - (24 * 60 * 60 * 1_000)) / 60_000) * 60_000;
|
||||
const ledger = createStatisticsLedger(now);
|
||||
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 500, "rdw_old", "Old", oldMinute);
|
||||
addStatisticsAccountBytesInPlace(ledger, "realdebrid", 100, "rdw_boundary", "Boundary", boundaryMinute);
|
||||
const accumulator = new RollingAccountStatisticsAccumulator(ledger, now);
|
||||
|
||||
accumulator.record("realdebrid", 50, "rdw_one", "Primary", now);
|
||||
accumulator.record("realdebrid", 75, "rdw_two", "Secondary", now);
|
||||
const current = accumulator.snapshot(now);
|
||||
|
||||
expect(current.downloadedBytes).toBe(225);
|
||||
expect(current.accounts.map((account) => [account.id, account.bytes])).toEqual([
|
||||
["rdw_boundary", 100],
|
||||
["rdw_two", 75],
|
||||
["rdw_one", 50]
|
||||
]);
|
||||
expect(current.accounts).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: "rdw_old" })]));
|
||||
|
||||
const expired = accumulator.snapshot(now + 60_000);
|
||||
expect(expired.downloadedBytes).toBe(125);
|
||||
expect(expired.accounts).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: "rdw_boundary" })]));
|
||||
});
|
||||
|
||||
it("aggregates every available day inside a rolling seven-day window without requiring seven complete days", () => {
|
||||
let ledger = createStatisticsLedger(localTime(10));
|
||||
ledger = recordStatisticsBytes(ledger, "realdebrid", 100, localTime(7));
|
||||
|
||||
@@ -467,8 +467,9 @@ function createDenseSnapshot(): UiSnapshot {
|
||||
totalRuntimeMs: 172800000,
|
||||
runtimeMeasuredAt: 1786312800000,
|
||||
statistics: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
startedAt: 1786053600000,
|
||||
minutes: [],
|
||||
days: [
|
||||
{
|
||||
day: "2026-08-08",
|
||||
|
||||
Reference in New Issue
Block a user