From 7121a334d27977fddf90299ccf632637201f153e Mon Sep 17 00:00:00 2001 From: Sucukdeluxe Date: Fri, 21 Aug 2026 08:25:32 +0200 Subject: [PATCH] feat(statistics): record rolling account traffic --- src/main/statistics-ledger.ts | 270 +++++++++++++++++++++++++++++++- src/shared/types.ts | 30 +++- tests/backup-payload.test.ts | 2 +- tests/statistics-ledger.test.ts | 117 ++++++++++++++ tests/visual/fixtures.ts | 3 +- 5 files changed, 415 insertions(+), 7 deletions(-) diff --git a/src/main/statistics-ledger.ts b/src/main/statistics-ledger.ts index ff696f7..1478fc2 100644 --- a/src/main/statistics-ledger.ts +++ b/src/main/statistics-ledger.ts @@ -4,9 +4,13 @@ import path from "node:path"; import { getProviderUsageDayKey } from "../shared/provider-daily-limits"; import type { DebridProvider, + StatisticsAccountMinuteUsage, + StatisticsAccountUsage, StatisticsDayBucket, StatisticsLedger, - StatisticsProviderBucket + StatisticsMinuteBucket, + StatisticsProviderBucket, + StatisticsRolling24Hours } from "../shared/types"; export { aggregateStatisticsRange } from "../shared/statistics-aggregation"; @@ -24,6 +28,24 @@ const providers = new Set([ ]); const renameRetryDelaysMs = [15, 40, 90]; +const minuteMs = 60_000; +const rollingWindowMs = 24 * 60 * minuteMs; +const retainedMinuteCount = 48 * 60; +const maximumAccountIdLength = 128; +const maximumAccountLabelLength = 96; + +const providerLabels: Record = { + realdebrid: "Real-Debrid", + megadebrid: "Mega-Debrid", + "megadebrid-api": "Mega-Debrid API", + "megadebrid-web": "Mega-Debrid Web", + bestdebrid: "BestDebrid", + alldebrid: "AllDebrid", + ddownload: "DDownload", + onefichier: "1Fichier", + debridlink: "Debrid-Link", + linksnappy: "LinkSnappy" +}; function finiteNonNegative(value: unknown): number { const number = Number(value); @@ -36,6 +58,97 @@ function asRecord(value: unknown): Record | null { : null; } +function minuteStart(epochMs: number): number { + return Math.floor(finiteNonNegative(epochMs) / minuteMs) * minuteMs; +} + +function validAccountId(value: unknown): string | null { + const id = String(value || "").trim(); + return id.length > 0 + && id.length <= maximumAccountIdLength + && /^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(id) + ? id + : null; +} + +function maskEmailLikeLabel(value: string): string { + const match = /^([^@\s]+)@([^@\s]+)$/.exec(value); + if (!match) { + return value; + } + const local = match[1]; + const hidden = "*".repeat(Math.max(3, Math.min(8, local.length - 1))); + return `${local.slice(0, 1)}${hidden}@${match[2]}`; +} + +function safeAccountLabel(value: unknown, provider: DebridProvider): string { + const clean = String(value || "") + .replace(/[\u0000-\u001f\u007f]/g, " ") + .replace(/\s+/g, " ") + .trim() + .slice(0, maximumAccountLabelLength); + return maskEmailLikeLabel(clean || providerLabels[provider]); +} + +function fallbackAccountId(provider: DebridProvider): string { + return `provider:${provider}`; +} + +function normalizeMinuteAccounts(value: unknown): Record { + const accounts: Record = {}; + for (const [rawId, rawUsage] of Object.entries(asRecord(value) ?? {})) { + const id = validAccountId(rawId); + const usage = asRecord(rawUsage); + const provider = String(usage?.provider || "") as DebridProvider; + const bytes = finiteNonNegative(usage?.bytes); + if (!id || !providers.has(provider) || bytes <= 0) { + continue; + } + accounts[id] = { + provider, + label: safeAccountLabel(usage?.label, provider), + bytes + }; + } + return accounts; +} + +function minimumRetainedMinute(now: number): number { + return minuteStart(now) - ((retainedMinuteCount - 1) * minuteMs); +} + +function normalizeMinutes(value: unknown, now: number): StatisticsMinuteBucket[] { + const currentMinute = minuteStart(now); + const minimumMinute = minimumRetainedMinute(now); + const buckets = new Map(); + for (const rawValue of Array.isArray(value) ? value : []) { + const record = asRecord(rawValue); + const minute = minuteStart(Number(record?.minute)); + if (!record || minute < minimumMinute || minute > currentMinute) { + continue; + } + const accounts = normalizeMinuteAccounts(record.accounts); + if (Object.keys(accounts).length === 0) { + continue; + } + const target = buckets.get(minute) ?? { minute, downloadedBytes: 0, accounts: {} }; + for (const [id, usage] of Object.entries(accounts)) { + const existing = target.accounts[id]; + if (existing && existing.provider === usage.provider) { + existing.bytes += usage.bytes; + existing.label = usage.label; + } else { + target.accounts[id] = { ...usage }; + } + } + target.downloadedBytes = Object.values(target.accounts).reduce((total, usage) => total + usage.bytes, 0); + buckets.set(minute, target); + } + return [...buckets.values()] + .sort((left, right) => left.minute - right.minute) + .slice(-retainedMinuteCount); +} + function emptyProviderBucket(): StatisticsProviderBucket { return { bytes: 0, completed: 0, failed: 0 }; } @@ -86,7 +199,7 @@ function normalizeDay(value: unknown): StatisticsDayBucket | null { } export function createStatisticsLedger(now = Date.now()): StatisticsLedger { - return { version: 1, startedAt: now, days: [] }; + return { version: 2, startedAt: now, days: [], minutes: [] }; } export function normalizeStatisticsLedger(value: unknown, now = Date.now()): StatisticsLedger { @@ -102,12 +215,161 @@ export function normalizeStatisticsLedger(value: unknown, now = Date.now()): Sta } } return { - version: 1, + version: 2, startedAt: finiteNonNegative(record.startedAt) || now, - days: [...byDay.values()].sort((left, right) => left.day.localeCompare(right.day)) + days: [...byDay.values()].sort((left, right) => left.day.localeCompare(right.day)), + minutes: normalizeMinutes(record.minutes, now) }; } +export function projectStatisticsLedger(ledger: StatisticsLedger, now = Date.now()): StatisticsLedger { + return normalizeStatisticsLedger({ ...ledger, minutes: [] }, now); +} + +export function addStatisticsAccountBytesInPlace( + ledger: StatisticsLedger, + provider: DebridProvider, + byteDelta: number, + accountId?: string, + accountLabel?: string, + epochMs = Date.now() +): StatisticsAccountUsage | null { + const bytes = finiteNonNegative(byteDelta); + if (bytes <= 0 || !providers.has(provider)) { + return null; + } + const minute = minuteStart(epochMs); + const id = validAccountId(accountId) ?? fallbackAccountId(provider); + const label = safeAccountLabel(accountLabel, provider); + let bucket: StatisticsMinuteBucket | undefined = ledger.minutes[ledger.minutes.length - 1]; + if (!bucket || bucket.minute !== minute) { + ledger.minutes = ledger.minutes.filter((entry) => entry.minute >= minimumRetainedMinute(epochMs) && entry.minute <= minute); + bucket = ledger.minutes.find((entry) => entry.minute === minute); + if (!bucket) { + bucket = { minute, downloadedBytes: 0, accounts: {} }; + ledger.minutes.push(bucket); + ledger.minutes.sort((left, right) => left.minute - right.minute); + if (ledger.minutes.length > retainedMinuteCount) { + ledger.minutes.splice(0, ledger.minutes.length - retainedMinuteCount); + } + } + } + if (!bucket) { + return null; + } + const existing = bucket.accounts[id]; + if (existing && existing.provider === provider) { + existing.bytes += bytes; + existing.label = label; + } else { + bucket.accounts[id] = { provider, label, bytes }; + } + bucket.downloadedBytes = Object.values(bucket.accounts).reduce((total, usage) => total + usage.bytes, 0); + return { id, provider, label, bytes }; +} + +function aggregateRollingAccountStatistics(ledger: StatisticsLedger, now: number): StatisticsRolling24Hours { + const currentMinute = minuteStart(now); + const from = minuteStart(now - rollingWindowMs); + const accounts = new Map(); + for (const bucket of ledger.minutes) { + if (bucket.minute < from || bucket.minute > currentMinute) { + continue; + } + for (const [id, usage] of Object.entries(bucket.accounts)) { + const existing = accounts.get(id); + if (existing && existing.provider === usage.provider) { + existing.bytes += usage.bytes; + existing.label = usage.label; + } else { + accounts.set(id, { id, provider: usage.provider, label: usage.label, bytes: usage.bytes }); + } + } + } + const rows = [...accounts.values()].sort((left, right) => + right.bytes - left.bytes + || left.label.localeCompare(right.label, "de-DE", { sensitivity: "base" }) + || left.id.localeCompare(right.id) + ); + return { + from, + to: now, + downloadedBytes: rows.reduce((total, account) => total + account.bytes, 0), + accounts: rows + }; +} + +export class RollingAccountStatisticsAccumulator { + private ledger: StatisticsLedger; + private minute = -1; + private aggregate: StatisticsRolling24Hours; + + public constructor(ledger: StatisticsLedger, now = Date.now()) { + this.ledger = ledger; + this.aggregate = aggregateRollingAccountStatistics(ledger, now); + this.minute = minuteStart(now); + } + + public record( + provider: DebridProvider, + byteDelta: number, + accountId?: string, + accountLabel?: string, + epochMs = Date.now() + ): void { + this.refresh(epochMs); + const recorded = addStatisticsAccountBytesInPlace( + this.ledger, + provider, + byteDelta, + accountId, + accountLabel, + epochMs + ); + if (!recorded || minuteStart(epochMs) < this.aggregate.from || minuteStart(epochMs) > this.minute) { + return; + } + const existing = this.aggregate.accounts.find((account) => account.id === recorded.id && account.provider === recorded.provider); + if (existing) { + existing.bytes += recorded.bytes; + existing.label = recorded.label; + } else { + this.aggregate.accounts.push({ ...recorded }); + } + this.aggregate.downloadedBytes += recorded.bytes; + this.aggregate.accounts.sort((left, right) => + right.bytes - left.bytes + || left.label.localeCompare(right.label, "de-DE", { sensitivity: "base" }) + || left.id.localeCompare(right.id) + ); + this.aggregate.to = epochMs; + } + + public snapshot(now = Date.now()): StatisticsRolling24Hours { + this.refresh(now); + return { + ...this.aggregate, + to: now, + accounts: this.aggregate.accounts.map((account) => ({ ...account })) + }; + } + + public reset(ledger: StatisticsLedger, now = Date.now()): void { + this.ledger = ledger; + this.minute = minuteStart(now); + this.aggregate = aggregateRollingAccountStatistics(ledger, now); + } + + private refresh(now: number): void { + const currentMinute = minuteStart(now); + if (currentMinute === this.minute) { + return; + } + this.minute = currentMinute; + this.aggregate = aggregateRollingAccountStatistics(this.ledger, now); + } +} + function updateDay( ledger: StatisticsLedger, epochMs: number, diff --git a/src/shared/types.ts b/src/shared/types.ts index 0a1d792..0073c74 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -57,10 +57,37 @@ export interface StatisticsDayBucket { providers: Partial>; } +export interface StatisticsAccountMinuteUsage { + provider: DebridProvider; + label: string; + bytes: number; +} + +export interface StatisticsMinuteBucket { + minute: number; + downloadedBytes: number; + accounts: Record; +} + +export interface StatisticsAccountUsage { + id: string; + provider: DebridProvider; + label: string; + bytes: number; +} + +export interface StatisticsRolling24Hours { + from: number; + to: number; + downloadedBytes: number; + accounts: StatisticsAccountUsage[]; +} + export interface StatisticsLedger { - version: 1; + version: 2; startedAt: number; days: StatisticsDayBucket[]; + minutes: StatisticsMinuteBucket[]; } export interface DownloadStats { @@ -76,6 +103,7 @@ export interface DownloadStats { totalRuntimeMs: number; runtimeMeasuredAt: number; statistics?: StatisticsLedger; + rolling24Hours?: StatisticsRolling24Hours; } export interface DebridAccountStatus { diff --git a/tests/backup-payload.test.ts b/tests/backup-payload.test.ts index aaa0813..5185779 100644 --- a/tests/backup-payload.test.ts +++ b/tests/backup-payload.test.ts @@ -12,7 +12,7 @@ const session: SessionState = { reconnectReason: "", paused: false, running: true, updatedAt: 0 }; const history: HistoryEntry[] = [{ id: "h1" } as unknown as HistoryEntry]; -const statistics: StatisticsLedger = { version: 1, startedAt: 1, days: [] }; +const statistics: StatisticsLedger = { version: 2, startedAt: 1, days: [], minutes: [] }; const baseInput = { appVersion: "1.7.183", exportedAt: "2026-06-07T00:00:00Z", session, history, statistics }; diff --git a/tests/statistics-ledger.test.ts b/tests/statistics-ledger.test.ts index 6385576..9e01e85 100644 --- a/tests/statistics-ledger.test.ts +++ b/tests/statistics-ledger.test.ts @@ -6,9 +6,13 @@ import { defaultSettings } from "../src/main/constants"; import { DownloadManager } from "../src/main/download-manager"; import { createStoragePaths, emptySession } from "../src/main/storage"; import { + RollingAccountStatisticsAccumulator, + addStatisticsAccountBytesInPlace, aggregateStatisticsRange, createStatisticsLedger, loadStatisticsLedger, + normalizeStatisticsLedger, + projectStatisticsLedger, recordStatisticsActiveInterval, recordStatisticsBytes, recordStatisticsOutcome, @@ -28,6 +32,119 @@ function localTime(day: number, hour = 12): number { } describe("statistics ledger", () => { + it("migrates version one ledgers without inventing minute history", () => { + const now = localTime(10); + const legacy = { + version: 1, + startedAt: now - 1_000, + days: [{ + day: "2026-08-10", + downloadedBytes: 1_024, + measuredBytes: 1_024, + completedFiles: 1, + failedFiles: 0, + activeDownloadMs: 100, + providers: { realdebrid: { bytes: 1_024, completed: 1, failed: 0 } } + }] + }; + + const migrated = normalizeStatisticsLedger(legacy, now); + + expect(migrated.version).toBe(2); + expect(migrated.days).toEqual(legacy.days); + expect(migrated.minutes).toEqual([]); + }); + + it("normalizes, merges, and bounds sparse account minute history", () => { + const now = localTime(10); + const currentMinute = Math.floor(now / 60_000) * 60_000; + const keptMinute = currentMinute - (47 * 60 * 60 * 1_000); + const raw = { + version: 2, + startedAt: now - 1_000, + days: [], + minutes: [ + { + minute: keptMinute, + downloadedBytes: 1, + accounts: { + rdw_one: { provider: "realdebrid", label: "secret@example.test", bytes: 20 } + } + }, + { + minute: keptMinute, + downloadedBytes: 1, + accounts: { + rdw_one: { provider: "realdebrid", label: "New label", bytes: 30 }, + "https://unsafe": { provider: "realdebrid", label: "Unsafe", bytes: 99 } + } + }, + { + minute: currentMinute - (49 * 60 * 60 * 1_000), + accounts: { rdw_old: { provider: "realdebrid", label: "Old", bytes: 500 } } + }, + { + minute: currentMinute + 120_000, + accounts: { rdw_future: { provider: "realdebrid", label: "Future", bytes: 500 } } + } + ] + }; + + const normalized = normalizeStatisticsLedger(raw, now); + + expect(normalized.minutes).toHaveLength(1); + expect(normalized.minutes[0].minute).toBe(keptMinute); + expect(normalized.minutes[0].downloadedBytes).toBe(50); + expect(normalized.minutes[0].accounts.rdw_one.bytes).toBe(50); + expect(normalized.minutes[0].accounts.rdw_one.label).not.toContain("secret@example.test"); + expect(normalized.minutes[0].accounts).not.toHaveProperty("https://unsafe"); + }); + + it("records separate accounts and projects no minute history into renderer snapshots", () => { + const now = localTime(10); + const ledger = createStatisticsLedger(now); + + addStatisticsAccountBytesInPlace(ledger, "realdebrid", 50, "rdw_one", "Primary", now); + addStatisticsAccountBytesInPlace(ledger, "realdebrid", 75, "rdw_two", "Secondary", now); + addStatisticsAccountBytesInPlace(ledger, "debridlink", 25, undefined, undefined, now); + + expect(ledger.minutes).toHaveLength(1); + expect(ledger.minutes[0].downloadedBytes).toBe(150); + expect(ledger.minutes[0].accounts).toMatchObject({ + rdw_one: { provider: "realdebrid", label: "Primary", bytes: 50 }, + rdw_two: { provider: "realdebrid", label: "Secondary", bytes: 75 }, + "provider:debridlink": { provider: "debridlink", label: "Debrid-Link", bytes: 25 } + }); + expect(projectStatisticsLedger(ledger, now).minutes).toEqual([]); + expect(ledger.minutes).toHaveLength(1); + }); + + it("maintains rolling account totals incrementally and expires the boundary minute", () => { + const now = localTime(10); + const oldMinute = now - (24 * 60 * 60 * 1_000) - 60_000; + const boundaryMinute = Math.floor((now - (24 * 60 * 60 * 1_000)) / 60_000) * 60_000; + const ledger = createStatisticsLedger(now); + addStatisticsAccountBytesInPlace(ledger, "realdebrid", 500, "rdw_old", "Old", oldMinute); + addStatisticsAccountBytesInPlace(ledger, "realdebrid", 100, "rdw_boundary", "Boundary", boundaryMinute); + const accumulator = new RollingAccountStatisticsAccumulator(ledger, now); + + accumulator.record("realdebrid", 50, "rdw_one", "Primary", now); + accumulator.record("realdebrid", 75, "rdw_two", "Secondary", now); + const current = accumulator.snapshot(now); + + expect(current.downloadedBytes).toBe(225); + expect(current.accounts.map((account) => [account.id, account.bytes])).toEqual([ + ["rdw_boundary", 100], + ["rdw_two", 75], + ["rdw_one", 50] + ]); + expect(current.accounts).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: "rdw_old" })])); + + const expired = accumulator.snapshot(now + 60_000); + expect(expired.downloadedBytes).toBe(125); + expect(expired.accounts).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: "rdw_boundary" })])); + }); + it("aggregates every available day inside a rolling seven-day window without requiring seven complete days", () => { let ledger = createStatisticsLedger(localTime(10)); ledger = recordStatisticsBytes(ledger, "realdebrid", 100, localTime(7)); diff --git a/tests/visual/fixtures.ts b/tests/visual/fixtures.ts index 1fdd3da..f0292e6 100644 --- a/tests/visual/fixtures.ts +++ b/tests/visual/fixtures.ts @@ -467,8 +467,9 @@ function createDenseSnapshot(): UiSnapshot { totalRuntimeMs: 172800000, runtimeMeasuredAt: 1786312800000, statistics: { - version: 1, + version: 2, startedAt: 1786053600000, + minutes: [], days: [ { day: "2026-08-08",