fix: strengthen live recovery and support diagnostics
Apply account and key changes to active queues without a restart and isolate provider attempt cancellation so fallback accounts remain usable. Preserve pause ownership, bound persisted HTTP 416 recovery, reconcile resets with authoritative state, and stabilize package ordering and live update cadence. Correlate rotation, conversion, resume, disk, queue-control, clipboard, and support-export events while redacting sensitive data at every persistent boundary and again in generated bundles. Release as v2.0.31 with updated English documentation and regression coverage.
This commit is contained in:
Sucukdeluxe
@@ -1,28 +1,45 @@
|
||||
import fs from "node:fs";
|
||||
import { logTimestamp } from "./log-timestamp";
|
||||
import path from "node:path";
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import type { RotationEvent } from "../shared/types";
|
||||
import path from "node:path";
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import type { RotationEvent } from "../shared/types";
|
||||
import { sanitizeDiagnosticAccountLabel, sanitizeDiagnosticFields, sanitizeDiagnosticText } from "./diagnostic-sanitizer";
|
||||
|
||||
export type RotationItemSink = (event: RotationEvent) => void;
|
||||
const rotationItemContext = new AsyncLocalStorage<RotationItemSink>();
|
||||
|
||||
export function runWithRotationItemSink<T>(sink: RotationItemSink, fn: () => Promise<T>): Promise<T> {
|
||||
return rotationItemContext.run(sink, fn);
|
||||
}
|
||||
export interface RotationCorrelationContext {
|
||||
attemptId?: string;
|
||||
itemId?: string;
|
||||
packageId?: string;
|
||||
}
|
||||
|
||||
export type CorrelatedRotationEvent = RotationEvent & RotationCorrelationContext;
|
||||
export type RotationItemSink = (event: CorrelatedRotationEvent) => void;
|
||||
|
||||
interface RotationItemContext extends RotationCorrelationContext {
|
||||
sink: RotationItemSink;
|
||||
}
|
||||
|
||||
const rotationItemContext = new AsyncLocalStorage<RotationItemContext>();
|
||||
|
||||
export function runWithRotationItemSink<T>(
|
||||
sink: RotationItemSink,
|
||||
fn: () => Promise<T>,
|
||||
correlation: RotationCorrelationContext = {}
|
||||
): Promise<T> {
|
||||
return rotationItemContext.run({ ...correlation, sink }, fn);
|
||||
}
|
||||
|
||||
type RotationLevel = "INFO" | "WARN" | "ERROR";
|
||||
|
||||
const ROTATION_EVENT_RING_MAX = 60;
|
||||
const rotationEventRing: RotationEvent[] = [];
|
||||
let rotationEventSeq = 0;
|
||||
let rotationEventListener: ((event: RotationEvent) => void) | null = null;
|
||||
|
||||
export function setRotationEventListener(listener: ((event: RotationEvent) => void) | null): void {
|
||||
rotationEventListener = listener;
|
||||
}
|
||||
|
||||
export function getRecentRotationEvents(limit = ROTATION_EVENT_RING_MAX): RotationEvent[] {
|
||||
const rotationEventRing: CorrelatedRotationEvent[] = [];
|
||||
let rotationEventSeq = 0;
|
||||
let rotationEventListener: ((event: CorrelatedRotationEvent) => void) | null = null;
|
||||
|
||||
export function setRotationEventListener(listener: ((event: CorrelatedRotationEvent) => void) | null): void {
|
||||
rotationEventListener = listener;
|
||||
}
|
||||
|
||||
export function getRecentRotationEvents(limit = ROTATION_EVENT_RING_MAX): CorrelatedRotationEvent[] {
|
||||
const slice = rotationEventRing.slice(-limit);
|
||||
slice.reverse();
|
||||
return slice;
|
||||
@@ -35,25 +52,28 @@ function pushRotationEvent(
|
||||
event: string,
|
||||
fields?: Record<string, unknown>,
|
||||
at = Date.now()
|
||||
): void {
|
||||
rotationEventSeq += 1;
|
||||
const entry: RotationEvent = {
|
||||
id: `rot_${at}_${rotationEventSeq}`,
|
||||
at,
|
||||
level,
|
||||
): CorrelatedRotationEvent {
|
||||
rotationEventSeq += 1;
|
||||
const context = rotationItemContext.getStore();
|
||||
const entry: CorrelatedRotationEvent = {
|
||||
id: `rot_${at}_${rotationEventSeq}`,
|
||||
at,
|
||||
level,
|
||||
provider,
|
||||
accountLabel,
|
||||
event,
|
||||
reason: fields && fields.reason != null ? String(fields.reason) : undefined,
|
||||
category: fields && fields.category != null ? String(fields.category) : undefined,
|
||||
cooldownSec: fields && fields.cooldownSec != null ? Number(fields.cooldownSec) || 0 : undefined,
|
||||
next: fields && fields.next != null ? String(fields.next) : undefined
|
||||
};
|
||||
|
||||
const itemSink = rotationItemContext.getStore();
|
||||
if (itemSink) {
|
||||
try {
|
||||
itemSink(entry);
|
||||
cooldownSec: fields && fields.cooldownSec != null ? Number(fields.cooldownSec) || 0 : undefined,
|
||||
next: fields && fields.next != null ? String(fields.next) : undefined,
|
||||
attemptId: context?.attemptId ? sanitizeDiagnosticText(context.attemptId) : undefined,
|
||||
itemId: context?.itemId ? sanitizeDiagnosticText(context.itemId) : undefined,
|
||||
packageId: context?.packageId ? sanitizeDiagnosticText(context.packageId) : undefined
|
||||
};
|
||||
|
||||
if (context) {
|
||||
try {
|
||||
context.sink(entry);
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
@@ -67,11 +87,12 @@ function pushRotationEvent(
|
||||
}
|
||||
if (rotationEventListener) {
|
||||
try {
|
||||
rotationEventListener(uiEntry);
|
||||
rotationEventListener(uiEntry);
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
const ROTATION_LOG_MAX_FILE_BYTES = Number(process.env.RD_ACCOUNT_ROTATION_LOG_MAX_BYTES || 5 * 1024 * 1024);
|
||||
const ROTATION_LOG_RETENTION_DAYS = Number(process.env.RD_ACCOUNT_ROTATION_LOG_RETENTION_DAYS || 14);
|
||||
@@ -82,14 +103,14 @@ function sanitizeFieldValue(value: unknown): string {
|
||||
if (value === undefined || value === null) {
|
||||
return "";
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return value.replace(/\r?\n/g, "\\n");
|
||||
if (typeof value === "string") {
|
||||
return sanitizeDiagnosticText(value);
|
||||
}
|
||||
if (typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(value).replace(/\r?\n/g, "\\n");
|
||||
try {
|
||||
return sanitizeDiagnosticText(JSON.stringify(value));
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
@@ -155,24 +176,34 @@ export function initAccountRotationLog(baseDir: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
export function logAccountRotation(
|
||||
export function logAccountRotation(
|
||||
level: RotationLevel,
|
||||
provider: string,
|
||||
accountLabel: string,
|
||||
event: string,
|
||||
fields?: Record<string, unknown>
|
||||
): void {
|
||||
pushRotationEvent(level, provider, accountLabel, event, fields);
|
||||
if (!rotationLogPath) {
|
||||
return;
|
||||
event: string,
|
||||
fields?: Record<string, unknown>
|
||||
): void {
|
||||
const safeProvider = sanitizeDiagnosticText(provider);
|
||||
const safeAccountLabel = sanitizeDiagnosticAccountLabel(accountLabel);
|
||||
const safeEvent = sanitizeDiagnosticText(event);
|
||||
const safeFields = sanitizeDiagnosticFields(fields);
|
||||
const entry = pushRotationEvent(level, safeProvider, safeAccountLabel, safeEvent, safeFields);
|
||||
if (!rotationLogPath) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
rotateIfNeeded(rotationLogPath);
|
||||
if (!fs.existsSync(rotationLogPath)) {
|
||||
fs.writeFileSync(rotationLogPath, "", "utf8");
|
||||
}
|
||||
const head = `${logTimestamp()} [${level}] ${provider} | ${accountLabel} | ${event}`;
|
||||
fs.appendFileSync(rotationLogPath, `${head}${formatFields(fields)}\n`, "utf8");
|
||||
}
|
||||
const head = `${logTimestamp()} [${level}] ${safeProvider} | ${safeAccountLabel} | ${safeEvent}`;
|
||||
const logFields = {
|
||||
...safeFields,
|
||||
...(entry.attemptId ? { attemptId: entry.attemptId } : {}),
|
||||
...(entry.itemId ? { itemId: entry.itemId } : {}),
|
||||
...(entry.packageId ? { packageId: entry.packageId } : {})
|
||||
};
|
||||
fs.appendFileSync(rotationLogPath, `${head}${formatFields(logFields)}\n`, "utf8");
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user