fix(storage): retry transient notification outbox renames

Retry bounded Windows EPERM, EACCES, and EBUSY rename failures for asynchronous and startup outbox persistence while preserving atomic writes and immediate failure for permanent errors.
This commit is contained in:
Sucukdeluxe
2026-08-23 00:30:46 +02:00
parent 4202640904
commit 44b2af11ce
2 changed files with 74 additions and 2 deletions
+37 -2
View File
@@ -160,6 +160,41 @@ function retryDelayMs(attempts: number): number {
return Math.min(MAX_RETRY_DELAY_MS, 1000 * (2 ** Math.min(30, Math.max(0, attempts - 1)))); return Math.min(MAX_RETRY_DELAY_MS, 1000 * (2 ** Math.min(30, Math.max(0, attempts - 1))));
} }
const RENAME_RETRY_DELAYS_MS = [15, 40, 90, 180];
function isTransientRenameError(error: unknown): boolean {
const code = String((error as NodeJS.ErrnoException)?.code || "");
return code === "EPERM" || code === "EACCES" || code === "EBUSY";
}
async function renameWithRetry(oldPath: string, newPath: string): Promise<void> {
for (let attempt = 0; ; attempt += 1) {
try {
await fsp.rename(oldPath, newPath);
return;
} catch (error) {
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
throw error;
}
await new Promise((resolve) => setTimeout(resolve, RENAME_RETRY_DELAYS_MS[attempt]));
}
}
}
function renameSyncWithRetry(oldPath: string, newPath: string): void {
for (let attempt = 0; ; attempt += 1) {
try {
fs.renameSync(oldPath, newPath);
return;
} catch (error) {
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
throw error;
}
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RENAME_RETRY_DELAYS_MS[attempt]);
}
}
}
export class NotificationOutbox { export class NotificationOutbox {
private events: NotificationEvent[] = []; private events: NotificationEvent[] = [];
private lastSuccessAt = 0; private lastSuccessAt = 0;
@@ -374,7 +409,7 @@ export class NotificationOutbox {
}; };
try { try {
await fsp.writeFile(tempPath, JSON.stringify(state), "utf8"); await fsp.writeFile(tempPath, JSON.stringify(state), "utf8");
await fsp.rename(tempPath, this.filePath); await renameWithRetry(tempPath, this.filePath);
} catch (error) { } catch (error) {
await fsp.rm(tempPath, { force: true }).catch(() => {}); await fsp.rm(tempPath, { force: true }).catch(() => {});
throw error; throw error;
@@ -393,7 +428,7 @@ export class NotificationOutbox {
}; };
try { try {
fs.writeFileSync(tempPath, JSON.stringify(state), "utf8"); fs.writeFileSync(tempPath, JSON.stringify(state), "utf8");
fs.renameSync(tempPath, this.filePath); renameSyncWithRetry(tempPath, this.filePath);
} catch (error) { } catch (error) {
try { try {
fs.rmSync(tempPath, { force: true }); fs.rmSync(tempPath, { force: true });
+37
View File
@@ -296,6 +296,23 @@ describe("NotificationOutbox", () => {
} }
}); });
it("retries a transient Windows rename failure before persisting", async () => {
const filePath = createOutboxFile();
const actualRename = fsp.rename;
const rename = vi.spyOn(fsp, "rename")
.mockRejectedValueOnce(Object.assign(new Error("locked"), { code: "EPERM" }))
.mockImplementation((oldPath, newPath) => actualRename(oldPath, newPath));
try {
const outbox = new NotificationOutbox({ filePath, now: () => 1000, send: async () => true });
await outbox.enqueue(event("rename-retry"));
expect(rename).toHaveBeenCalledTimes(2);
expect(persisted(filePath).events.map((queuedEvent) => queuedEvent.id)).toEqual(["rename-retry"]);
} finally {
rename.mockRestore();
}
});
it("persists a cleaned empty legacy file atomically during load", () => { it("persists a cleaned empty legacy file atomically during load", () => {
const filePath = createOutboxFile(); const filePath = createOutboxFile();
const rename = vi.spyOn(fs, "renameSync"); const rename = vi.spyOn(fs, "renameSync");
@@ -328,6 +345,26 @@ describe("NotificationOutbox", () => {
rename.mockRestore(); rename.mockRestore();
}); });
it("retries a transient synchronous Windows rename failure during load", () => {
const filePath = createOutboxFile();
fs.writeFileSync(filePath, JSON.stringify({ version: 1, events: [], lastSuccessAt: 0, lastFailureAt: 0 }), "utf8");
const actualRenameSync = fs.renameSync;
const rename = vi.spyOn(fs, "renameSync")
.mockImplementationOnce(() => {
throw Object.assign(new Error("locked"), { code: "EBUSY" });
})
.mockImplementation((oldPath, newPath) => actualRenameSync(oldPath, newPath));
try {
const outbox = new NotificationOutbox({ filePath, send: async () => true, now: () => 1000 });
expect(outbox.getStatus().queued).toBe(0);
expect(rename).toHaveBeenCalledTimes(2);
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
} finally {
rename.mockRestore();
}
});
it("drops expired events before persisting or sending", async () => { it("drops expired events before persisting or sending", async () => {
const filePath = createOutboxFile(); const filePath = createOutboxFile();
const send = vi.fn().mockResolvedValue(true); const send = vi.fn().mockResolvedValue(true);