fix(storage): retry transient notification outbox renames

Retry bounded Windows EPERM, EACCES, and EBUSY rename failures for asynchronous and startup outbox persistence while preserving atomic writes and immediate failure for permanent errors.
This commit is contained in:
Sucukdeluxe
2026-08-23 00:30:46 +02:00
parent 4202640904
commit 44b2af11ce
2 changed files with 74 additions and 2 deletions
+37 -2
View File
@@ -160,6 +160,41 @@ function retryDelayMs(attempts: number): number {
return Math.min(MAX_RETRY_DELAY_MS, 1000 * (2 ** Math.min(30, Math.max(0, attempts - 1))));
}
const RENAME_RETRY_DELAYS_MS = [15, 40, 90, 180];
function isTransientRenameError(error: unknown): boolean {
const code = String((error as NodeJS.ErrnoException)?.code || "");
return code === "EPERM" || code === "EACCES" || code === "EBUSY";
}
async function renameWithRetry(oldPath: string, newPath: string): Promise<void> {
for (let attempt = 0; ; attempt += 1) {
try {
await fsp.rename(oldPath, newPath);
return;
} catch (error) {
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
throw error;
}
await new Promise((resolve) => setTimeout(resolve, RENAME_RETRY_DELAYS_MS[attempt]));
}
}
}
function renameSyncWithRetry(oldPath: string, newPath: string): void {
for (let attempt = 0; ; attempt += 1) {
try {
fs.renameSync(oldPath, newPath);
return;
} catch (error) {
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
throw error;
}
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RENAME_RETRY_DELAYS_MS[attempt]);
}
}
}
export class NotificationOutbox {
private events: NotificationEvent[] = [];
private lastSuccessAt = 0;
@@ -374,7 +409,7 @@ export class NotificationOutbox {
};
try {
await fsp.writeFile(tempPath, JSON.stringify(state), "utf8");
await fsp.rename(tempPath, this.filePath);
await renameWithRetry(tempPath, this.filePath);
} catch (error) {
await fsp.rm(tempPath, { force: true }).catch(() => {});
throw error;
@@ -393,7 +428,7 @@ export class NotificationOutbox {
};
try {
fs.writeFileSync(tempPath, JSON.stringify(state), "utf8");
fs.renameSync(tempPath, this.filePath);
renameSyncWithRetry(tempPath, this.filePath);
} catch (error) {
try {
fs.rmSync(tempPath, { force: true });
+37
View File
@@ -296,6 +296,23 @@ describe("NotificationOutbox", () => {
}
});
it("retries a transient Windows rename failure before persisting", async () => {
const filePath = createOutboxFile();
const actualRename = fsp.rename;
const rename = vi.spyOn(fsp, "rename")
.mockRejectedValueOnce(Object.assign(new Error("locked"), { code: "EPERM" }))
.mockImplementation((oldPath, newPath) => actualRename(oldPath, newPath));
try {
const outbox = new NotificationOutbox({ filePath, now: () => 1000, send: async () => true });
await outbox.enqueue(event("rename-retry"));
expect(rename).toHaveBeenCalledTimes(2);
expect(persisted(filePath).events.map((queuedEvent) => queuedEvent.id)).toEqual(["rename-retry"]);
} finally {
rename.mockRestore();
}
});
it("persists a cleaned empty legacy file atomically during load", () => {
const filePath = createOutboxFile();
const rename = vi.spyOn(fs, "renameSync");
@@ -328,6 +345,26 @@ describe("NotificationOutbox", () => {
rename.mockRestore();
});
it("retries a transient synchronous Windows rename failure during load", () => {
const filePath = createOutboxFile();
fs.writeFileSync(filePath, JSON.stringify({ version: 1, events: [], lastSuccessAt: 0, lastFailureAt: 0 }), "utf8");
const actualRenameSync = fs.renameSync;
const rename = vi.spyOn(fs, "renameSync")
.mockImplementationOnce(() => {
throw Object.assign(new Error("locked"), { code: "EBUSY" });
})
.mockImplementation((oldPath, newPath) => actualRenameSync(oldPath, newPath));
try {
const outbox = new NotificationOutbox({ filePath, send: async () => true, now: () => 1000 });
expect(outbox.getStatus().queued).toBe(0);
expect(rename).toHaveBeenCalledTimes(2);
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
} finally {
rename.mockRestore();
}
});
it("drops expired events before persisting or sending", async () => {
const filePath = createOutboxFile();
const send = vi.fn().mockResolvedValue(true);