fix(storage): retry transient notification outbox renames
Retry bounded Windows EPERM, EACCES, and EBUSY rename failures for asynchronous and startup outbox persistence while preserving atomic writes and immediate failure for permanent errors.
This commit is contained in:
@@ -160,6 +160,41 @@ function retryDelayMs(attempts: number): number {
|
||||
return Math.min(MAX_RETRY_DELAY_MS, 1000 * (2 ** Math.min(30, Math.max(0, attempts - 1))));
|
||||
}
|
||||
|
||||
const RENAME_RETRY_DELAYS_MS = [15, 40, 90, 180];
|
||||
|
||||
function isTransientRenameError(error: unknown): boolean {
|
||||
const code = String((error as NodeJS.ErrnoException)?.code || "");
|
||||
return code === "EPERM" || code === "EACCES" || code === "EBUSY";
|
||||
}
|
||||
|
||||
async function renameWithRetry(oldPath: string, newPath: string): Promise<void> {
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
await fsp.rename(oldPath, newPath);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
|
||||
throw error;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, RENAME_RETRY_DELAYS_MS[attempt]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function renameSyncWithRetry(oldPath: string, newPath: string): void {
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
fs.renameSync(oldPath, newPath);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
|
||||
throw error;
|
||||
}
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RENAME_RETRY_DELAYS_MS[attempt]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class NotificationOutbox {
|
||||
private events: NotificationEvent[] = [];
|
||||
private lastSuccessAt = 0;
|
||||
@@ -374,7 +409,7 @@ export class NotificationOutbox {
|
||||
};
|
||||
try {
|
||||
await fsp.writeFile(tempPath, JSON.stringify(state), "utf8");
|
||||
await fsp.rename(tempPath, this.filePath);
|
||||
await renameWithRetry(tempPath, this.filePath);
|
||||
} catch (error) {
|
||||
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
||||
throw error;
|
||||
@@ -393,7 +428,7 @@ export class NotificationOutbox {
|
||||
};
|
||||
try {
|
||||
fs.writeFileSync(tempPath, JSON.stringify(state), "utf8");
|
||||
fs.renameSync(tempPath, this.filePath);
|
||||
renameSyncWithRetry(tempPath, this.filePath);
|
||||
} catch (error) {
|
||||
try {
|
||||
fs.rmSync(tempPath, { force: true });
|
||||
|
||||
@@ -296,6 +296,23 @@ describe("NotificationOutbox", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("retries a transient Windows rename failure before persisting", async () => {
|
||||
const filePath = createOutboxFile();
|
||||
const actualRename = fsp.rename;
|
||||
const rename = vi.spyOn(fsp, "rename")
|
||||
.mockRejectedValueOnce(Object.assign(new Error("locked"), { code: "EPERM" }))
|
||||
.mockImplementation((oldPath, newPath) => actualRename(oldPath, newPath));
|
||||
try {
|
||||
const outbox = new NotificationOutbox({ filePath, now: () => 1000, send: async () => true });
|
||||
await outbox.enqueue(event("rename-retry"));
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(2);
|
||||
expect(persisted(filePath).events.map((queuedEvent) => queuedEvent.id)).toEqual(["rename-retry"]);
|
||||
} finally {
|
||||
rename.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("persists a cleaned empty legacy file atomically during load", () => {
|
||||
const filePath = createOutboxFile();
|
||||
const rename = vi.spyOn(fs, "renameSync");
|
||||
@@ -328,6 +345,26 @@ describe("NotificationOutbox", () => {
|
||||
rename.mockRestore();
|
||||
});
|
||||
|
||||
it("retries a transient synchronous Windows rename failure during load", () => {
|
||||
const filePath = createOutboxFile();
|
||||
fs.writeFileSync(filePath, JSON.stringify({ version: 1, events: [], lastSuccessAt: 0, lastFailureAt: 0 }), "utf8");
|
||||
const actualRenameSync = fs.renameSync;
|
||||
const rename = vi.spyOn(fs, "renameSync")
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error("locked"), { code: "EBUSY" });
|
||||
})
|
||||
.mockImplementation((oldPath, newPath) => actualRenameSync(oldPath, newPath));
|
||||
try {
|
||||
const outbox = new NotificationOutbox({ filePath, send: async () => true, now: () => 1000 });
|
||||
|
||||
expect(outbox.getStatus().queued).toBe(0);
|
||||
expect(rename).toHaveBeenCalledTimes(2);
|
||||
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
|
||||
} finally {
|
||||
rename.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("drops expired events before persisting or sending", async () => {
|
||||
const filePath = createOutboxFile();
|
||||
const send = vi.fn().mockResolvedValue(true);
|
||||
|
||||
Reference in New Issue
Block a user