fix(storage): retry transient notification outbox renames
Retry bounded Windows EPERM, EACCES, and EBUSY rename failures for asynchronous and startup outbox persistence while preserving atomic writes and immediate failure for permanent errors.
This commit is contained in:
@@ -160,6 +160,41 @@ function retryDelayMs(attempts: number): number {
|
|||||||
return Math.min(MAX_RETRY_DELAY_MS, 1000 * (2 ** Math.min(30, Math.max(0, attempts - 1))));
|
return Math.min(MAX_RETRY_DELAY_MS, 1000 * (2 ** Math.min(30, Math.max(0, attempts - 1))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const RENAME_RETRY_DELAYS_MS = [15, 40, 90, 180];
|
||||||
|
|
||||||
|
function isTransientRenameError(error: unknown): boolean {
|
||||||
|
const code = String((error as NodeJS.ErrnoException)?.code || "");
|
||||||
|
return code === "EPERM" || code === "EACCES" || code === "EBUSY";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renameWithRetry(oldPath: string, newPath: string): Promise<void> {
|
||||||
|
for (let attempt = 0; ; attempt += 1) {
|
||||||
|
try {
|
||||||
|
await fsp.rename(oldPath, newPath);
|
||||||
|
return;
|
||||||
|
} catch (error) {
|
||||||
|
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, RENAME_RETRY_DELAYS_MS[attempt]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function renameSyncWithRetry(oldPath: string, newPath: string): void {
|
||||||
|
for (let attempt = 0; ; attempt += 1) {
|
||||||
|
try {
|
||||||
|
fs.renameSync(oldPath, newPath);
|
||||||
|
return;
|
||||||
|
} catch (error) {
|
||||||
|
if (!isTransientRenameError(error) || attempt >= RENAME_RETRY_DELAYS_MS.length) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RENAME_RETRY_DELAYS_MS[attempt]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class NotificationOutbox {
|
export class NotificationOutbox {
|
||||||
private events: NotificationEvent[] = [];
|
private events: NotificationEvent[] = [];
|
||||||
private lastSuccessAt = 0;
|
private lastSuccessAt = 0;
|
||||||
@@ -374,7 +409,7 @@ export class NotificationOutbox {
|
|||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
await fsp.writeFile(tempPath, JSON.stringify(state), "utf8");
|
await fsp.writeFile(tempPath, JSON.stringify(state), "utf8");
|
||||||
await fsp.rename(tempPath, this.filePath);
|
await renameWithRetry(tempPath, this.filePath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
||||||
throw error;
|
throw error;
|
||||||
@@ -393,7 +428,7 @@ export class NotificationOutbox {
|
|||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
fs.writeFileSync(tempPath, JSON.stringify(state), "utf8");
|
fs.writeFileSync(tempPath, JSON.stringify(state), "utf8");
|
||||||
fs.renameSync(tempPath, this.filePath);
|
renameSyncWithRetry(tempPath, this.filePath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
try {
|
try {
|
||||||
fs.rmSync(tempPath, { force: true });
|
fs.rmSync(tempPath, { force: true });
|
||||||
|
|||||||
@@ -296,6 +296,23 @@ describe("NotificationOutbox", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("retries a transient Windows rename failure before persisting", async () => {
|
||||||
|
const filePath = createOutboxFile();
|
||||||
|
const actualRename = fsp.rename;
|
||||||
|
const rename = vi.spyOn(fsp, "rename")
|
||||||
|
.mockRejectedValueOnce(Object.assign(new Error("locked"), { code: "EPERM" }))
|
||||||
|
.mockImplementation((oldPath, newPath) => actualRename(oldPath, newPath));
|
||||||
|
try {
|
||||||
|
const outbox = new NotificationOutbox({ filePath, now: () => 1000, send: async () => true });
|
||||||
|
await outbox.enqueue(event("rename-retry"));
|
||||||
|
|
||||||
|
expect(rename).toHaveBeenCalledTimes(2);
|
||||||
|
expect(persisted(filePath).events.map((queuedEvent) => queuedEvent.id)).toEqual(["rename-retry"]);
|
||||||
|
} finally {
|
||||||
|
rename.mockRestore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("persists a cleaned empty legacy file atomically during load", () => {
|
it("persists a cleaned empty legacy file atomically during load", () => {
|
||||||
const filePath = createOutboxFile();
|
const filePath = createOutboxFile();
|
||||||
const rename = vi.spyOn(fs, "renameSync");
|
const rename = vi.spyOn(fs, "renameSync");
|
||||||
@@ -328,6 +345,26 @@ describe("NotificationOutbox", () => {
|
|||||||
rename.mockRestore();
|
rename.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("retries a transient synchronous Windows rename failure during load", () => {
|
||||||
|
const filePath = createOutboxFile();
|
||||||
|
fs.writeFileSync(filePath, JSON.stringify({ version: 1, events: [], lastSuccessAt: 0, lastFailureAt: 0 }), "utf8");
|
||||||
|
const actualRenameSync = fs.renameSync;
|
||||||
|
const rename = vi.spyOn(fs, "renameSync")
|
||||||
|
.mockImplementationOnce(() => {
|
||||||
|
throw Object.assign(new Error("locked"), { code: "EBUSY" });
|
||||||
|
})
|
||||||
|
.mockImplementation((oldPath, newPath) => actualRenameSync(oldPath, newPath));
|
||||||
|
try {
|
||||||
|
const outbox = new NotificationOutbox({ filePath, send: async () => true, now: () => 1000 });
|
||||||
|
|
||||||
|
expect(outbox.getStatus().queued).toBe(0);
|
||||||
|
expect(rename).toHaveBeenCalledTimes(2);
|
||||||
|
expect(fs.existsSync(`${filePath}.tmp`)).toBe(false);
|
||||||
|
} finally {
|
||||||
|
rename.mockRestore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("drops expired events before persisting or sending", async () => {
|
it("drops expired events before persisting or sending", async () => {
|
||||||
const filePath = createOutboxFile();
|
const filePath = createOutboxFile();
|
||||||
const send = vi.fn().mockResolvedValue(true);
|
const send = vi.fn().mockResolvedValue(true);
|
||||||
|
|||||||
Reference in New Issue
Block a user