Administrator/Multi-Hoster-Upload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 47 Wiki Activity

Harden renderer startup readiness recovery

Browse Source 
Require a bounded Ready signal after each main-document load, cancel the deadline on successful readiness or window disposal, and keep recovery limited to one reload before the branded failure surface.

Route renderer initialization failures through production startup handlers and reject malformed authenticated remote keyboard payloads without throwing or logging.
This commit is contained in:
Sucukdeluxe
2026-08-13 22:25:28 +02:00
parent d5ff45644c
commit a4d0854e76
4 changed files with 436 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/startup-renderer.js
+124 -12
View File
@@ -18,13 +18,39 @@ function createStartupFailureDocument(language) {
return `<!doctype html><html lang="${german ? 'de' : 'en'}"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Multi Hoster Uploader</title><style>html,body{height:100%;margin:0;background:#1f1f1f;color:#f4f4f4;font:15px system-ui,sans-serif}body{display:grid;place-items:center}.card{width:min(520px,calc(100% - 48px));padding:28px;border:1px solid #444;border-radius:12px;background:#292929;box-shadow:0 18px 50px #0008}h1{margin:0 0 12px;font-size:22px}p{margin:0 0 22px;color:#c8c8c8;line-height:1.5}button{min-height:38px;padding:0 18px;border:1px solid #555;border-radius:7px;background:#363636;color:#fff;font-weight:650;cursor:pointer}button:hover{background:#414141}</style></head><body><main class="card"><h1>${title}</h1><p>${detail}</p><button type="button" onclick="window.close()">${close}</button></main></body></html>`; return `<!doctype html><html lang="${german ? 'de' : 'en'}"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Multi Hoster Uploader</title><style>html,body{height:100%;margin:0;background:#1f1f1f;color:#f4f4f4;font:15px system-ui,sans-serif}body{display:grid;place-items:center}.card{width:min(520px,calc(100% - 48px));padding:28px;border:1px solid #444;border-radius:12px;background:#292929;box-shadow:0 18px 50px #0008}h1{margin:0 0 12px;font-size:22px}p{margin:0 0 22px;color:#c8c8c8;line-height:1.5}button{min-height:38px;padding:0 18px;border:1px solid #555;border-radius:7px;background:#363636;color:#fff;font-weight:650;cursor:pointer}button:hover{background:#414141}</style></head><body><main class="card"><h1>${title}</h1><p>${detail}</p><button type="button" onclick="window.close()">${close}</button></main></body></html>`;
} }
function createStartupRecoveryCoordinator({ load, reload, reveal, showFailure, close }) { function createStartupRecoveryCoordinator({
load,
reload,
reveal,
showFailure,
close,
readyTimeoutMs = 15000,
scheduleReadyDeadline = setTimeout,
cancelReadyDeadline = clearTimeout
}) {
let initialLoad; let initialLoad;
let crashReloads = 0; let crashReloads = 0;
let terminalFailure; let terminalFailure;
let recovery;
let readyDeadline;
let rendererDocument = 0;
let awaitingReady = false;
let stopped = false;
function clearRendererDeadline() {
if (readyDeadline !== undefined) cancelReadyDeadline(readyDeadline);
readyDeadline = undefined;
}
function abandonRendererDocument() {
awaitingReady = false;
clearRendererDeadline();
}
function endWithFailure(failure) { function endWithFailure(failure) {
if (stopped) return Promise.resolve(false);
if (!terminalFailure) { if (!terminalFailure) {
abandonRendererDocument();
terminalFailure = (async () => { terminalFailure = (async () => {
if (typeof showFailure !== 'function') { if (typeof showFailure !== 'function') {
await close(failure); await close(failure);
@@ -41,28 +67,48 @@ function createStartupRecoveryCoordinator({ load, reload, reveal, showFailure, c
return terminalFailure; return terminalFailure;
} }
async function recoverRenderer(phase, details) { function recoverRenderer(phase, details) {
if (stopped) return Promise.resolve(false);
if (terminalFailure) return terminalFailure; if (terminalFailure) return terminalFailure;
if (crashReloads >= 1) { if (recovery) return recovery;
return endWithFailure({ phase, attempt: crashReloads + 1, details }); abandonRendererDocument();
} const currentRecovery = (async () => {
crashReloads++; if (crashReloads >= 1) {
try { return endWithFailure({ phase, attempt: crashReloads + 1, details });
await reload(); }
} catch (error) { crashReloads++;
return endWithFailure({ phase: 'renderer-reload', attempt: crashReloads, details, error }); try {
} await reload();
return true;
} catch (error) {
if (stopped) return false;
return endWithFailure({ phase: 'renderer-reload', attempt: crashReloads, details, error });
}
})();
recovery = currentRecovery;
currentRecovery.then(
() => {
if (recovery === currentRecovery) recovery = undefined;
},
() => {
if (recovery === currentRecovery) recovery = undefined;
}
);
return currentRecovery;
} }
return { return {
loadInitial(...args) { loadInitial(...args) {
if (stopped) return Promise.resolve(false);
if (terminalFailure) return terminalFailure; if (terminalFailure) return terminalFailure;
if (!initialLoad) { if (!initialLoad) {
initialLoad = (async () => { initialLoad = (async () => {
for (let attempt = 1; attempt <= 2; attempt++) { for (let attempt = 1; attempt <= 2; attempt++) {
if (stopped) return false;
try { try {
return await load(...args); return await load(...args);
} catch (error) { } catch (error) {
if (stopped) return false;
if (attempt === 2) { if (attempt === 2) {
await endWithFailure({ phase: 'initial-load', attempt, error }); await endWithFailure({ phase: 'initial-load', attempt, error });
} }
@@ -78,11 +124,76 @@ function createStartupRecoveryCoordinator({ load, reload, reveal, showFailure, c
rendererInitializationFailed(details) { rendererInitializationFailed(details) {
return recoverRenderer('renderer-initialization', details); return recoverRenderer('renderer-initialization', details);
}, },
rendererLoadStarted() {
if (stopped || terminalFailure) return false;
clearRendererDeadline();
rendererDocument++;
awaitingReady = true;
return true;
},
rendererLoaded() {
if (stopped || terminalFailure || !awaitingReady) return false;
clearRendererDeadline();
const document = rendererDocument;
readyDeadline = scheduleReadyDeadline(() => {
if (stopped || terminalFailure || !awaitingReady || rendererDocument !== document) return false;
readyDeadline = undefined;
return recoverRenderer('renderer-ready-timeout', { timeoutMs: readyTimeoutMs });
}, readyTimeoutMs);
if (readyDeadline && typeof readyDeadline.unref === 'function') readyDeadline.unref();
return true;
},
rendererReady() { rendererReady() {
if (terminalFailure) return false; if (stopped || terminalFailure) return false;
abandonRendererDocument();
crashReloads = 0; crashReloads = 0;
reveal(); reveal();
return true; return true;
},
dispose() {
if (stopped) return false;
stopped = true;
abandonRendererDocument();
return true;
}
};
}
function createStartupRendererHandlers({ window, coordinator, onReady, onInitializationFailed }) {
let disposed = false;
function accepts(event) {
return !disposed && window && !window.isDestroyed() && event && event.sender === window.webContents;
}
return {
documentLoadStarted() {
if (disposed) return false;
return coordinator.rendererLoadStarted();
},
documentLoaded() {
if (disposed) return false;
return coordinator.rendererLoaded();
},
rendererCrashed(details) {
if (disposed) return false;
return coordinator.rendererCrashed(details);
},
rendererInitializationFailed(event, details) {
if (!accepts(event)) return false;
if (typeof onInitializationFailed === 'function') onInitializationFailed(details);
return coordinator.rendererInitializationFailed(details);
},
rendererReady(event) {
if (!accepts(event)) return false;
const ready = coordinator.rendererReady();
if (ready && typeof onReady === 'function') onReady();
return ready;
},
dispose() {
if (disposed) return false;
disposed = true;
return coordinator.dispose();
} }
}; };
} }
@@ -105,6 +216,7 @@ module.exports = {
configureStartupRenderer, configureStartupRenderer,
createStartupFailureDocument, createStartupFailureDocument,
createStartupRecoveryCoordinator, createStartupRecoveryCoordinator,
createStartupRendererHandlers,
createStartupWindow, createStartupWindow,
resolveStartupLanguage resolveStartupLanguage
}; };
main.js
+33 -10
View File
@@ -9,6 +9,7 @@ const {
configureStartupRenderer, configureStartupRenderer,
createStartupFailureDocument, createStartupFailureDocument,
createStartupRecoveryCoordinator, createStartupRecoveryCoordinator,
createStartupRendererHandlers,
createStartupWindow, createStartupWindow,
resolveStartupLanguage resolveStartupLanguage
} = require('./lib/startup-renderer'); } = require('./lib/startup-renderer');
@@ -135,7 +136,9 @@ const uploadBatchMutationGates = new WeakMap();
const uploadRecoveryStates = new WeakMap(); const uploadRecoveryStates = new WeakMap();
let lastSessionSummary = null; let lastSessionSummary = null;
let startupRecoveryCoordinator = null; let startupRecoveryCoordinator = null;
let startupRendererHandlers = null;
let sourceDeleteJournal = null; let sourceDeleteJournal = null;
const RENDERER_READY_TIMEOUT_MS = 15000;
const pendingUploadFinalizations = new Map(); const pendingUploadFinalizations = new Map();
function requestUploadFinalization(summary, historyPersisted) { function requestUploadFinalization(summary, historyPersisted) {
@@ -1462,6 +1465,11 @@ function createWindow() {
if (isInPlace || !isMainFrame) return; if (isInPlace || !isMainFrame) return;
closeHandshakeReady = false; closeHandshakeReady = false;
restoreClosePreparation(closePreparationAttempt); restoreClosePreparation(closePreparationAttempt);
if (startupRendererHandlers) startupRendererHandlers.documentLoadStarted();
});
mainWindow.webContents.on('did-finish-load', () => {
if (startupRendererHandlers) startupRendererHandlers.documentLoaded();
}); });
mainWindow.webContents.on('render-process-gone', (_event, details) => { mainWindow.webContents.on('render-process-gone', (_event, details) => {
@@ -1509,7 +1517,27 @@ function createWindow() {
if (mainWindow && !mainWindow.isDestroyed() && !mainWindow.isVisible()) mainWindow.show(); if (mainWindow && !mainWindow.isDestroyed() && !mainWindow.isVisible()) mainWindow.show();
}, },
showFailure: () => mainWindow.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(createStartupFailureDocument(startupLanguage))}`), showFailure: () => mainWindow.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(createStartupFailureDocument(startupLanguage))}`),
close: () => app.exit(1) close: () => app.exit(1),
readyTimeoutMs: RENDERER_READY_TIMEOUT_MS
});
startupRendererHandlers = createStartupRendererHandlers({
window: mainWindow,
coordinator: startupRecoveryCoordinator,
onReady: () => {
closeHandshakeReady = true;
},
onInitializationFailed: (details) => {
const message = details && typeof details.message === 'string' ? details.message : 'Renderer initialization failed';
_writeCrashLog('RENDERER INITIALIZATION FAILED', new Error(message), details);
debugLog(`RENDERER INITIALIZATION FAILED: ${message}`);
}
});
const currentStartupRendererHandlers = startupRendererHandlers;
const currentStartupRecoveryCoordinator = startupRecoveryCoordinator;
mainWindow.once('closed', () => {
currentStartupRendererHandlers.dispose();
if (startupRendererHandlers === currentStartupRendererHandlers) startupRendererHandlers = null;
if (startupRecoveryCoordinator === currentStartupRecoveryCoordinator) startupRecoveryCoordinator = null;
}); });
void startupRecoveryCoordinator.loadInitial(rendererTarget, rendererOptions); void startupRecoveryCoordinator.loadInitial(rendererTarget, rendererOptions);
} }
@@ -2985,18 +3013,11 @@ ipcMain.handle('app:quit', () => {
}); });
ipcMain.on('app:close-handshake-ready', (event) => { ipcMain.on('app:close-handshake-ready', (event) => {
if (mainWindow && !mainWindow.isDestroyed() && event.sender === mainWindow.webContents) { if (startupRendererHandlers) startupRendererHandlers.rendererReady(event);
closeHandshakeReady = true;
if (startupRecoveryCoordinator) startupRecoveryCoordinator.rendererReady();
}
}); });
ipcMain.on('app:renderer-initialization-failed', (event, details) => { ipcMain.on('app:renderer-initialization-failed', (event, details) => {
if (!mainWindow || mainWindow.isDestroyed() || event.sender !== mainWindow.webContents) return; if (startupRendererHandlers) void startupRendererHandlers.rendererInitializationFailed(event, details);
const message = details && typeof details.message === 'string' ? details.message : 'Renderer initialization failed';
_writeCrashLog('RENDERER INITIALIZATION FAILED', new Error(message), details);
debugLog(`RENDERER INITIALIZATION FAILED: ${message}`);
if (startupRecoveryCoordinator) void startupRecoveryCoordinator.rendererInitializationFailed(details);
}); });
ipcMain.on('app:close-preparation-started', (event, attempt) => { ipcMain.on('app:close-preparation-started', (event, attempt) => {
@@ -3481,11 +3502,13 @@ ipcMain.on('remote:capture-log', (_event, msg) => {
// IPC: Input events from capture window // IPC: Input events from capture window
ipcMain.on('remote:input-event', (_event, data) => { ipcMain.on('remote:input-event', (_event, data) => {
if (!mainWindow || mainWindow.isDestroyed()) return; if (!mainWindow || mainWindow.isDestroyed()) return;
if (!data || typeof data !== 'object' || Array.isArray(data)) return;
const config = configStore.load(); const config = configStore.load();
const remote = config.globalSettings && config.globalSettings.remote; const remote = config.globalSettings && config.globalSettings.remote;
if (!remote || !remote.allowInput) return; if (!remote || !remote.allowInput) return;
if (data.role !== 'admin') return; if (data.role !== 'admin') return;
if ((data.type === 'keydown' || data.type === 'keyup') && (typeof data.key !== 'string' || data.key.length === 0)) return;
// Capture includes window frame (title bar) but NOT invisible DWM borders // Capture includes window frame (title bar) but NOT invisible DWM borders
// sendInputEvent coordinates are relative to web content area // sendInputEvent coordinates are relative to web content area
tests/main-remote-input.test.js
+76
View File
@@ -0,0 +1,76 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
function loadRemoteInputHandler(sendInputEvent, debugLog = () => {}) {
const source = fs.readFileSync(path.join(__dirname, '..', 'main.js'), 'utf8');
const handlerStart = source.indexOf("ipcMain.on('remote:input-event'");
const handlerEnd = source.indexOf('\nfunction buildModifiers', handlerStart);
const modifiersEnd = source.indexOf('\n// IPC: Get capture source ID', handlerEnd);
assert.notEqual(handlerStart, -1);
assert.notEqual(handlerEnd, -1);
assert.notEqual(modifiersEnd, -1);
let inputHandler;
const mainWindow = {
isDestroyed: () => false,
getBounds: () => ({ x: 0, y: 0, width: 1100, height: 750 }),
getContentBounds: () => ({ x: 7, y: 30, width: 1086, height: 713 }),
webContents: { sendInputEvent }
};
const context = vm.createContext({
ipcMain: {
on(channel, handler) {
if (channel === 'remote:input-event') inputHandler = handler;
}
},
mainWindow,
configStore: {
load: () => ({ globalSettings: { remote: { allowInput: true } } })
},
debugLog,
process: { platform: 'win32' },
isFinite
});
vm.runInContext(source.slice(handlerStart, handlerEnd) + source.slice(handlerEnd, modifiersEnd), context);
assert.equal(typeof inputHandler, 'function');
return inputHandler;
}
test('authenticated keyboard input without a string key is discarded without throwing', () => {
const sent = [];
const logs = [];
const handler = loadRemoteInputHandler(event => sent.push(event), (...args) => logs.push(args));
const invalidPayloads = [
{ role: 'admin', type: 'keydown' },
{ role: 'admin', type: 'keydown', key: null },
{ role: 'admin', type: 'keydown', key: 1 },
{ role: 'admin', type: 'keydown', key: '' },
{ role: 'admin', type: 'keyup' },
{ role: 'admin', type: 'keyup', key: {} }
];
for (const payload of invalidPayloads) {
assert.doesNotThrow(() => handler({}, payload));
}
assert.deepEqual(sent, []);
assert.deepEqual(logs, []);
});
test('authenticated keyboard input with a string key keeps normal keydown and keyup behavior', () => {
const sent = [];
const handler = loadRemoteInputHandler(event => sent.push(event));
handler({}, { role: 'admin', type: 'keydown', key: 'a', ctrl: true });
handler({}, { role: 'admin', type: 'keyup', key: 'a', ctrl: true });
assert.deepEqual(JSON.parse(JSON.stringify(sent)), [
{ type: 'keyDown', keyCode: 'a', modifiers: ['control'] },
{ type: 'char', keyCode: 'a', modifiers: ['control'] },
{ type: 'keyUp', keyCode: 'a', modifiers: ['control'] }
]);
});
tests/startup-renderer.test.js
+203 -2
View File
@@ -7,6 +7,7 @@ const {
configureStartupRenderer, configureStartupRenderer,
createStartupFailureDocument, createStartupFailureDocument,
createStartupRecoveryCoordinator, createStartupRecoveryCoordinator,
createStartupRendererHandlers,
createStartupWindow, createStartupWindow,
resolveStartupLanguage resolveStartupLanguage
} = require('../lib/startup-renderer'); } = require('../lib/startup-renderer');
@@ -28,8 +29,11 @@ test('main process wires bounded startup recovery into real load and crash paths
assert.match(source, /createStartupRecoveryCoordinator/); assert.match(source, /createStartupRecoveryCoordinator/);
assert.match(source, /startupRecoveryCoordinator\.loadInitial/); assert.match(source, /startupRecoveryCoordinator\.loadInitial/);
assert.match(source, /startupRecoveryCoordinator\.rendererCrashed/); assert.match(source, /startupRecoveryCoordinator\.rendererCrashed/);
assert.match(source, /startupRecoveryCoordinator\.rendererInitializationFailed/); assert.match(source, /createStartupRendererHandlers/);
assert.match(source, /startupRecoveryCoordinator\.rendererReady/); assert.match(source, /startupRendererHandlers\.documentLoadStarted/);
assert.match(source, /startupRendererHandlers\.documentLoaded/);
assert.match(source, /startupRendererHandlers\.rendererInitializationFailed/);
assert.match(source, /startupRendererHandlers\.rendererReady/);
assert.match(source, /createStartupFailureDocument/); assert.match(source, /createStartupFailureDocument/);
}); });
@@ -59,6 +63,35 @@ class TestBrowserWindow extends EventEmitter {
} }
} }
function createManualScheduler() {
let nextId = 1;
const pending = new Map();
return {
schedule(callback, delay) {
const handle = { id: nextId++, unref() {} };
pending.set(handle, { callback, delay });
return handle;
},
cancel(handle) {
pending.delete(handle);
},
count() {
return pending.size;
},
delays() {
return Array.from(pending.values(), entry => entry.delay);
},
async fireNext() {
const entry = pending.entries().next().value;
assert.ok(entry);
const [handle, timer] = entry;
pending.delete(handle);
await timer.callback();
}
};
}
test('configureStartupRenderer leaves hardware acceleration enabled for a local Windows session', () => { test('configureStartupRenderer leaves hardware acceleration enabled for a local Windows session', () => {
let calls = 0; let calls = 0;
configureStartupRenderer({ disableHardwareAcceleration() { calls++; } }, { SESSIONNAME: 'Console' }, 'win32'); configureStartupRenderer({ disableHardwareAcceleration() { calls++; } }, { SESSIONNAME: 'Console' }, 'win32');
@@ -311,6 +344,174 @@ test('renderer initialization failures share the bounded recovery path', async (
}]); }]);
}); });
test('production startup handlers route renderer initialization failure through bounded recovery', async () => {
const failures = [];
let reloadCalls = 0;
let reportedFailure;
const webContents = {};
const coordinator = createStartupRecoveryCoordinator({
load() {},
async reload() {
reloadCalls++;
},
reveal() {},
async showFailure(failure) {
failures.push(failure);
},
close() {}
});
const handlers = createStartupRendererHandlers({
window: { isDestroyed: () => false, webContents },
coordinator,
onReady() {},
onInitializationFailed(details) {
reportedFailure = details;
}
});
const details = { message: 'top-level initialization failed' };
await handlers.rendererInitializationFailed({ sender: webContents }, details);
await handlers.rendererInitializationFailed({ sender: webContents }, details);
assert.equal(reloadCalls, 1);
assert.equal(reportedFailure, details);
assert.deepEqual(failures, [{
phase: 'renderer-initialization',
attempt: 2,
details
}]);
});
test('production startup handlers enforce the Ready deadline after every main document load', async () => {
const scheduler = createManualScheduler();
const failures = [];
let reloadCalls = 0;
const coordinator = createStartupRecoveryCoordinator({
load() {},
async reload() {
reloadCalls++;
},
reveal() {},
async showFailure(failure) {
failures.push(failure);
},
close() {},
readyTimeoutMs: 25,
scheduleReadyDeadline: scheduler.schedule,
cancelReadyDeadline: scheduler.cancel
});
const handlers = createStartupRendererHandlers({
window: { isDestroyed: () => false, webContents: {} },
coordinator,
onReady() {},
onInitializationFailed() {}
});
handlers.documentLoadStarted();
handlers.documentLoaded();
assert.deepEqual(scheduler.delays(), [25]);
await scheduler.fireNext();
assert.equal(reloadCalls, 1);
handlers.documentLoadStarted();
handlers.documentLoaded();
await scheduler.fireNext();
assert.equal(reloadCalls, 1);
assert.deepEqual(failures, [{
phase: 'renderer-ready-timeout',
attempt: 2,
details: { timeoutMs: 25 }
}]);
});
test('production startup handlers cancel the Ready deadline after a valid Ready signal', async () => {
const scheduler = createManualScheduler();
let reloadCalls = 0;
let readyCalls = 0;
const webContents = {};
const coordinator = createStartupRecoveryCoordinator({
load() {},
async reload() {
reloadCalls++;
},
reveal() {},
close() {},
readyTimeoutMs: 25,
scheduleReadyDeadline: scheduler.schedule,
cancelReadyDeadline: scheduler.cancel
});
const handlers = createStartupRendererHandlers({
window: { isDestroyed: () => false, webContents },
coordinator,
onReady() {
readyCalls++;
},
onInitializationFailed() {}
});
handlers.documentLoadStarted();
handlers.documentLoaded();
const ready = handlers.rendererReady({ sender: webContents });
assert.equal(ready, true);
assert.equal(readyCalls, 1);
assert.equal(scheduler.count(), 0);
assert.equal(reloadCalls, 0);
});
test('Ready before did-finish-load prevents a stale deadline', () => {
const scheduler = createManualScheduler();
const webContents = {};
const coordinator = createStartupRecoveryCoordinator({
load() {},
reload() {},
reveal() {},
close() {},
readyTimeoutMs: 25,
scheduleReadyDeadline: scheduler.schedule,
cancelReadyDeadline: scheduler.cancel
});
const handlers = createStartupRendererHandlers({
window: { isDestroyed: () => false, webContents },
coordinator,
onReady() {},
onInitializationFailed() {}
});
handlers.documentLoadStarted();
handlers.rendererReady({ sender: webContents });
handlers.documentLoaded();
assert.equal(scheduler.count(), 0);
});
test('disposing startup handlers cancels a pending Ready deadline', () => {
const scheduler = createManualScheduler();
const coordinator = createStartupRecoveryCoordinator({
load() {},
reload() {},
reveal() {},
close() {},
readyTimeoutMs: 25,
scheduleReadyDeadline: scheduler.schedule,
cancelReadyDeadline: scheduler.cancel
});
const handlers = createStartupRendererHandlers({
window: { isDestroyed: () => false, webContents: {} },
coordinator,
onReady() {},
onInitializationFailed() {}
});
handlers.documentLoadStarted();
handlers.documentLoaded();
handlers.dispose();
assert.equal(scheduler.count(), 0);
});
test('a successful renderer ready event reveals content and resets crash recovery', async () => { test('a successful renderer ready event reveals content and resets crash recovery', async () => {
const crashes = [ const crashes = [
{ reason: 'crashed', exitCode: 21 }, { reason: 'crashed', exitCode: 21 },