From 35364ca5880ef96e365b65fe55b769d60455972a Mon Sep 17 00:00:00 2001 From: Sucukdeluxe <259325684+Sucukdeluxe@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:46:35 +0200 Subject: [PATCH] fix(ci): stabilize Electron smoke flakiness at its sources The workspace cutter drop check waited a fixed 250ms before asserting the populated cutter controls, but the drop triggers asynchronous IPC and media probing, so loaded runners regularly lost that race and failed with a misleading error. The check now polls the exact same UI conditions with a bounded deadline before asserting. All window screenshots in the cutter and workspace smokes carry an explicit generous timeout because large two-times-DPR captures exceeded the Playwright default under load. The focused Electron smoke and the cutter media matrix additionally retry exactly once in both CI workflows, mirroring the established packaging retries for transient runner turbulence, and the CI contract now enforces those retries instead of the former single-line steps. --- .gitea/workflows/windows-ci.yml | 18 +++++++++++--- .github/workflows/windows-ci.yml | 18 +++++++++++--- scripts/smoke-test-ci-contract.js | 4 +-- scripts/smoke-test-cutter.js | 18 +++++++------- scripts/smoke-test-workspace-ui.js | 40 ++++++++++++++++-------------- 5 files changed, 60 insertions(+), 38 deletions(-) diff --git a/.gitea/workflows/windows-ci.yml b/.gitea/workflows/windows-ci.yml index b1923ef..ab7e64a 100644 --- a/.gitea/workflows/windows-ci.yml +++ b/.gitea/workflows/windows-ci.yml @@ -67,11 +67,21 @@ jobs: run: npm run test:unit timeout-minutes: 10 - name: Focused Electron smoke - run: npm run test:e2e:focused - timeout-minutes: 10 + run: | + npm run test:e2e:focused + if ($LASTEXITCODE -ne 0) { + npm run test:e2e:focused + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + timeout-minutes: 20 - name: Cutter media matrix - run: node scripts/smoke-test-cutter-media-matrix.js - timeout-minutes: 10 + run: | + node scripts/smoke-test-cutter-media-matrix.js + if ($LASTEXITCODE -ne 0) { + node scripts/smoke-test-cutter-media-matrix.js + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + timeout-minutes: 20 - name: Clean managed tools provision and repair run: npm run test:managed-tools-live timeout-minutes: 10 diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml index b3f7b85..ea4b838 100644 --- a/.github/workflows/windows-ci.yml +++ b/.github/workflows/windows-ci.yml @@ -93,11 +93,21 @@ jobs: run: npm run test:unit timeout-minutes: 10 - name: Focused Electron smoke - run: npm run test:e2e:focused - timeout-minutes: 10 + run: | + npm run test:e2e:focused + if ($LASTEXITCODE -ne 0) { + npm run test:e2e:focused + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + timeout-minutes: 20 - name: Cutter media matrix - run: node scripts/smoke-test-cutter-media-matrix.js - timeout-minutes: 10 + run: | + node scripts/smoke-test-cutter-media-matrix.js + if ($LASTEXITCODE -ne 0) { + node scripts/smoke-test-cutter-media-matrix.js + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + timeout-minutes: 20 - name: Clean managed tools provision and repair run: npm run test:managed-tools-live timeout-minutes: 10 diff --git a/scripts/smoke-test-ci-contract.js b/scripts/smoke-test-ci-contract.js index 29518dc..06906b2 100644 --- a/scripts/smoke-test-ci-contract.js +++ b/scripts/smoke-test-ci-contract.js @@ -591,9 +591,7 @@ for (const relativePath of ['.github/workflows/windows-ci.yml', '.gitea/workflow 'npm run test:cutter-matrix-contract', 'npm run test:live-integration-contract', 'npm run test:unit', - 'npm run test:e2e:focused', 'npm run build', - 'node scripts/smoke-test-cutter-media-matrix.js', 'npm run test:managed-tools-live', 'npm run test:packaged-launch', 'npm run test:installer' @@ -621,7 +619,7 @@ for (const relativePath of ['.github/workflows/windows-ci.yml', '.gitea/workflow const verifyLiveContractIndex = stepIndexByRun(verifyJob, 'npm run test:live-integration-contract'); check(verifyBuildIndex >= 0 && verifyBuildIndex < verifyLiveContractIndex, `${relativePath} verify must build before the live integration contract`); check(verifyBuildIndex < stepIndexByRun(verifyJob, 'npm run test:managed-tools-live'), `${relativePath} runs the live managed-tools check before build`); - for (const command of ['npm run pack', 'npm run dist:ci']) { + for (const command of ['npm run pack', 'npm run dist:ci', 'npm run test:e2e:focused', 'node scripts/smoke-test-cutter-media-matrix.js']) { check(hasSingleConditionalRetry(verifyJob?.steps.map((step) => step.raw).join('\n') || '', command), `${relativePath} does not retry transient ${command} failures exactly once`); } check(hasSingleConditionalRetry(verifyJob?.steps.map((step) => step.raw).join('\n') || '', 'npx install-electron --no'), `${relativePath} does not retry Electron binary provisioning exactly once`); diff --git a/scripts/smoke-test-cutter.js b/scripts/smoke-test-cutter.js index fc6da93..a15676c 100644 --- a/scripts/smoke-test-cutter.js +++ b/scripts/smoke-test-cutter.js @@ -460,7 +460,7 @@ async function run() { && emptyLayout.contained, `Empty cutter layout is stretched or displaced: ${JSON.stringify(emptyLayout)}` ); - await win.screenshot({ path: path.join(cutterArtifactDir, 'empty.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'empty.png') }); await win.setViewportSize({ width: 2048, height: 1152 }); const emptyFullscreenLayout = await win.evaluate(() => { const preview = document.getElementById('cutterPreview').getBoundingClientRect(); @@ -485,7 +485,7 @@ async function run() { && emptyFullscreenLayout.contained, `Fullscreen empty cutter layout is stretched or displaced: ${JSON.stringify(emptyFullscreenLayout)}` ); - await win.screenshot({ path: path.join(cutterArtifactDir, 'empty-fullscreen.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'empty-fullscreen.png') }); await win.setViewportSize({ width: 1440, height: 900 }); const emptyVolumeBefore = await win.evaluate(() => ({ width: document.getElementById('cutterVolume').getBoundingClientRect().width, @@ -1333,7 +1333,7 @@ async function run() { `Short moving media did not show a sharp timeline quickly: ${JSON.stringify({ scrubFirstAssetsReadyMs, scrubFirstAssetQuality })}` ); if (process.env.TWITCH_VOD_MANAGER_SCRUB_MEDIA) { - await win.screenshot({ path: path.join(cutterArtifactDir, 'real-clip-ready.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'real-clip-ready.png') }); await win.evaluate(() => window.updateCutterZoom(Number(document.getElementById('cutterZoom').max))); await win.waitForTimeout(500); realMaximumZoomState = await win.evaluate(() => { @@ -1349,7 +1349,7 @@ async function run() { }; }); check(realMaximumZoomState.renderedCount >= 150 && realMaximumZoomState.imageNodes === realMaximumZoomState.renderedCount && realMaximumZoomState.firstImageWidth <= 100, `Real maximum zoom still stretches thumbnail frames: ${JSON.stringify(realMaximumZoomState)}`); - await win.screenshot({ path: path.join(cutterArtifactDir, 'real-clip-maximum-zoom.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'real-clip-maximum-zoom.png') }); await win.evaluate(() => window.updateCutterZoom(1)); } await win.evaluate(async () => { @@ -1538,7 +1538,7 @@ async function run() { ])); const mediumFirstAssetsReadyMs = Date.now() - mediumLoadStarted; if (process.env.TWITCH_VOD_MANAGER_MEDIUM_MEDIA) { - await win.screenshot({ path: path.join(cutterArtifactDir, 'medium-clip-ready.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'medium-clip-ready.png') }); } check(mediumWaveformReadyMs < 1000, `The full-resolution 58-second waveform was not independently ready within one second: ${mediumWaveformReadyMs}ms`); const mediumZoomReuseBefore = await win.evaluate(() => ({ @@ -1551,7 +1551,7 @@ async function run() { await win.evaluate(() => window.updateCutterZoom(Number(document.getElementById('cutterZoom').max))); await win.waitForTimeout(500); if (process.env.TWITCH_VOD_MANAGER_MEDIUM_MEDIA) { - await win.screenshot({ path: path.join(cutterArtifactDir, 'medium-clip-maximum-zoom.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'medium-clip-maximum-zoom.png') }); } const mediumZoomReuseAfter = await win.evaluate(() => ({ thumbnailSource: document.querySelector('#cutterThumbnailStrip img').src, @@ -2137,7 +2137,7 @@ async function run() { volumeWidth: document.getElementById('cutterVolume').getBoundingClientRect().width, timeLeft: document.querySelector('.cutter-player-time').getBoundingClientRect().left })); - await win.screenshot({ path: path.join(cutterArtifactDir, 'volume-expanded.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'volume-expanded.png') }); check(expandedVolumeLayout.controlsWidth <= expandedVolumeLayout.clientWidth + 1 && expandedVolumeLayout.documentOverflow <= 1, `Expanded volume control causes overflow: ${JSON.stringify(expandedVolumeLayout)}`); check(focusedVolumeAppearance.active && focusedVolumeAppearance.appearance === 'none' && focusedVolumeAppearance.borderTopWidth === 0 && (focusedVolumeAppearance.outlineStyle === 'none' || focusedVolumeAppearance.outlineWidth === 0), `Focused volume control shows a rectangular native outline: ${JSON.stringify(focusedVolumeAppearance)}`); check( @@ -2322,7 +2322,7 @@ async function run() { }; }); check(assetDensity.zoom >= 8 && assetDensity.devicePixelRatio === 2 && assetDensity.thumbnailCount >= 190 && assetDensity.imageNodes === assetDensity.renderedFrames && assetDensity.renderedFrames >= 150 && assetDensity.renderedFrameIndexes.length === assetDensity.renderedFrames && assetDensity.renderedFrameIndexes[0] === 0 && assetDensity.renderedFrameIndexes.at(-1) === assetDensity.thumbnailCount - 1 && new Set(assetDensity.renderedFrameIndexes).size === assetDensity.renderedFrames && assetDensity.framePixelWidth > 0 && assetDensity.framePixelWidth <= assetDensity.sourceFrameWidth && assetDensity.sourceFrameWidth >= 320 && assetDensity.sourceFrameHeight >= 180 && assetDensity.canvasPixelWidth >= assetDensity.canvasCssWidth * 2 - 1 && assetDensity.waveformNaturalWidth >= assetDensity.targetWidth * 0.95 && assetDensity.waveformVerticalDensity >= 1 && assetDensity.waveformFilter === 'none' && assetDensity.waveformOpacity === 1 && assetDensity.trackEdgeError <= 1 && assetDensity.cutTimeError <= 1.5, `Timeline media is being upscaled, filtered or misaligned: ${JSON.stringify(assetDensity)}`); - await win.screenshot({ path: path.join(cutterArtifactDir, 'maximum-zoom.png') }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'maximum-zoom.png') }); const maximumZoomGeometryDelta = Math.max(...Object.keys(wheelZoomBefore.geometry).flatMap((key) => { const before = wheelZoomBefore.geometry[key]; const after = assetDensity.geometry[key]; @@ -2458,7 +2458,7 @@ async function run() { document.getElementById('cutterTab').scrollTop = 0; }); await win.waitForTimeout(250); - await win.screenshot({ path: path.join(cutterArtifactDir, 'editor.png'), fullPage: true }); + await win.screenshot({ timeout: 120000, path: path.join(cutterArtifactDir, 'editor.png'), fullPage: true }); const multiAudio = await verifyMultiAudioExport(win, environment, multiAudioInputFile, multiAudioOutputFile); const multiAudioSelection = multiAudio.selection; const multiAudioExportResult = multiAudio.exportResult; diff --git a/scripts/smoke-test-workspace-ui.js b/scripts/smoke-test-workspace-ui.js index 0ad565a..d297c14 100644 --- a/scripts/smoke-test-workspace-ui.js +++ b/scripts/smoke-test-workspace-ui.js @@ -458,7 +458,11 @@ async function run() { apiType: typeof window.api.selectDroppedVideo }; }); - await win.waitForTimeout(250); + await win.waitForFunction(() => { + const info = document.getElementById('cutterInfo'); + const cut = document.getElementById('btnCut'); + return Boolean(info?.classList.contains('shown') && cut && cut.disabled === false); + }, { timeout: 30000 }).catch(() => undefined); const cutterDropPaths = await app.evaluate(() => ({ ...globalThis.__workspaceCutterDropPaths })); const cutterDropUi = await win.evaluate(() => ({ filePath: document.getElementById('cutterFilePath')?.value || '', @@ -710,7 +714,7 @@ async function run() { check(downloadingKeyboardState.state === 'downloading' && downloadingStateAfterEnter === 'downloading', 'Keyboard activation changes the downloading state'); check(downloadingKeyboardState.progressWithinPopover && downloadingKeyboardState.trackWithinPopover, `Downloading progress exceeds the update popover: ${JSON.stringify(downloadingKeyboardState.geometry)}`); const updateProgressViewport = await win.evaluate(() => ({ width: window.innerWidth, height: window.innerHeight })); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-update-downloading-${updateProgressViewport.width}x${updateProgressViewport.height}.png`), fullPage: true }); @@ -1116,7 +1120,7 @@ async function run() { }; }, { cardSelector: pane.card, outputId: pane.output }); diagnosticLayouts.push({ target, pane: pane.id, ...layout }); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-settings-${pane.id}-${target.width}x${target.height}.png`), fullPage: true }); @@ -1185,7 +1189,7 @@ async function run() { check(cleanupOverflow.document <= 1 && cleanupOverflow.tab <= 1, `Cleanup Settings causes horizontal overflow: ${JSON.stringify(cleanupOverflow)}`); await win.evaluate(() => window.changeLanguage('de')); await win.waitForTimeout(160); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-settings-storage-1280x800.png'), fullPage: true }); @@ -1611,7 +1615,7 @@ async function run() { await win.setViewportSize(TARGETS[0]); await win.evaluate(() => window.showTab('vods')); await win.waitForTimeout(160); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-vods-fixture-${TARGETS[0].width}x${TARGETS[0].height}.png`), fullPage: true }); @@ -1717,13 +1721,13 @@ async function run() { check(systemLightTheme.bodyColor === lightTheme.bodyColor, `System-Light text ${systemLightTheme.bodyColor} does not match Light ${lightTheme.bodyColor}`); check(systemLightTheme.checkboxColor === lightTheme.checkboxColor && systemLightTheme.checkboxBackground === lightTheme.checkboxBackground, `System-Light checkbox does not match explicit Light: ${JSON.stringify({ lightTheme, systemLightTheme })}`); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-settings-system-light-${TARGETS[0].width}x${TARGETS[0].height}.png`), fullPage: true }); await win.locator('#workspaceThemePicker [data-theme="light"]').click(); await win.waitForTimeout(160); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-settings-light-${TARGETS[0].width}x${TARGETS[0].height}.png`), fullPage: true }); @@ -1757,7 +1761,7 @@ async function run() { check(updateScreenshotState.state === 'available', `Update screenshot uses ${updateScreenshotState.state} instead of available state`); check(updateScreenshotState.laterVisible && updateScreenshotState.dismissVisible, 'Update screenshot does not expose both Later and Close'); await win.waitForTimeout(160); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-update-${TARGETS[0].width}x${TARGETS[0].height}.png`), fullPage: true }); @@ -1904,7 +1908,7 @@ async function run() { check(streamerSidebarLayout.contextMenuActions.join(',') === 'auto,vod,record', 'Streamer context menu does not expose AUTO, VOD and REC actions'); check(streamerSidebarLayout.counterAtTitleEdge && streamerSidebarLayout.counterHasBorder, 'Streamer counter is not rendered as a title-edge badge'); check(streamerSidebarLayout.counterTextWithLiveStreamer === '1' && !streamerSidebarLayout.counterContainsLiveSuffix, `Streamer counter mixes total and live state: ${streamerSidebarLayout.counterTextWithLiveStreamer}`); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-streamer-context-menu.png'), fullPage: true }); @@ -1945,7 +1949,7 @@ async function run() { renderStreamers(); }); await win.waitForTimeout(150); - await win.screenshot({ path: path.join(artifactDir, 'workspace-streamer-slide-mid.png'), fullPage: true }); + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-streamer-slide-mid.png'), fullPage: true }); const emptyStreamerIndicatorOpacity = await win.evaluate(async () => { config.streamers = []; currentStreamer = null; @@ -2117,7 +2121,7 @@ async function run() { check(hdHoverFirst?.durationOpacity === '1', `VOD duration disappears during preview: ${hdHoverFirst?.durationOpacity}`); check(Number(hdHoverFirst?.durationZIndex) > 2, `VOD duration is behind the preview overlay: ${hdHoverFirst?.durationZIndex}`); check(hdHoverSecond?.includes(previewFrames[1]), `VOD hover does not cycle to the next 1080p frame: ${hdHoverSecond}`); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-vod-hover-hd.png'), fullPage: true }); @@ -2192,7 +2196,7 @@ async function run() { check(!queueWorkspaceView.streamersVisible && queueWorkspaceView.queueVisible, 'Queue switch leaves the streamer list in the queue workspace'); check(queueWorkspaceView.queueCanFillSidebar, 'Queue workspace does not use the available sidebar height'); check(queueWorkspaceView.countBackground === 'rgb(31, 122, 67)' && queueWorkspaceView.countColor === 'rgb(255, 255, 255)', 'Queue counter does not use the readable green treatment'); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-queue-view.png'), fullPage: true }); @@ -2247,7 +2251,7 @@ async function run() { await win.waitForTimeout(460); await win.evaluate(() => window.setVodsWorkspace('queue')); await win.waitForTimeout(150); - await win.screenshot({ path: path.join(artifactDir, 'workspace-context-switcher-slide-mid.png'), fullPage: true }); + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-context-switcher-slide-mid.png'), fullPage: true }); await win.evaluate(() => window.setVodsWorkspace('streamers')); await win.evaluate(() => window.showTab('settings')); @@ -2315,7 +2319,7 @@ async function run() { await win.waitForTimeout(460); await win.evaluate(() => window.changeLanguage('de')); await win.waitForTimeout(150); - await win.screenshot({ path: path.join(artifactDir, 'workspace-language-switcher-slide-mid.png'), fullPage: true }); + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-language-switcher-slide-mid.png'), fullPage: true }); const settingsNavigationMotion = await win.evaluate(async () => { const list = document.querySelector('[data-context-for="settings"] .context-list'); @@ -2368,7 +2372,7 @@ async function run() { updatesButton.click(); }); await win.waitForTimeout(150); - await win.screenshot({ path: path.join(artifactDir, 'workspace-settings-navigation-slide-mid.png'), fullPage: true }); + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-settings-navigation-slide-mid.png'), fullPage: true }); await win.evaluate(() => { window.changeLanguage('en'); window.showTab('vods'); @@ -2415,7 +2419,7 @@ async function run() { await win.locator('.top-nav button[data-tab="settings"]').hover(); await win.evaluate(() => window.showTab('settings')); await win.waitForTimeout(170); - await win.screenshot({ path: path.join(artifactDir, 'workspace-top-nav-slide-mid.png'), fullPage: true }); + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, 'workspace-top-nav-slide-mid.png'), fullPage: true }); await win.emulateMedia({ reducedMotion: 'no-preference' }); const responsiveTabs = {}; @@ -2475,7 +2479,7 @@ async function run() { } if (target.width === TARGETS[0].width) { - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-${tab}-${target.width}x${target.height}.png`), fullPage: true }); @@ -2499,7 +2503,7 @@ async function run() { check(geometry.sidebarWidth >= 260 && geometry.sidebarWidth <= 272, `Context sidebar width is ${geometry.sidebarWidth}px at ${target.width}x${target.height}`); check(geometry.toolbarHeight >= 59 && geometry.toolbarHeight <= 61, `Workspace toolbar height is ${geometry.toolbarHeight}px at ${target.width}x${target.height}`); check(!geometry.updateVisible, `Update action is visible without an available update at ${target.width}x${target.height}`); - await win.screenshot({ + await win.screenshot({ timeout: 120000, path: path.join(artifactDir, `workspace-${target.width}x${target.height}.png`), fullPage: true });