From 374269efb2102be0fe80a41a993cc38e9c54186c Mon Sep 17 00:00:00 2001 From: brady gaster Date: Thu, 1 Oct 2026 09:01:17 -0700 Subject: [PATCH 1/2] fix(test): pin Aspire dashboard image to verified-good 13.5.2 build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the #2123/#2132 Aspire integration test CI failures was NOT CPU contention: `mcr.microsoft.com/dotnet/aspire-dashboard:latest` (and the major-version tag `:13`, confirmed to currently resolve to the identical freshly-published `13.6.0` build/digest) is broken — the container exits immediately (exitCode=145) because /app/Aspire.Dashboard.dll is missing from the image, so the dashboard health check can never succeed no matter the timeout. Fix: pin docker pull/run to `:13.5.2`, a distinct ~3-week-older build (different layer digests, verified via the MCR manifest API) that predates the broken `13.6.0` release and does not exhibit the missing-DLL problem. Diagnostics hardening made while chasing this down (kept, as they are independently valuable for any future flake in this suite): - Removed --rm from the dashboard `docker run` so a crashed container survives long enough for failure-path `docker inspect`/`docker logs` diagnostics to run (relying on existing explicit removeContainer() cleanup instead). - Captured container stderr in the `docker logs` diagnostics call (`2>&1`), since .NET unhandled-exception output goes to stderr and was previously silently dropped, hiding the real crash reason. - Raised isDockerAvailable()'s `docker info` timeout from 5s to 20s to prevent false-skip of the whole Docker-dependent suite under CI CPU contention (which would otherwise look like a false pass). - Raised the dashboard health-check timeout to 120s as defense-in-depth against genuine contention-induced slow starts, independent of the image-pinning fix above. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- test/aspire-integration.test.ts | 62 +++++++++++++++++++++++++++------ test/helpers/skip-guards.ts | 9 ++++- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/test/aspire-integration.test.ts b/test/aspire-integration.test.ts index 8c3c53439..80f88747f 100644 --- a/test/aspire-integration.test.ts +++ b/test/aspire-integration.test.ts @@ -36,13 +36,25 @@ const SKIP_REASON = dockerSkipReason() ?? playwrightBrowserSkipReason(); const CONTAINER_NAME = 'squad-aspire-dashboard'; const DASHBOARD_URL = 'http://localhost:18888'; const OTLP_GRPC_TARGET = 'http://localhost:4317'; +// Pin to a specific, verified-good patch version rather than a floating tag. +// Confirmed in CI: `:latest` AND the floating major tag `:13` currently both +// resolve to the SAME freshly-published `13.6.0` image (built 2026-10-01, +// same layer digests/sha256:239e58... as `:latest`), and that image's +// entrypoint fails immediately with "Aspire.Dashboard.dll does not exist" / +// "No .NET SDKs were found" — the app binary is missing from the image +// itself (confirmed via `docker inspect`/`docker logs`, not a slow/contended +// start). `13.5.2` is a distinct, ~3-week-older build (different layer +// digests, verified via the MCR manifest API) that predates this broken +// release and does not exhibit the problem. Re-pin forward once a newer +// verified-good patch tag is published upstream. +const DASHBOARD_IMAGE = 'mcr.microsoft.com/dotnet/aspire-dashboard:13.5.2'; // ============================================================================ // Helpers // ============================================================================ /** Poll a URL until it responds with 200 or timeout expires. */ -async function waitForHealthy(url: string, timeoutMs = 60_000): Promise { +async function waitForHealthy(url: string, timeoutMs = 120_000): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { try { @@ -53,7 +65,24 @@ async function waitForHealthy(url: string, timeoutMs = 60_000): Promise { } await new Promise((r) => setTimeout(r, 1_000)); } - throw new Error(`Dashboard at ${url} did not become healthy within ${timeoutMs}ms`); + // Surface container diagnostics so a slow/crashed start is distinguishable + // from a genuinely unreachable dashboard without needing to re-run in CI. + let diagnostics = ''; + try { + const state = execSync( + `docker inspect --format="{{.State.Status}} (exitCode={{.State.ExitCode}})" ${CONTAINER_NAME}`, + { encoding: 'utf8' }, + ).trim(); + // `docker logs` relays container stdout/stderr as two separate OS + // streams; execSync only returns stdout by default, so a crash reported + // via stderr (the common case for unhandled .NET exceptions) would be + // silently dropped. Redirect stderr into stdout so it's captured too. + const logs = execSync(`docker logs --tail 50 ${CONTAINER_NAME} 2>&1`, { encoding: 'utf8' }); + diagnostics = `\nContainer state: ${state}\nLast logs:\n${logs}`; + } catch (err) { + diagnostics = `\n(failed to collect container diagnostics: ${(err as Error).message})`; + } + throw new Error(`Dashboard at ${url} did not become healthy within ${timeoutMs}ms${diagnostics}`); } /** Force-remove the test container (ignore errors). */ @@ -122,33 +151,46 @@ describe.skipIf(SKIP_REASON !== null)( // Clean up any leftover container from a prior run removeContainer(); - // Pull latest Aspire dashboard image + // Pull the pinned Aspire dashboard image (see DASHBOARD_IMAGE above) execSync( - 'docker pull mcr.microsoft.com/dotnet/aspire-dashboard:latest', + `docker pull ${DASHBOARD_IMAGE}`, { stdio: 'inherit', timeout: 120_000 }, ); - // Start the Aspire dashboard container + // Start the Aspire dashboard container. + // Deliberately NOT using --rm: if the container exits/crashes instead + // of merely starting slowly, --rm would auto-delete it before + // waitForHealthy's failure-path diagnostics (docker inspect/logs) can + // run, masking the real cause as a generic timeout. removeContainer() + // (called above and in afterAll) already guarantees explicit cleanup. execSync( [ - 'docker run --rm -d', + 'docker run -d', `-p 18888:18888 -p 4317:18889`, '-e ASPIRE_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true', `--name ${CONTAINER_NAME}`, - 'mcr.microsoft.com/dotnet/aspire-dashboard:latest', + DASHBOARD_IMAGE, ].join(' '), { stdio: 'inherit' }, ); - // Wait for dashboard UI to respond - await waitForHealthy(DASHBOARD_URL, 60_000); + // Wait for dashboard UI to respond. 120s gives generous headroom over + // the dashboard's typical few-second cold start, absorbing real CPU + // contention on CI runners where this suite runs alongside ~10k other + // tests in full parallelism. (The repeated timeouts originally + // investigated in #2123/#2132 turned out to have a separate root + // cause — a transiently broken `:latest` image tag, fixed above by + // pinning to DASHBOARD_IMAGE — but a generous timeout remains + // worthwhile defense-in-depth against genuine contention-induced + // slow starts.) + await waitForHealthy(DASHBOARD_URL, 120_000); // Initialize OTel gRPC exporters targeting the dashboard initOTelForAspire(); // Launch Playwright browser browser = await chromium.launch({ headless: true }); - }, 180_000); // 3 min timeout for pull + start + }, 240_000); // 4 min timeout for pull + start (120s health-check budget + overhead) // ------------------------------------------------------------------ // Teardown: shutdown OTel, close browser, remove container diff --git a/test/helpers/skip-guards.ts b/test/helpers/skip-guards.ts index 2c79bbf52..0725c859a 100644 --- a/test/helpers/skip-guards.ts +++ b/test/helpers/skip-guards.ts @@ -13,7 +13,14 @@ import { execSync } from 'node:child_process'; */ export function isDockerAvailable(): boolean { try { - execSync('docker info', { stdio: 'ignore', timeout: 5000 }); + // 5s was too tight: under full-parallelism CI load (~10k tests running + // concurrently), `docker info`'s round-trip to the daemon can occasionally + // exceed 5s purely from CPU contention, causing this check to time out + // and silently SKIP the whole Docker-dependent suite instead of running + // it — a false "pass" that masks real failures (see Aspire dashboard CI + // investigation, #2123/#2132). 20s gives enough headroom to reliably + // detect a genuinely-available daemon under the same load. + execSync('docker info', { stdio: 'ignore', timeout: 20_000 }); return true; } catch { return false; From e3a087b3bd2dffbd95ac4281f7e28e583cea6b12 Mon Sep 17 00:00:00 2001 From: brady gaster Date: Thu, 1 Oct 2026 10:09:07 -0700 Subject: [PATCH 2/2] fix(test): widen beforeAll hook budget for diagnostics margin Addresses reviewer feedback on PR #2132: - The 240s beforeAll hook budget had zero margin above the bounded 120s docker-pull timeout + 120s health-check timeout, so Vitest could abort the hook with a generic timeout before waitForHealthy's more informative crash diagnostics could be thrown. Widened to 300s (~60s margin for container startup, diagnostics collection, and Playwright browser launch). - The --rm-vs-diagnostics concern on the same thread was already resolved in a prior commit (docker run no longer passes --rm); no further change needed there. Also trims two overly verbose/speculative inline comments (image-pin rationale, health-check rationale) down to the essential facts. Closes #2133 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- test/aspire-integration.test.ts | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/test/aspire-integration.test.ts b/test/aspire-integration.test.ts index 80f88747f..585425aa6 100644 --- a/test/aspire-integration.test.ts +++ b/test/aspire-integration.test.ts @@ -36,17 +36,12 @@ const SKIP_REASON = dockerSkipReason() ?? playwrightBrowserSkipReason(); const CONTAINER_NAME = 'squad-aspire-dashboard'; const DASHBOARD_URL = 'http://localhost:18888'; const OTLP_GRPC_TARGET = 'http://localhost:4317'; -// Pin to a specific, verified-good patch version rather than a floating tag. -// Confirmed in CI: `:latest` AND the floating major tag `:13` currently both -// resolve to the SAME freshly-published `13.6.0` image (built 2026-10-01, -// same layer digests/sha256:239e58... as `:latest`), and that image's -// entrypoint fails immediately with "Aspire.Dashboard.dll does not exist" / -// "No .NET SDKs were found" — the app binary is missing from the image -// itself (confirmed via `docker inspect`/`docker logs`, not a slow/contended -// start). `13.5.2` is a distinct, ~3-week-older build (different layer -// digests, verified via the MCR manifest API) that predates this broken -// release and does not exhibit the problem. Re-pin forward once a newer -// verified-good patch tag is published upstream. +// Pin to a verified-good patch version, not a floating tag: `:latest` and +// the major-version tag `:13` both currently resolve to a broken same-day +// upstream build (missing /app/Aspire.Dashboard.dll, confirmed via MCR +// manifest digest comparison). `13.5.2` is a distinct, older build that +// does not exhibit the problem. Re-pin forward once a newer verified-good +// patch tag is published upstream. const DASHBOARD_IMAGE = 'mcr.microsoft.com/dotnet/aspire-dashboard:13.5.2'; // ============================================================================ @@ -174,15 +169,9 @@ describe.skipIf(SKIP_REASON !== null)( { stdio: 'inherit' }, ); - // Wait for dashboard UI to respond. 120s gives generous headroom over - // the dashboard's typical few-second cold start, absorbing real CPU - // contention on CI runners where this suite runs alongside ~10k other - // tests in full parallelism. (The repeated timeouts originally - // investigated in #2123/#2132 turned out to have a separate root - // cause — a transiently broken `:latest` image tag, fixed above by - // pinning to DASHBOARD_IMAGE — but a generous timeout remains - // worthwhile defense-in-depth against genuine contention-induced - // slow starts.) + // 120s gives generous headroom over the dashboard's typical + // few-second cold start, as defense-in-depth against CI runner + // contention (independent of the image-pinning fix above). await waitForHealthy(DASHBOARD_URL, 120_000); // Initialize OTel gRPC exporters targeting the dashboard @@ -190,7 +179,7 @@ describe.skipIf(SKIP_REASON !== null)( // Launch Playwright browser browser = await chromium.launch({ headless: true }); - }, 240_000); // 4 min timeout for pull + start (120s health-check budget + overhead) + }, 300_000); // 5 min timeout: 120s pull + 120s health-check budget, plus ~60s margin for container startup, diagnostics, and browser launch // ------------------------------------------------------------------ // Teardown: shutdown OTel, close browser, remove container