From 15c96ce7ca6d5ebabe0a0334b75e44664acfb048 Mon Sep 17 00:00:00 2001 From: Brady Gaster Date: Thu, 1 Oct 2026 10:33:50 -0700 Subject: [PATCH] test: harden Aspire dashboard startup Pin the dashboard fixture to the verified 13.5.2 manifest and diagnose startup exits immediately with bounded container state and logs. Add deterministic regression coverage for the missing-DLL exit and hung health-request paths. Closes #2133 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- test/aspire-integration.test.ts | 282 ++++++++++++++++++++++++++------ 1 file changed, 236 insertions(+), 46 deletions(-) diff --git a/test/aspire-integration.test.ts b/test/aspire-integration.test.ts index 585425aa6..e1b2a8618 100644 --- a/test/aspire-integration.test.ts +++ b/test/aspire-integration.test.ts @@ -8,8 +8,8 @@ * Requires Docker. Skipped when SKIP_DOCKER_TESTS=1 or Docker unavailable. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { execSync } from 'node:child_process'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { execSync, spawnSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import { chromium, type Browser } from 'playwright'; import { trace, metrics } from '@opentelemetry/api'; @@ -34,50 +34,151 @@ function playwrightBrowserSkipReason(): string | null { const SKIP_REASON = dockerSkipReason() ?? playwrightBrowserSkipReason(); const CONTAINER_NAME = 'squad-aspire-dashboard'; +const DASHBOARD_IMAGE = 'mcr.microsoft.com/dotnet/aspire-dashboard:13.5.2@sha256:0ef531119b8073aed12b0db2b4e4ab02866c6c69b7a52264269abd00cfb48a34'; const DASHBOARD_URL = 'http://localhost:18888'; const OTLP_GRPC_TARGET = 'http://localhost:4317'; -// Pin to a verified-good patch version, not a floating tag: `:latest` and -// the major-version tag `:13` both currently resolve to a broken same-day -// upstream build (missing /app/Aspire.Dashboard.dll, confirmed via MCR -// manifest digest comparison). `13.5.2` is a distinct, older build that -// does not exhibit the problem. Re-pin forward once a newer verified-good -// patch tag is published upstream. -const DASHBOARD_IMAGE = 'mcr.microsoft.com/dotnet/aspire-dashboard:13.5.2'; +const HEALTH_TIMEOUT_MS = 120_000; +const HEALTH_REQUEST_TIMEOUT_MS = 5_000; +const SETUP_TIMEOUT_MS = 300_000; +const DOCKER_LOG_TAIL_LINES = 50; +const MAX_DOCKER_LOG_CHARS = 32_000; +const DOCKER_LOG_MAX_BUFFER_BYTES = 64 * 1024; // ============================================================================ // Helpers // ============================================================================ -/** Poll a URL until it responds with 200 or timeout expires. */ -async function waitForHealthy(url: string, timeoutMs = 120_000): Promise { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { +interface ContainerState { + Status?: string; + Running?: boolean; + ExitCode?: number; + Error?: string; +} + +interface HealthProbe { + fetch: typeof fetch; + getContainerState: () => ContainerState | null; + getContainerLogs: () => string; + timeoutSignal: (milliseconds: number) => AbortSignal; + now: () => number; + sleep: (milliseconds: number) => Promise; +} + +interface DockerLogResult { + stdout: string; + stderr: string; + error?: Error; +} + +function getContainerState(): ContainerState | null { + try { + const output = execSync( + `docker inspect --format='{{json .State}}' ${CONTAINER_NAME}`, + { encoding: 'utf8' }, + ); + return JSON.parse(output) as ContainerState; + } catch { + return null; + } +} + +function runContainerLogs(): DockerLogResult { + const result = spawnSync( + 'docker', + ['logs', '--tail', String(DOCKER_LOG_TAIL_LINES), CONTAINER_NAME], + { + encoding: 'utf8', + maxBuffer: DOCKER_LOG_MAX_BUFFER_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + error: result.error, + }; +} + +function mergeBoundedDockerLogs(stdout: string, stderr: string): string { + const logs = [stdout.trim(), stderr.trim()].filter(Boolean).join('\n'); + if (logs.length <= MAX_DOCKER_LOG_CHARS) return logs; + + const prefix = `[truncated to last ${MAX_DOCKER_LOG_CHARS} characters]\n`; + return prefix + logs.slice(-(MAX_DOCKER_LOG_CHARS - prefix.length)); +} + +function getContainerLogs(runLogs: () => DockerLogResult = runContainerLogs): string { + const result = runLogs(); + const logs = mergeBoundedDockerLogs(result.stdout, result.stderr); + if (logs) return logs; + if (result.error) { + return `unavailable (${result.error.message})`; + } + return 'unavailable'; +} + +const DEFAULT_HEALTH_PROBE: HealthProbe = { + fetch, + getContainerState, + getContainerLogs, + timeoutSignal: (milliseconds) => AbortSignal.timeout(milliseconds), + now: Date.now, + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), +}; + +function startupFailure( + message: string, + state: ContainerState | null, + logs: string, +): Error { + const stateSummary = state + ? `status=${state.Status ?? 'unknown'}, running=${state.Running ?? false}, exitCode=${state.ExitCode ?? 'unknown'}, error=${state.Error || 'none'}` + : 'unavailable'; + return new Error(`${message}\nContainer state: ${stateSummary}\nContainer logs:\n${logs || '(empty)'}`); +} + +/** Poll the dashboard while failing immediately if its container exits. */ +async function waitForHealthy( + url: string, + timeoutMs = HEALTH_TIMEOUT_MS, + probe: HealthProbe = DEFAULT_HEALTH_PROBE, +): Promise { + const deadline = probe.now() + timeoutMs; + while (probe.now() < deadline) { + const remainingBeforeRequestMs = deadline - probe.now(); + if (remainingBeforeRequestMs <= 0) break; + const requestTimeoutMs = Math.min( + HEALTH_REQUEST_TIMEOUT_MS, + remainingBeforeRequestMs, + ); try { - const res = await fetch(url); + const res = await probe.fetch(url, { + signal: probe.timeoutSignal(requestTimeoutMs), + }); if (res.ok) return; } catch { // not ready yet } - await new Promise((r) => setTimeout(r, 1_000)); - } - // Surface container diagnostics so a slow/crashed start is distinguishable - // from a genuinely unreachable dashboard without needing to re-run in CI. - let diagnostics = ''; - try { - const state = execSync( - `docker inspect --format="{{.State.Status}} (exitCode={{.State.ExitCode}})" ${CONTAINER_NAME}`, - { encoding: 'utf8' }, - ).trim(); - // `docker logs` relays container stdout/stderr as two separate OS - // streams; execSync only returns stdout by default, so a crash reported - // via stderr (the common case for unhandled .NET exceptions) would be - // silently dropped. Redirect stderr into stdout so it's captured too. - const logs = execSync(`docker logs --tail 50 ${CONTAINER_NAME} 2>&1`, { encoding: 'utf8' }); - diagnostics = `\nContainer state: ${state}\nLast logs:\n${logs}`; - } catch (err) { - diagnostics = `\n(failed to collect container diagnostics: ${(err as Error).message})`; + + const state = probe.getContainerState(); + if (state && state.Running === false && state.Status !== 'created') { + throw startupFailure( + `Dashboard container exited before ${url} became healthy`, + state, + probe.getContainerLogs(), + ); + } + + const remainingMs = deadline - probe.now(); + if (remainingMs <= 0) break; + await probe.sleep(Math.min(1_000, remainingMs)); } - throw new Error(`Dashboard at ${url} did not become healthy within ${timeoutMs}ms${diagnostics}`); + + throw startupFailure( + `Dashboard at ${url} did not become healthy within ${timeoutMs}ms`, + probe.getContainerState(), + probe.getContainerLogs(), + ); } /** Force-remove the test container (ignore errors). */ @@ -134,6 +235,101 @@ async function shutdownOTel(): Promise { // Test suite // ============================================================================ +describe('Aspire dashboard startup diagnostics', () => { + it('uses an immutable known-good dashboard image', () => { + expect(DASHBOARD_IMAGE).toMatch( + /^mcr\.microsoft\.com\/dotnet\/aspire-dashboard:\d+\.\d+\.\d+@sha256:[a-f0-9]{64}$/, + ); + expect(DASHBOARD_IMAGE).not.toContain(':latest'); + }); + + it('fails immediately with container state and logs when startup exits', async () => { + const sleep = vi.fn(async () => {}); + const probe: HealthProbe = { + fetch: vi.fn(async () => { throw new Error('connection refused'); }), + getContainerState: () => ({ + Status: 'exited', + Running: false, + ExitCode: 145, + }), + getContainerLogs: () => "The application '/app/Aspire.Dashboard.dll' does not exist.", + timeoutSignal: () => new AbortController().signal, + now: () => 0, + sleep, + }; + + await expect(waitForHealthy(DASHBOARD_URL, HEALTH_TIMEOUT_MS, probe)).rejects.toThrow( + /container exited[\s\S]*exitCode=145[\s\S]*Aspire\.Dashboard\.dll/, + ); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('retains bounded Docker stdout and stderr when logs succeeds', () => { + const logs = getContainerLogs(() => ({ + stdout: 'Dashboard starting', + stderr: "The application '/app/Aspire.Dashboard.dll' does not exist.", + })); + + expect(logs).toContain('Dashboard starting'); + expect(logs).toContain('Aspire.Dashboard.dll'); + + const boundedLogs = getContainerLogs(() => ({ + stdout: 'x'.repeat(MAX_DOCKER_LOG_CHARS), + stderr: "The application '/app/Aspire.Dashboard.dll' does not exist.", + })); + expect(boundedLogs).toContain('Aspire.Dashboard.dll'); + expect(boundedLogs.length).toBeLessThanOrEqual(MAX_DOCKER_LOG_CHARS); + }); + + it('aborts a health request that never returns headers at the deadline', async () => { + const timeoutMs = HEALTH_REQUEST_TIMEOUT_MS + 1_000; + const requestedTimeouts: number[] = []; + let now = 0; + const sleep = vi.fn(async (milliseconds: number) => { + now += milliseconds; + }); + const hangingFetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + if (!signal) { + reject(new Error('missing abort signal')); + return; + } + const abort = () => { + reject(new Error('request aborted')); + }; + if (signal.aborted) abort(); + else signal.addEventListener('abort', abort, { once: true }); + })); + const probe: HealthProbe = { + fetch: hangingFetch as typeof fetch, + getContainerState: () => ({ Status: 'running', Running: true }), + getContainerLogs: () => 'still running', + timeoutSignal: (milliseconds) => { + requestedTimeouts.push(milliseconds); + const controller = new AbortController(); + queueMicrotask(() => { + now += milliseconds; + controller.abort(); + }); + return controller.signal; + }, + now: () => now, + sleep, + }; + + await expect(waitForHealthy(DASHBOARD_URL, timeoutMs, probe)).rejects.toThrow( + `did not become healthy within ${timeoutMs}ms`, + ); + expect(requestedTimeouts).toEqual([HEALTH_REQUEST_TIMEOUT_MS]); + expect(hangingFetch).toHaveBeenCalledWith( + DASHBOARD_URL, + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + expect(sleep).toHaveBeenCalledExactlyOnceWith(1_000); + }); +}); + describe.skipIf(SKIP_REASON !== null)( `Aspire dashboard integration (${SKIP_REASON ?? 'enabled'})`, () => { @@ -146,40 +342,34 @@ describe.skipIf(SKIP_REASON !== null)( // Clean up any leftover container from a prior run removeContainer(); - // Pull the pinned Aspire dashboard image (see DASHBOARD_IMAGE above) + // Pull the immutable dashboard image validated by this fixture execSync( `docker pull ${DASHBOARD_IMAGE}`, { stdio: 'inherit', timeout: 120_000 }, ); - // Start the Aspire dashboard container. - // Deliberately NOT using --rm: if the container exits/crashes instead - // of merely starting slowly, --rm would auto-delete it before - // waitForHealthy's failure-path diagnostics (docker inspect/logs) can - // run, masking the real cause as a generic timeout. removeContainer() - // (called above and in afterAll) already guarantees explicit cleanup. + // Start the Aspire dashboard container execSync( [ 'docker run -d', `-p 18888:18888 -p 4317:18889`, - '-e ASPIRE_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true', + '-e DASHBOARD__FRONTEND__AUTHMODE=Unsecured', + '-e DASHBOARD__OTLP__AUTHMODE=Unsecured', `--name ${CONTAINER_NAME}`, DASHBOARD_IMAGE, ].join(' '), { stdio: 'inherit' }, ); - // 120s gives generous headroom over the dashboard's typical - // few-second cold start, as defense-in-depth against CI runner - // contention (independent of the image-pinning fix above). - await waitForHealthy(DASHBOARD_URL, 120_000); + // Wait for dashboard UI to respond + await waitForHealthy(DASHBOARD_URL); // Initialize OTel gRPC exporters targeting the dashboard initOTelForAspire(); // Launch Playwright browser browser = await chromium.launch({ headless: true }); - }, 300_000); // 5 min timeout: 120s pull + 120s health-check budget, plus ~60s margin for container startup, diagnostics, and browser launch + }, SETUP_TIMEOUT_MS); // 5 min timeout for pull + health check under CI contention // ------------------------------------------------------------------ // Teardown: shutdown OTel, close browser, remove container