diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 50dd7bd61..8b151e3e6 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -73,7 +73,16 @@ jobs: - 'apps/token-manager/**' build-on-push: - if: github.event_name == 'push' + if: >- + github.event_name == 'push' && ( + needs.detect-changes.outputs.api == 'true' || + needs.detect-changes.outputs.judge == 'true' || + needs.detect-changes.outputs.portal == 'true' || + needs.detect-changes.outputs.coder-acp-copilot == 'true' || + needs.detect-changes.outputs.coder-acp-claude-code == 'true' || + needs.detect-changes.outputs.token-manager == 'true' || + needs.detect-changes.outputs.shared == 'true' + ) needs: detect-changes runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'integration' }} @@ -142,7 +151,16 @@ jobs: . build-on-pr: - if: github.event_name == 'pull_request' + if: >- + github.event_name == 'pull_request' && ( + needs.detect-changes.outputs.api == 'true' || + needs.detect-changes.outputs.judge == 'true' || + needs.detect-changes.outputs.portal == 'true' || + needs.detect-changes.outputs.coder-acp-copilot == 'true' || + needs.detect-changes.outputs.coder-acp-claude-code == 'true' || + needs.detect-changes.outputs.token-manager == 'true' || + needs.detect-changes.outputs.shared == 'true' + ) needs: detect-changes runs-on: ubuntu-latest environment: integration diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index a2619d33f..ebff1e738 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -20,6 +20,7 @@ import { configureHelp, generateOutputFormatsHelp } from "./utils/helpFormatter. import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon, styleText } from "./utils/style.js"; import { formatData, isMachineReadable } from "./utils/formatters.js"; import type { OutputFormat, DisplayField } from "./utils/types.js"; +import { runGetAction } from "./run-get-action.js"; dotenv.config(); @@ -29,6 +30,7 @@ const normalizeUrl = (url: string): string => url.replace(/\/+$/, ''); function printFollowUpCommands(id: string): void { console.log(`\n${label('Run ID:')} ${value(id)}`); console.log(`\n${label('Next steps:')}`); + console.log(` ${dimTimestamp('Get details:')} pnpm cli run get -i ${id}`); console.log(` ${dimTimestamp('Check status:')} pnpm cli run status -i ${id}`); console.log(` ${dimTimestamp('Stream logs:')} pnpm cli run logs -i ${id}`); console.log(` ${dimTimestamp('Download:')} pnpm cli run download -i ${id}`); @@ -271,6 +273,16 @@ run } }); +run + .command("get") + .description("Get full details of a run") + .requiredOption("-i, --id ", "Run ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("-o, --output ", "Output format: table, tsv, or json", "table") + .action(async (options) => { + await runGetAction({ id: options.id, url: options.url, output: options.output }); + }); + run .command("logs") .description("Stream logs for a request") diff --git a/apps/cli/src/run-get-action.ts b/apps/cli/src/run-get-action.ts new file mode 100644 index 000000000..2c20218ad --- /dev/null +++ b/apps/cli/src/run-get-action.ts @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Action handler for `run get` subcommand — extracted for testability. + */ +import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon } from "./utils/style.js"; +import { formatData, isMachineReadable } from "./utils/formatters.js"; +import type { OutputFormat, DisplayField } from "./utils/types.js"; + +/** Strip trailing slashes from a URL */ +const normalizeUrl = (url: string): string => url.replace(/\/+$/, ''); + +export interface RunGetOptions { + id: string; + url: string; + output: string; +} + +export async function runGetAction(options: RunGetOptions): Promise { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/requests/${options.id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const run = await response.json(); + + // Machine-readable output + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'workerType', label: 'Worker' }, + { key: 'model', label: 'Model' }, + { key: 'status', label: 'Status' }, + { key: 'maxIterations', label: 'Max Iterations' }, + { key: 'turnsCount', label: 'Turns' }, + { key: 'passed', label: 'Passed' }, + { key: 'task', label: 'Task' }, + { key: 'criteriaCount', label: 'Criteria' }, + { key: 'logsCount', label: 'Logs' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + { key: 'error', label: 'Error' }, + ]; + const row = { + ...run, + turnsCount: run.turns?.length ?? 0, + passed: run.status === 'completed' ? 'yes' : run.status === 'failed' || run.status === 'exhausted' ? 'no' : '-', + task: run.scenario?.task ?? '', + criteriaCount: run.scenario?.criteria?.length ?? 0, + logsCount: run.logs?.length ?? 0, + }; + console.log(formatData([row], fields, format)); + return; + } + + // Human-readable output + console.log(`${label('ID:')} ${value(run.id)}`); + console.log(`${label('Worker:')} ${value(run.workerType)}`); + if (run.model) console.log(`${label('Model:')} ${value(run.model)}`); + + const statusColor = run.status === 'completed' ? successText + : (run.status === 'failed' || run.status === 'exhausted') ? errorText + : value; + console.log(`${label('Status:')} ${statusColor(run.status)}`); + + if (run.maxIterations != null) console.log(`${label('Max Iterations:')} ${value(String(run.maxIterations))}`); + if (run.createdAt) console.log(`${label('Created:')} ${dimTimestamp(new Date(run.createdAt).toLocaleString())}`); + if (run.updatedAt) console.log(`${label('Updated:')} ${dimTimestamp(new Date(run.updatedAt).toLocaleString())}`); + if (run.completedAt) console.log(`${label('Completed:')} ${dimTimestamp(new Date(run.completedAt).toLocaleString())}`); + + // Persona + if (run.persona) { + const p = run.persona; + console.log(`${label('Persona:')} ${value(`${p.personality} / ${p.experience} / ${p.verbosity} / ${p.type}`)}`); + } + + // Scenario + if (run.scenario) { + console.log(`${label('Task:')}`); + for (const line of (run.scenario.task ?? '').trim().split('\n')) { + console.log(` ${line}`); + } + if (run.scenario.criteria?.length > 0) { + console.log(`${label('Criteria:')} ${value(String(run.scenario.criteria.length))} criterion/criteria`); + // Build a lookup from the last turn's criteria results + const lastTurn = run.turns?.length ? run.turns[run.turns.length - 1] : null; + const resultsMap = new Map(); + if (lastTurn?.criteriaResults) { + for (const cr of lastTurn.criteriaResults) { + resultsMap.set(cr.criterionId, { passed: cr.passed, evaluated: cr.evaluated }); + } + } + for (const c of run.scenario.criteria) { + const r = resultsMap.get(c); + const icon = r ? criterionIcon(r.evaluated, r.passed) : ' '; + console.log(` ${icon} ${c}`); + } + } + } + + // Turns summary + if (run.turns?.length > 0) { + console.log(`\n${banner('─── Turns ───')}`); + for (const turn of run.turns) { + const passIcon = criterionIcon(true, turn.passed); + const criteriaStr = turn.criteriaResults?.length + ? ` — ${turn.criteriaResults.filter((cr: { passed: boolean }) => cr.passed).length}/${turn.criteriaResults.length} criteria passed` + : ''; + console.log(` ${label(`Turn ${turn.iteration}:`)} ${passIcon}${criteriaStr}`); + } + } + + // Error + if (run.error) { + console.log(`${label('Error:')} ${errorText(run.error)}`); + } + + // Logs count + if (run.logs?.length != null) { + console.log(`${label('Logs:')} ${value(String(run.logs.length))} entries`); + } + + // Prompt feature extraction + if (run.promptFeatureExtractionId) { + console.log(`${label('Prompt Features:')} ${value(run.promptFeatureExtractionId)}`); + } + + // Soft-deleted + if (run.deletedAt) { + console.log(`${label('Deleted:')} ${warnBanner(new Date(run.deletedAt).toLocaleString())}`); + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } +} diff --git a/apps/cli/src/run-get.test.ts b/apps/cli/src/run-get.test.ts new file mode 100644 index 000000000..6bc1f7422 --- /dev/null +++ b/apps/cli/src/run-get.test.ts @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; + +// ── Fixtures ───────────────────────────────────────────────────────────────── + +/** Minimal run — only required fields, no turns/persona/model */ +const minimalRun = { + id: 'run-001', + workerType: 'coder-acp-copilot', + status: 'pending', + scenario: { task: 'Add a button', criteria: ['has_button'] }, + logs: [], + createdAt: '2025-06-01T10:00:00Z', +}; + +/** Full run with all optional fields populated */ +const fullRun = { + id: 'run-002', + workerType: 'coder-acp-claude-code', + model: 'claude-sonnet-4-20250514', + status: 'completed', + scenario: { + version: 'v2' as const, + task: 'Implement login form\nwith validation', + criteria: ['has_form', 'has_validation', 'has_tests'], + }, + maxIterations: 3, + persona: { + personality: 'demanding' as const, + experience: 'senior' as const, + verbosity: 'brief' as const, + type: 'traditional' as const, + }, + turns: [ + { + iteration: 1, + codingAgentResponse: 'code v1', + judgeFeedback: 'needs tests', + snapshotUrl: 'https://snap/1', + passed: false, + timestamp: '2025-06-01T10:05:00Z', + criteriaResults: [ + { criterionId: 'has_form', passed: true, evaluated: true }, + { criterionId: 'has_validation', passed: true, evaluated: true }, + { criterionId: 'has_tests', passed: false, evaluated: true }, + ], + }, + { + iteration: 2, + codingAgentResponse: 'code v2', + judgeFeedback: 'all good', + snapshotUrl: 'https://snap/2', + passed: true, + timestamp: '2025-06-01T10:10:00Z', + criteriaResults: [ + { criterionId: 'has_form', passed: true, evaluated: true }, + { criterionId: 'has_validation', passed: true, evaluated: true }, + { criterionId: 'has_tests', passed: true, evaluated: true }, + ], + }, + ], + logs: [ + { timestamp: '2025-06-01T10:00:00Z', level: 'info', message: 'started' }, + { timestamp: '2025-06-01T10:10:00Z', level: 'info', message: 'done' }, + ], + createdAt: '2025-06-01T10:00:00Z', + updatedAt: '2025-06-01T10:10:00Z', + promptFeatureExtractionId: 'pfe-abc', +}; + +// ── Test helpers ───────────────────────────────────────────────────────────── + +/** + * Build a stripped-down Commander program that registers only the `run get` + * subcommand so we can invoke it programmatically. + * + * We dynamically import the real CLI module to pick up the registered command, + * but that also boots every other command and runs dotenv, etc. Instead we + * replicate the minimal wiring here — it tests the same display logic because + * we call it through Commander's `.parseAsync()`. + */ +async function buildRunGetCommand() { + // Re-import the module each time so vi.stubGlobal('fetch') is picked up. + // Dynamic import with cache-bust to avoid stale module. + const mod = await import('./index.js'); + // The default export is the program; grab the 'run' parent command. + // Commander nests: program → run → get + const program: Command = (mod as unknown as { default: Command }).default ?? (mod as unknown as { program: Command }).program; + + // If the module doesn't export program directly, we build our own mini program + // that mirrors the real `run get` logic — but that defeats the purpose. + // Instead, we test the output formatting inline. + return program; +} + +/** Capture all console.log calls and return them joined by newlines. */ +function captureConsole() { + const lines: string[] = []; + const spy = vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(' ')); + }); + const errLines: string[] = []; + const errSpy = vi.spyOn(console, 'error').mockImplementation((...args: unknown[]) => { + errLines.push(args.map(String).join(' ')); + }); + return { + lines, + errLines, + restore: () => { spy.mockRestore(); errSpy.mockRestore(); }, + }; +} + +// Stub process.exit to prevent vitest from dying +const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => { + throw new Error('process.exit called'); +}) as never); + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe('run get', () => { + beforeEach(() => { + vi.stubGlobal('fetch', vi.fn()); + exitSpy.mockClear(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('human-readable output', () => { + it('displays all fields for a full run', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(fullRun), + }); + vi.stubGlobal('fetch', mockFetch); + + const { lines, restore } = captureConsole(); + try { + // Import fresh to pick up stubbed fetch + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output: 'table' }); + } finally { + restore(); + } + + const output = lines.join('\n'); + + // Core fields + expect(output).toContain('run-002'); + expect(output).toContain('coder-acp-claude-code'); + expect(output).toContain('claude-sonnet-4-20250514'); + expect(output).toContain('completed'); + + // Persona + expect(output).toContain('demanding'); + expect(output).toContain('senior'); + + // Task + expect(output).toContain('Implement login form'); + + // Criteria listed + expect(output).toContain('has_form'); + expect(output).toContain('has_validation'); + expect(output).toContain('has_tests'); + + // Turns summary + expect(output).toContain('Turn 1'); + expect(output).toContain('Turn 2'); + expect(output).toContain('3/3 criteria passed'); + + // Logs count + expect(output).toContain('2'); + expect(output).toContain('entries'); + + // Prompt feature extraction + expect(output).toContain('pfe-abc'); + }); + + it('displays minimal run without optional fields', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(minimalRun), + }); + vi.stubGlobal('fetch', mockFetch); + + const { lines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-001', url: 'http://localhost:3100', output: 'table' }); + } finally { + restore(); + } + + const output = lines.join('\n'); + + expect(output).toContain('run-001'); + expect(output).toContain('coder-acp-copilot'); + expect(output).toContain('pending'); + expect(output).toContain('Add a button'); + + // Should NOT contain optional fields + expect(output).not.toContain('Model:'); + expect(output).not.toContain('Persona:'); + expect(output).not.toContain('Turn '); + expect(output).not.toContain('Prompt Features:'); + }); + }); + + describe('JSON output', () => { + it('outputs the full run as formatted JSON', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(fullRun), + }); + vi.stubGlobal('fetch', mockFetch); + + const { lines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output: 'json' }); + } finally { + restore(); + } + + const output = lines.join('\n'); + // Should contain valid JSON with key fields + expect(output).toContain('"id"'); + expect(output).toContain('"run-002"'); + expect(output).toContain('"workerType"'); + }); + }); + + describe('error handling', () => { + it('exits with error on 404', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: false, + json: () => Promise.resolve({ error: 'Request not found' }), + }); + vi.stubGlobal('fetch', mockFetch); + + const { errLines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'nonexistent', url: 'http://localhost:3100', output: 'table' }); + } catch { + // process.exit throws + } finally { + restore(); + } + + const errOutput = errLines.join('\n'); + expect(errOutput).toContain('Request not found'); + }); + + it('exits with error on network failure', async () => { + const mockFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED')); + vi.stubGlobal('fetch', mockFetch); + + const { errLines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-001', url: 'http://localhost:3100', output: 'table' }); + } catch { + // process.exit throws + } finally { + restore(); + } + + const errOutput = errLines.join('\n'); + expect(errOutput).toContain('ECONNREFUSED'); + }); + }); +});