From 0946ea9594c263bc8765e285770ac3076e0520e5 Mon Sep 17 00:00:00 2001 From: Brady Gaster Date: Tue, 1 Sep 2026 17:11:19 -0700 Subject: [PATCH] fix(gh-aw): make Cast failures truthful Materialize and authenticate the Cast validator outside the agent prompt, preserve exact validation authorization, and emit bounded factual terminal failures without success-shaped reporting.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>\nCopilot-Session: 84876c36-36ab-4d53-a689-6ab9a52fe551 --- test/gh-aw-cast-validator.test.ts | 272 +++++++++++++++- test/gh-aw-quality.test.ts | 179 ++++++++++- workflows/shared/squad-cast-validator.md | 4 +- workflows/squad.md | 387 ++++++++++++++++++----- 4 files changed, 751 insertions(+), 91 deletions(-) diff --git a/test/gh-aw-cast-validator.test.ts b/test/gh-aw-cast-validator.test.ts index 6461c58e8..b67c7b87f 100644 --- a/test/gh-aw-cast-validator.test.ts +++ b/test/gh-aw-cast-validator.test.ts @@ -1,8 +1,9 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { spawnSync } from 'node:child_process'; +import { compileFunction, constants as vmConstants } from 'node:vm'; import { gunzipSync, gzipSync } from 'node:zlib'; import { afterEach, describe, expect, it } from 'vitest'; import { requirePosixShell } from './posix-shell'; @@ -166,6 +167,37 @@ function validatorCommand(): string { return command; } +function validatorRunnerSource(): string { + const workflow = readFileSync(workflowPath, 'utf8'); + const runner = workflow.match( + /cat > "\$validator_runner" <<'SQUAD_CAST_VALIDATOR_RUNNER'\r?\n([\s\S]*?)\r?\n SQUAD_CAST_VALIDATOR_RUNNER/, + )?.[1]; + if (!runner) { + throw new Error('Prepared Cast validator runner is missing'); + } + return runner + .split(/\r?\n/) + .map(line => line.startsWith(' ') ? line.slice(6) : line) + .join('\n'); +} + +function castFailureScript(): string { + const lines = readFileSync(workflowPath, 'utf8').split(/\r?\n/); + const step = lines.findIndex(line => line.includes('- name: Fail Cast terminal outcome')); + expect(step).toBeGreaterThan(-1); + const start = lines.findIndex((line, index) => index > step && /^\s+script:\s*\|$/.test(line)); + expect(start).toBeGreaterThan(step); + const indent = lines[start].match(/^\s*/)![0].length; + const script: string[] = []; + + for (let index = start + 1; index < lines.length; index++) { + const line = lines[index]; + if (line.trim() && line.match(/^\s*/)![0].length <= indent) break; + script.push(line.trim() ? line.slice(indent + 2) : ''); + } + return script.join('\n'); +} + function sha256(content: string | Buffer): string { return createHash('sha256').update(content).digest('hex'); } @@ -186,12 +218,23 @@ function materializeSkill( write(root, path, content); } +function materializeRunner( + fixture: ReturnType, + source = validatorRunnerSource(), +): string { + const runner = join(fixture.runnerTemp, 'run-squad-cast-validator'); + writeFileSync(runner, source, 'utf8'); + chmodSync(runner, 0o500); + return runner; +} + function runValidatorCommand( fixture: ReturnType, cwd = fixture.root, + runnerSource = validatorRunnerSource(), ) { const shell = process.platform === 'win32' ? requirePosixShell() : 'bash'; - return spawnSync(shell, ['-c', validatorCommand()], { + return spawnSync(shell, [materializeRunner(fixture, runnerSource)], { cwd, encoding: 'utf8', env: { @@ -206,6 +249,44 @@ function authorizesPullRequest(result: ReturnType): return result.status === 0 && result.stdout === 'Cast validation passed.\n'; } +function failureRecord(result: ReturnType): { + outcome: string; + stage: string; + command_category: string; + exit_status: string; + stderr: string; +} { + expect(result.status).not.toBe(0); + return JSON.parse(result.stdout) as ReturnType; +} + +async function runCastFailureJobOutput(outputContent: string): Promise { + const root = mkdtempSync(join(tmpdir(), 'gh-aw-cast-failure-job-')); + workspaces.push(root); + const output = join(root, 'agent-output.json'); + writeFileSync(output, outputContent); + const failures: string[] = []; + const previousOutput = process.env.GH_AW_AGENT_OUTPUT; + process.env.GH_AW_AGENT_OUTPUT = output; + + try { + const compiled = compileFunction( + `return (async () => {\n${castFailureScript()}\n})();`, + ['core'], + { importModuleDynamically: vmConstants.USE_MAIN_CONTEXT_DEFAULT_LOADER }, + ) as (core: { setFailed: (message: string) => void }) => Promise; + await compiled({ setFailed: message => failures.push(message) }); + } finally { + if (previousOutput === undefined) delete process.env.GH_AW_AGENT_OUTPUT; + else process.env.GH_AW_AGENT_OUTPUT = previousOutput; + } + return failures; +} + +async function runCastFailureJob(item: Record): Promise { + return runCastFailureJobOutput(JSON.stringify({ items: [{ type: 'cast_failure', ...item }] })); +} + afterEach(() => { for (const workspace of workspaces.splice(0)) { rmSync(workspace, { recursive: true, force: true }); @@ -224,31 +305,33 @@ describe('GH-AW Cast final-tree validator', () => { const normalizedEmbedded = Buffer.from(embedded.toString('utf8').replace(/\r\n/g, '\n')); expect(normalizedEmbedded).toEqual(canonical); - const commandDigest = validatorCommand().match( + const commandDigest = validatorRunnerSource().match( /validator_expected_sha256="([a-f0-9]{64})"/, )?.[1]; expect(commandDigest, 'runtime command must pin the canonical validator digest').toBeDefined(); expect(commandDigest).toBe(sha256(canonical)); }); - it('keeps validator bytes and transcription instructions out of the agent command', () => { + it('keeps validator bytes and extraction logic out of the agent command', () => { const command = validatorCommand(); + const runner = validatorRunnerSource(); const workflow = readFileSync(workflowPath, 'utf8'); - expect(command.length).toBeLessThan(3_000); + expect(command.trim()).toBe('"${RUNNER_TEMP:?}/run-squad-cast-validator"'); expect(command).not.toMatch(/[A-Za-z0-9+/]{256}/); - expect(command).not.toMatch(/cat\s+<<|SQUAD_CAST_VALIDATOR_B64'\s*\\/); + expect(command).not.toMatch(/cat\s+<<|base64|gzip|awk|validator_expected_sha256/); expect(command).not.toContain('H4sI'); expect(helperSource()).not.toMatch(/cat\s+< { expect(authorizesPullRequest(result)).toBe(false); }); + it('rejects authenticated validator bytes that fail node --check', () => { + const fixture = createFixture(); + const invalidProgram = 'const = ;\n'; + materializeSkill( + fixture.root, + undefined, + replaceValidatorPayload(materializedSkillSource(), invalidProgram), + ); + const authenticatedInvalidRunner = validatorRunnerSource().replace( + /validator_expected_sha256="[a-f0-9]{64}"/, + `validator_expected_sha256="${sha256(invalidProgram)}"`, + ); + + const result = runValidatorCommand(fixture, fixture.root, authenticatedInvalidRunner); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain('SyntaxError'); + expect(result.stderr).toContain('validate-gh-aw-cast.mjs'); + expect(result.stdout).not.toContain('Cast validation passed.'); + expect(authorizesPullRequest(result)).toBe(false); + }); + it('rejects a syntactically valid impostor that prints the exact success sentinel', () => { const fixture = createFixture(); const impostor = replaceValidatorPayload( @@ -362,6 +466,150 @@ describe('GH-AW Cast final-tree validator', () => { expect(authorizesPullRequest(result)).toBe(false); }); + it('surfaces bounded failure stages with the captured exit status and complete stderr', async () => { + const missing = createFixture(); + const missingResult = runValidatorCommand(missing); + + const ambiguous = createFixture(); + materializeSkill(ambiguous.root); + materializeSkill(ambiguous.root, '.claude/skills/squad-cast-validator/SKILL.md'); + const ambiguousResult = runValidatorCommand(ambiguous); + + const extraction = createFixture(); + materializeSkill( + extraction.root, + undefined, + materializedSkillSource().replace( + '', + '', + ), + ); + const extractionResult = runValidatorCommand(extraction); + + const integrity = createFixture(); + materializeSkill( + integrity.root, + undefined, + replaceValidatorPayload(materializedSkillSource(), "console.log('Cast validation passed.');\n"), + ); + const integrityResult = runValidatorCommand(integrity); + + const syntax = createFixture(); + const invalidProgram = 'const = ;\n'; + materializeSkill( + syntax.root, + undefined, + replaceValidatorPayload(materializedSkillSource(), invalidProgram), + ); + const syntaxResult = runValidatorCommand( + syntax, + syntax.root, + validatorRunnerSource().replace( + /validator_expected_sha256="[a-f0-9]{64}"/, + `validator_expected_sha256="${sha256(invalidProgram)}"`, + ), + ); + + const validation = createFixture(); + materializeSkill(validation.root); + write( + validation.root, + '.github/agents/squad.agent.md', + `${coordinatorMarkdown()}\nRead \`packages/squad-cli/src/internal.ts\` before dispatching.\n`, + ); + const validationResult = runValidatorCommand(validation); + + for (const { stage, commandCategory, result } of [ + { stage: 'discovery', commandCategory: 'validator skill discovery', result: missingResult }, + { stage: 'uniqueness', commandCategory: 'validator skill uniqueness', result: ambiguousResult }, + { stage: 'extraction', commandCategory: 'validator payload extraction', result: extractionResult }, + { stage: 'integrity', commandCategory: 'SHA-256 authentication', result: integrityResult }, + { stage: 'syntax', commandCategory: 'node --check', result: syntaxResult }, + { stage: 'validation', commandCategory: 'validator execution', result: validationResult }, + ]) { + expect(authorizesPullRequest(result)).toBe(false); + expect(result.status).not.toBeNull(); + expect(result.stderr).not.toBe(''); + const record = failureRecord(result); + expect(record).toEqual({ + outcome: 'cast_failure', + stage, + command_category: commandCategory, + exit_status: String(result.status), + stderr: result.stderr, + }); + const failures = await runCastFailureJob({ + stage: record.stage, + command_category: record.command_category, + exit_status: record.exit_status, + stderr: record.stderr, + }); + expect(failures).toEqual([ + [ + 'Cast did not complete.', + `Stage: ${stage}`, + `Command category: ${commandCategory}`, + `Exit status: ${result.status}`, + 'Stderr:', + result.stderr, + ].join('\n'), + ]); + } + }); + + it('preserves empty stderr when a validator command is unavailable', async () => { + const failures = await runCastFailureJob({ + stage: 'discovery', + command_category: 'validator skill discovery', + exit_status: 'unavailable', + stderr: '', + }); + expect(failures).toEqual([ + [ + 'Cast did not complete.', + 'Stage: discovery', + 'Command category: validator skill discovery', + 'Exit status: unavailable', + 'Stderr:', + '', + ].join('\n'), + ]); + }); + + it('fails malformed agent output with a deterministic diagnostic', async () => { + const failures = await runCastFailureJobOutput('{not-json'); + expect(failures).toHaveLength(1); + expect(failures[0]).toMatch(/^Unable to read Cast failure output: /); + }); + + it('fails non-object output items with a deterministic diagnostic', async () => { + const failures = await runCastFailureJobOutput(JSON.stringify({ + items: [null, { type: 'cast_failure' }], + })); + expect(failures).toEqual(['Cast failure output item 0 is not an object.']); + }); + + it('diagnoses conflicting failure and pull-request outputs without claiming prevention', async () => { + const failures = await runCastFailureJobOutput(JSON.stringify({ + items: [ + { + type: 'cast_failure', + stage: 'validation', + command_category: 'validator execution', + exit_status: '1', + stderr: 'Cast validation failed.', + }, + { type: 'create_pull_request' }, + ], + })); + expect(failures).toEqual([ + [ + 'Conflicting Cast terminal outputs: found 1 create_pull_request item(s) with cast_failure.', + 'This post-agent diagnostic cannot prevent a concurrently materialized pull request.', + ].join('\n'), + ]); + }); + it('accepts a self-contained descriptive Cast tree', () => { const fixture = createFixture(); const result = validate(fixture.root, fixture.payload); diff --git a/test/gh-aw-quality.test.ts b/test/gh-aw-quality.test.ts index 3079acf08..1d7ef7f84 100644 --- a/test/gh-aw-quality.test.ts +++ b/test/gh-aw-quality.test.ts @@ -300,7 +300,7 @@ describe('gh-aw: safe-output configuration', () => { it('each safe-output has a max value that is a positive integer ≤ 1000', () => { for (const [name, config] of Object.entries(safeOutputs)) { - if (name === 'data' || name === 'messages') continue; + if (name === 'data' || name === 'messages' || name === 'jobs') continue; expect(config.max, `${name} should have a max field`).toBeDefined(); const max = config.max as number; expect(max, `${name}.max should be > 0`).toBeGreaterThan(0); @@ -309,6 +309,22 @@ describe('gh-aw: safe-output configuration', () => { } }); + it('each custom safe-output job has a bounded max value', () => { + const jobsBlock = frontmatter.match( + /^ jobs:\n([\s\S]*?)(?=^ [\w-]+:)/m, + )?.[1] ?? ''; + const jobs = [...jobsBlock.matchAll( + /^ ([\w-]+):\n([\s\S]*?)(?=^ [\w-]+:|(?![\s\S]))/gm, + )]; + expect(jobs.length).toBeGreaterThan(0); + for (const [, name, config] of jobs) { + const max = Number(config.match(/^ max:\s*(\d+)\s*$/m)?.[1]); + expect(max, `${name}.max should be > 0`).toBeGreaterThan(0); + expect(max, `${name}.max should be ≤ 1000`).toBeLessThanOrEqual(1000); + expect(Number.isInteger(max), `${name}.max should be an integer`).toBe(true); + } + }); + it('labels arrays contain only non-empty strings', () => { for (const [name, config] of Object.entries(safeOutputs)) { if (config.labels) { @@ -923,7 +939,15 @@ describe('gh-aw: prompt budget & planning import regression', () => { // SHA-256 authentication command. That command is authored between marker comments // inside the `squad-cast` inline skill in workflows/squad.md; gh-aw strips the full // skill from the ambient prompt and loads it on demand. - const SOURCE_GROWTH_BUDGET_KB = 183; + // Raised 183 -> 189 KB by the truthful Cast terminal contract and its typed + // post-agent failure job. The ambient 40 KB guard still passes; the behavioral + // detail is confined to the on-demand `squad-cast` skill, while the safe job is + // required executable policy rather than ambient model guidance. + // Raised 189 -> 194 KB when the authenticated extraction and validation sequence + // moved from agent-transcribed prompt text into a deterministic pre-agent runner. + // The runner also emits a machine-readable factual failure record. The Cast skill + // shrank, and the ambient prompt remains below its independently enforced 40 KB cap. + const SOURCE_GROWTH_BUDGET_KB = 194; const SOURCE_GROWTH_BUDGET_BYTES = SOURCE_GROWTH_BUDGET_KB * 1024; it('squad-planning-ontology.md is in the imports list', () => { @@ -1374,6 +1398,54 @@ describe('gh-aw: compiled workflow shell input security contract', () => { ); }, 20000); + it('compiles Cast failure into a queryable post-agent job that fails the run', () => { + const compiled = lockText(); + const failureJob = compiled.match( + /^ cast_failure:\n[\s\S]*?(?=^ conclusion:)/m, + )?.[0] ?? ''; + const conclusionNeeds = compiled.match( + /^ conclusion:\n needs:\n([\s\S]*?)(?= if:)/m, + )?.[1] ?? ''; + expect(failureJob).toContain('name: Fail incomplete Cast'); + expect(failureJob).toContain('runs-on: ubuntu-slim'); + expect(failureJob).toContain("contains(needs.agent.outputs.output_types, 'cast_failure')"); + expect(failureJob).toContain("item.type === 'cast_failure'"); + expect(failureJob).toContain("item.type === 'create_pull_request'"); + expect(failureJob).toContain('process.env.GH_AW_AGENT_OUTPUT'); + expect(failureJob).toContain('Cast did not complete.'); + expect(failureJob).toContain('core.setFailed'); + expect(conclusionNeeds).toContain('- cast_failure'); + }, 20000); + + it('prepares the materialized Cast validator runner outside the agent prompt', () => { + const compiled = lockText(); + const runnerStep = compiled.match( + / - name: Prepare deterministic Cast validator runner\n[\s\S]*?(?=\n - (?:continue-on-error:|name:))/, + )?.[0] ?? ''; + const normalizedRunnerStep = runnerStep.replace(/\\"/g, '"'); + expect(normalizedRunnerStep).toContain('cat > "$validator_runner"'); + expect(normalizedRunnerStep).toContain('validator_expected_sha256="82aa5620d81e26513658fbde210b0f8d2ac3bc7572e672b421aaa17a2832e8cc"'); + expect(normalizedRunnerStep).toContain("outcome: 'cast_failure'"); + expect(normalizedRunnerStep).toContain('chmod 500 "$validator_runner"'); + expect(compiled.indexOf('name: Prepare deterministic Cast validator runner')).toBeLessThan( + compiled.indexOf('name: Restore inline skills from activation artifact'), + ); + }, 20000); + + it('keeps the v0.87.10 completion hook neutral when a custom safe job fails', () => { + const compiled = lockText(); + const completionStep = compiled.match( + / - name: Update reaction comment with completion status\n[\s\S]*?(?=\n detection:)/, + )?.[0] ?? ''; + expect(completionStep).toContain('GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}'); + expect(completionStep).toContain('GH_AW_SAFE_OUTPUTS_RESULT: ${{ needs.safe_outputs.result }}'); + expect(completionStep).not.toContain('needs.cast_failure.result'); + expect(completionStep).toContain( + 'This completion message does not indicate Cast success. For Cast, only a linked Cast pull request indicates success.', + ); + expect(completionStep).not.toMatch(/runSuccess[^\\n]*completed successfully/i); + }, 20000); + it('preserves the standalone release selection in the compiled install step (#1884)', () => { const compiled = lockText(); const pin = readText(join(SHARED_DIR, 'squad.md')).match( @@ -2530,6 +2602,56 @@ describe('gh-aw: Cast replaces disposable bootstrap state (#1909)', () => { const cast = squadContent.match( /## skill: `squad-cast`\n[\s\S]*?(?=\n## skill:|$)/, )?.[0] ?? ''; + const frontmatter = extractFrontmatter(SQUAD_WORKFLOW); + + function assertTruthfulCastTerminalContract(workflow: string): void { + const candidateCast = workflow.match( + /## skill: `squad-cast`\n[\s\S]*?(?=\n## skill:|$)/, + )?.[0] ?? ''; + const failureBranch = candidateCast.match( + /For outcomes 2 or 3[\s\S]*?(?=\n##### Step 8:)/, + )?.[0] ?? ''; + + expect(candidateCast).toMatch( + /only exit status zero with stdout exactly\s+`Cast validation passed\.` authorizes exactly one `create-pull-request`/s, + ); + expect(candidateCast).toContain('"${RUNNER_TEMP:?}/run-squad-cast-validator"'); + expect(candidateCast).toContain('Do not transcribe validator bytes or extraction commands'); + expect(candidateCast).toContain('emits one JSON\nrecord on stdout'); + expect(candidateCast.match( + /[\s\S]*?/, + )?.[0]).not.toMatch(/base64|gzip|awk|validator_expected_sha256/); + expect(failureBranch).toContain('## ❌ Cast did not complete'); + expect(failureBranch).toContain('No team or Cast pull request was delivered.'); + expect(failureBranch).toContain('complete stderr exactly as observed'); + expect(failureBranch).toContain('One `add-comment`'); + expect(failureBranch).toContain('One `cast_failure`'); + expect(failureBranch).toContain('identical `stage`, `command_category`,'); + expect(failureBranch).toContain('Never call `noop` or `report_incomplete`'); + expect(failureBranch).toContain('never\ncall `create-pull-request`'); + expect(failureBranch).toContain('rerun `/squad cast`'); + expect(failureBranch).toContain('Do not suggest re-materializing, reinstalling, or'); + expect(failureBranch).toContain('runs only when this output is emitted'); + expect(failureBranch).toContain('cannot detect that omission'); + expect(failureBranch).toContain('or independently\ngate `create-pull-request`'); + expect(failureBranch).toContain('cannot prevent a pull\nrequest that is materialized concurrently'); + expect(failureBranch).not.toMatch(/failure signal is independently enforced/i); + expect(failureBranch).not.toMatch(/(?:fail-closed|independent(?:ly)? fail-closed) gate/i); + expect(failureBranch).not.toMatch(/call `noop` and stop/i); + expect(failureBranch).not.toMatch(/re-?materialize the (?:committed )?(?:validator )?payload/i); + + for (const [stage, category] of [ + ['discovery', 'validator skill discovery'], + ['uniqueness', 'validator skill uniqueness'], + ['extraction', 'validator payload extraction'], + ['integrity', 'SHA-256 authentication'], + ['syntax', 'node --check'], + ['validation', 'validator execution'], + ]) { + expect(candidateCast).toContain(`\`${stage}\``); + expect(candidateCast).toContain(`\`${category}\``); + } + } it('uses an explicit fresh-artifact payload allowlist instead of staging .squad wholesale', () => { expect(cast).toContain('Never stage `.squad/` wholesale'); @@ -2581,13 +2703,58 @@ describe('gh-aw: Cast replaces disposable bootstrap state (#1909)', () => { expect(cast).toContain('must be self-contained for the final Cast tree'); }); - it('runs the deterministic final-tree validator immediately before safe output', () => { + it('enforces mutually exclusive factual Cast terminal outcomes', () => { expect(cast).toContain('Deterministic final-tree validation'); expect(cast).toContain('$RUNNER_TEMP/squad-cast-payload.json'); expect(cast).toContain('squad-cast-validator'); - expect(cast).toMatch(/Only a zero exit status.*authorizes.*`create-pull-request`/s); - expect(cast).toMatch(/exits nonzero.*`add-comment`.*`noop`/s); - expect(cast).toMatch(/does not expose an independent\s+post-agent hook/s); + assertTruthfulCastTerminalContract(squadContent); + }); + + it('configures one bounded durable Cast failure output that calls core.setFailed', () => { + const castFailureJob = frontmatter.match( + /^ jobs:\n cast-failure:[\s\S]*?(?=^ data:)/m, + )?.[0] ?? ''; + expect(castFailureJob).not.toBe(''); + expect(castFailureJob).toContain('max: 1'); + expect(castFailureJob).toContain('runs-on: ubuntu-slim'); + expect(castFailureJob).toContain("item.type === 'cast_failure'"); + expect(castFailureJob).toContain("item.type === 'create_pull_request'"); + expect(castFailureJob).toContain('cannot prevent a concurrently materialized pull request'); + expect(castFailureJob).toContain('process.env.GH_AW_AGENT_OUTPUT'); + expect(castFailureJob).toContain('core.setFailed'); + expect(castFailureJob).toContain('Cast did not complete.'); + expect(castFailureJob).toContain('failure.stderr'); + expect(castFailureJob).toContain("['validation', 'validator execution']"); + expect(castFailureJob).toContain("['syntax', 'node --check']"); + }); + + it('does not treat gh-aw run conclusion or report_incomplete as proof of Cast success', () => { + expect(frontmatter).toContain( + 'run-success: "🤖 [{workflow_name}]({run_url}) finished processing. This completion message does not indicate Cast success. For Cast, only a linked Cast pull request indicates success."', + ); + expect(cast).toContain('Built-in `report_incomplete` only warns'); + expect(cast).toContain('it does not fail the run'); + expect(cast).toContain('Only a linked Cast PR is the success signal'); + expect(cast).toContain('A red `cast_failure` job can therefore still select this\nneutral hook; it does not select `run-failure`'); + }); + + it('kills claims that the post-agent job independently enforces failure or PR authorization', () => { + const overclaimMutation = squadContent + .replace( + /When the agent emits `cast_failure`, the typed job[\s\S]*?gate `create-pull-request`\./, + 'The failure signal is independently enforced after the agent and fail-closed for create-pull-request.', + ); + expect(() => assertTruthfulCastTerminalContract(overclaimMutation)).toThrow(); + }); + + it('kills restoration of the old add-comment plus noop success-shaped path', () => { + const oldPathMutation = squadContent + .replace( + /- One `cast_failure` with the identical[\s\S]*?complete `stderr`\./, + '- Call `noop` and stop.', + ) + .replace('Never call `noop` or `report_incomplete`', 'Call `noop`'); + expect(() => assertTruthfulCastTerminalContract(oldPathMutation)).toThrow(); }); }); diff --git a/workflows/shared/squad-cast-validator.md b/workflows/shared/squad-cast-validator.md index 3381e6cc6..bbe2cdfe4 100644 --- a/workflows/shared/squad-cast-validator.md +++ b/workflows/shared/squad-cast-validator.md @@ -6,8 +6,8 @@ description: Store the reviewed GH-AW Cast final-tree validator as a disk-only p # GH-AW Cast final-tree validator This inline skill is a disk-only validator container. Do not invoke or load it -into model context. The dispatcher supplies a short command that locates this -materialized `SKILL.md` and extracts the compressed validator directly on disk. +into model context. Before the agent turn, the workflow prepares a runner that +locates this materialized `SKILL.md` and extracts the validator directly on disk. H4sIAAAAAAACCrU723LbOJbvqco/II6rScYSlcy87MqtaB3HPe0px8na7u3dkeQIIiEJbYpQA6BtRdQ87gfsJ+6XbB1cSJAiHc90rV8k4XLuOBcc+PWrXiZ4 diff --git a/workflows/squad.md b/workflows/squad.md index c26091311..af2aff929 100644 --- a/workflows/squad.md +++ b/workflows/squad.md @@ -48,10 +48,240 @@ tools: github: mode: gh-proxy toolsets: [default] +steps: + - name: Prepare deterministic Cast validator runner + shell: bash + run: | + set -euo pipefail + validator_runner="${RUNNER_TEMP:?}/run-squad-cast-validator" + cat > "$validator_runner" <<'SQUAD_CAST_VALIDATOR_RUNNER' + #!/usr/bin/env bash + set -u -o pipefail + cd "${GITHUB_WORKSPACE:?}" + + stderr_file="${RUNNER_TEMP:?}/squad-cast-validator.stderr" + validator_matches="${RUNNER_TEMP:?}/squad-cast-validator-matches.txt" + validator_script="${RUNNER_TEMP:?}/validate-gh-aw-cast.mjs" + validator_output="${RUNNER_TEMP:?}/squad-cast-validator.stdout" + expected_output="${RUNNER_TEMP:?}/squad-cast-validator.expected" + + fail_cast() { + local stage="$1" + local command_category="$2" + local exit_status="$3" + node - "$stage" "$command_category" "$exit_status" "$stderr_file" <<'NODE' + const fs = require('node:fs'); + const [stage, commandCategory, exitStatus, stderrPath] = process.argv.slice(2); + process.stdout.write(`${JSON.stringify({ + outcome: 'cast_failure', + stage, + command_category: commandCategory, + exit_status: exitStatus, + stderr: fs.readFileSync(stderrPath, 'utf8'), + })}\n`); + NODE + cat "$stderr_file" >&2 + exit "$exit_status" + } + + : > "$stderr_file" + find "${GITHUB_WORKSPACE}" -maxdepth 6 -name "SKILL.md" \ + -path "*/squad-cast-validator/SKILL.md" -print \ + > "$validator_matches" 2> "$stderr_file" + status=$? + if [ "$status" -ne 0 ]; then + fail_cast "discovery" "validator skill discovery" "$status" + fi + + validator_count="$(wc -l < "$validator_matches" | tr -d '[:space:]')" + case "$validator_count" in + 0) + printf 'Cast validator skill not found under GITHUB_WORKSPACE.\n' > "$stderr_file" + fail_cast "discovery" "validator skill discovery" "1" + ;; + 1) ;; + *) + { + printf 'Expected exactly one Cast validator skill, found %s:\n' "$validator_count" + sed 's/^/ /' "$validator_matches" + } > "$stderr_file" + fail_cast "uniqueness" "validator skill uniqueness" "1" + ;; + esac + validator_skill="$(sed -n '1p' "$validator_matches")" + + : > "$stderr_file" + { + awk ' + { sub(/\r$/, "", $0) } + $0 == "" { if (inside || seen) exit 41; inside = seen = 1; next } + $0 == "" { if (!inside || ended) exit 42; inside = 0; ended = 1; next } + inside { print } + END { if (!seen || inside || !ended) exit 43 } + ' "$validator_skill" \ + | tr -d '\r\n' \ + | base64 --decode \ + | gzip --decompress \ + | sed 's/\r$//' + } > "$validator_script" 2> "$stderr_file" + status=$? + if [ "$status" -ne 0 ]; then + printf '%s\n' \ + 'Cast validator payload extraction failed; expected one marker pair with valid base64+gzip data.' \ + >> "$stderr_file" + fail_cast "extraction" "validator payload extraction" "$status" + fi + + validator_script="$(cd "$(dirname "$validator_script")" && pwd -P)/$(basename "$validator_script")" + validator_expected_sha256="82aa5620d81e26513658fbde210b0f8d2ac3bc7572e672b421aaa17a2832e8cc" + : > "$stderr_file" + validator_actual_sha256="$( + node -e 'const c=require("node:crypto"),f=require("node:fs");process.stdout.write(c.createHash("sha256").update(f.readFileSync(process.argv[1])).digest("hex"))' \ + "$validator_script" 2> "$stderr_file" + )" + status=$? + if [ "$status" -ne 0 ]; then + fail_cast "integrity" "SHA-256 authentication" "$status" + fi + if [ "$validator_actual_sha256" != "$validator_expected_sha256" ]; then + printf 'Cast validator SHA-256 mismatch: expected %s, got %s.\n' \ + "$validator_expected_sha256" "$validator_actual_sha256" > "$stderr_file" + fail_cast "integrity" "SHA-256 authentication" "1" + fi + + : > "$stderr_file" + node --check "$validator_script" > /dev/null 2> "$stderr_file" + status=$? + if [ "$status" -ne 0 ]; then + fail_cast "syntax" "node --check" "$status" + fi + + : > "$stderr_file" + node "$validator_script" \ + --root "$PWD" \ + --payload "${RUNNER_TEMP:?}/squad-cast-payload.json" \ + > "$validator_output" 2> "$stderr_file" + status=$? + if [ "$status" -ne 0 ]; then + fail_cast "validation" "validator execution" "$status" + fi + + printf 'Cast validation passed.\n' > "$expected_output" + if ! cmp -s "$expected_output" "$validator_output"; then + validator_observed="$(cat "$validator_output")" + printf 'Cast validator did not emit the exact authorization output: <%s>\n' \ + "$validator_observed" > "$stderr_file" + fail_cast "validation" "validator execution" "1" + fi + cat "$validator_output" + SQUAD_CAST_VALIDATOR_RUNNER + chmod 500 "$validator_runner" safe-outputs: messages: append-only-comments: true + run-success: "🤖 [{workflow_name}]({run_url}) finished processing. This completion message does not indicate Cast success. For Cast, only a linked Cast pull request indicates success." + run-failure: "🤖 [{workflow_name}]({run_url}) {status}. The requested result was not delivered." pull-request-created: "🤖 Squad created [PR #{item_number}]({item_url}) for review. If its checks show `action_required`, approve the workflow run before merging." + jobs: + cast-failure: + name: Fail incomplete Cast + description: "Fail a Cast run from an emitted factual failure record." + runs-on: ubuntu-slim + max: 1 + inputs: + stage: + description: "Bounded Cast failure stage." + required: true + type: string + command_category: + description: "Exact command category for the stage." + required: true + type: string + exit_status: + description: "Observed command exit status, or unavailable when the command did not start." + required: true + type: string + stderr: + description: "Complete observed stderr without diagnosis or rewriting, including an empty string." + required: true + type: string + steps: + - name: Fail Cast terminal outcome + uses: actions/github-script@v9 + with: + script: | + const fs = await import('node:fs'); + const outputPath = process.env.GH_AW_AGENT_OUTPUT; + if (!outputPath) { + core.setFailed('GH_AW_AGENT_OUTPUT is unavailable for the Cast failure job.'); + return; + } + let output; + try { + output = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + } catch (error) { + core.setFailed(`Unable to read Cast failure output: ${error instanceof Error ? error.message : String(error)}`); + return; + } + if (!output || !Array.isArray(output.items)) { + core.setFailed('Cast failure output does not contain an items array.'); + return; + } + const malformedIndex = output.items.findIndex( + item => item === null || typeof item !== 'object' || Array.isArray(item), + ); + if (malformedIndex !== -1) { + core.setFailed(`Cast failure output item ${malformedIndex} is not an object.`); + return; + } + const failures = output.items.filter(item => item.type === 'cast_failure'); + if (failures.length !== 1) { + core.setFailed(`Expected exactly one cast_failure item, found ${failures.length}.`); + return; + } + const pullRequests = output.items.filter(item => item.type === 'create_pull_request'); + if (pullRequests.length > 0) { + core.setFailed([ + `Conflicting Cast terminal outputs: found ${pullRequests.length} create_pull_request item(s) with cast_failure.`, + 'This post-agent diagnostic cannot prevent a concurrently materialized pull request.', + ].join('\n')); + return; + } + + const categories = new Map([ + ['discovery', 'validator skill discovery'], + ['uniqueness', 'validator skill uniqueness'], + ['extraction', 'validator payload extraction'], + ['integrity', 'SHA-256 authentication'], + ['syntax', 'node --check'], + ['validation', 'validator execution'], + ]); + const failure = failures[0]; + if (!categories.has(failure.stage)) { + core.setFailed(`Invalid Cast failure stage: ${String(failure.stage)}`); + return; + } + if (failure.command_category !== categories.get(failure.stage)) { + core.setFailed(`Invalid command category for Cast failure stage ${failure.stage}.`); + return; + } + if (!/^(?:[0-9]{1,3}|unavailable)$/.test(String(failure.exit_status))) { + core.setFailed(`Invalid Cast failure exit status: ${String(failure.exit_status)}`); + return; + } + if (typeof failure.stderr !== 'string') { + core.setFailed('Cast failure stderr must be a string.'); + return; + } + + core.setFailed([ + 'Cast did not complete.', + `Stage: ${failure.stage}`, + `Command category: ${failure.command_category}`, + `Exit status: ${failure.exit_status}`, + 'Stderr:', + failure.stderr, + ].join('\n')); data: type: object properties: @@ -812,67 +1042,15 @@ its charter from scratch. Natural-language review is not the gate. Immediately before requesting safe output, create `$RUNNER_TEMP/squad-cast-payload.json` as a JSON array containing -every concrete Step 6 payload path, then run the exact command below. Do not -invoke or load `squad-cast-validator` into model context, and do not rewrite, -shorten, or substitute this command. The inline skill is already materialized -on disk by gh-aw before the agent runs. +every concrete Step 6 payload path, then invoke the prepared runner with the +exact command below. Do not transcribe validator bytes or extraction commands, +and do not invoke or load `squad-cast-validator` into model context. The runner +was prepared before this turn and deterministically consumes the validator +skill that gh-aw materialized on disk. ```bash -set -euo pipefail -cd "${GITHUB_WORKSPACE:?}" - -validator_matches="${RUNNER_TEMP:?}/squad-cast-validator-matches.txt" -find "${GITHUB_WORKSPACE}" -maxdepth 6 -name "SKILL.md" \ - -path "*/squad-cast-validator/SKILL.md" -print > "$validator_matches" -validator_count="$(wc -l < "$validator_matches" | tr -d '[:space:]')" -case "$validator_count" in - 0) echo "Cast validator skill not found under GITHUB_WORKSPACE." >&2; exit 1 ;; - 1) ;; - *) printf 'Expected exactly one Cast validator skill, found %s:\n' "$validator_count" >&2 - sed 's/^/ /' "$validator_matches" >&2 - exit 1 ;; -esac -validator_skill="$(sed -n '1p' "$validator_matches")" -validator_script="${RUNNER_TEMP:?}/validate-gh-aw-cast.mjs" -if ! awk ' - { sub(/\r$/, "", $0) } - $0 == "" { if (inside || seen) exit 41; inside = seen = 1; next } - $0 == "" { if (!inside || ended) exit 42; inside = 0; ended = 1; next } - inside { print } - END { if (!seen || inside || !ended) exit 43 } -' "$validator_skill" \ - | tr -d '\r\n' \ - | base64 --decode \ - | gzip --decompress \ - | sed 's/\r$//' \ - > "$validator_script"; then - echo "Cast validator payload extraction failed; expected one marker pair with valid base64+gzip data." >&2 - exit 1 -fi - -validator_script="$(cd "$(dirname "$validator_script")" && pwd -P)/$(basename "$validator_script")" -validator_expected_sha256="82aa5620d81e26513658fbde210b0f8d2ac3bc7572e672b421aaa17a2832e8cc" -validator_actual_sha256="$( - node -e 'const c=require("node:crypto"),f=require("node:fs");process.stdout.write(c.createHash("sha256").update(f.readFileSync(process.argv[1])).digest("hex"))' \ - "$validator_script" -)" -if [ "$validator_actual_sha256" != "$validator_expected_sha256" ]; then - printf 'Cast validator SHA-256 mismatch: expected %s, got %s.\n' \ - "$validator_expected_sha256" "$validator_actual_sha256" >&2 - exit 1 -fi -node --check "$validator_script" -validator_output="$( - node "$validator_script" \ - --root "$PWD" \ - --payload "${RUNNER_TEMP:?}/squad-cast-payload.json" -)" -printf '%s\n' "$validator_output" -if [ "$validator_output" != "Cast validation passed." ]; then - printf 'Cast validator did not emit the exact authorization output: <%s>\n' "$validator_output" >&2 - exit 1 -fi +"${RUNNER_TEMP:?}/run-squad-cast-validator" ``` @@ -884,19 +1062,86 @@ case-sensitively with the explicit payload and final tree; and rejects inactive/support roles, standalone templates/state, non-GH-AW clients, internal source paths, globs, placeholders, and fictional/inactive sample labels. -Only a zero exit status from this command with exact output -`Cast validation passed.` authorizes the -`create-pull-request` request. If the command is unavailable, cannot be -materialized exactly, or exits nonzero, post one actionable `add-comment` -containing its complete failure list and telling the user to rerun -`{canonical_command}`; call `noop` and stop. Do not call `create-pull-request`, -and do not describe partial output as a successful Cast. - -This is the strongest deterministic boundary available in gh-aw's single-agent -architecture: it runs against the agent's final working tree immediately before -the built-in safe-output request. gh-aw does not expose an independent -post-agent hook that can conditionally authorize `create-pull-request`, so do -not describe this as a post-agent or independently fail-closed gate. +Treat the validator command as one terminal branch point. On success it emits +only `Cast validation passed.`. On failure it exits nonzero, emits one JSON +record on stdout containing the bounded stage, command category, exit status, +and complete stderr, and repeats that stderr verbatim on stderr. Use that record +without rewriting it. Do not infer a root cause from the repository, generated +files, or the stage that failed. + +Exactly one of these mutually exclusive outcomes is allowed: + +1. **Validated success** — only exit status zero with stdout exactly + `Cast validation passed.` authorizes exactly one `create-pull-request` + request. Do not emit `cast_failure`, `report_incomplete`, `noop`, or a + failure `add-comment`. +2. **Validation rejected** — when the authenticated validator runs and exits + nonzero, do not request a PR. Classify the stage as `validation` and the + command category as `validator execution`. +3. **Validator unavailable or pre-validation failure** — when discovery, + uniqueness, extraction, integrity, or syntax does not complete, do not + request a PR. Classify only from the observed command boundary: + + | Stage | Command category | + |---|---| + | `discovery` | `validator skill discovery` | + | `uniqueness` | `validator skill uniqueness` | + | `extraction` | `validator payload extraction` | + | `integrity` | `SHA-256 authentication` | + | `syntax` | `node --check` | + +For outcomes 2 or 3, emit exactly two safe outputs: + +- One `add-comment` on the triggering issue with this factual shape: + + ````markdown + ## ❌ Cast did not complete + + No team or Cast pull request was delivered. + + - **Stage:** `{stage}` + - **Command category:** `{command_category}` + - **Exit status:** `{exit_status}` + - **Stderr:** + + ```text + {complete stderr exactly as observed, including empty output} + ``` + + **Next action:** After the reported failure is corrected or confirmed + transient, rerun `/squad cast`. Cast reruns remain safe and reuse any open + Cast PR. + ```` + +- One `cast_failure` with the identical `stage`, `command_category`, + `exit_status`, and complete `stderr`. The typed post-agent safe-output job + runs only when this output is emitted. It validates the bounded record and + calls `core.setFailed`, making the GitHub Actions run conclude `failure`. + +Then stop. Never call `noop` or `report_incomplete` on a Cast failure, never +call `create-pull-request`, and never describe Cast, team state, or delivery as +successful or complete. Do not suggest re-materializing, reinstalling, or +rewriting the committed validator payload. + +The PR authorization remains inside the agent turn. Pinned gh-aw v0.87.10 has no +unconditional post-agent verifier or conditional authorization hook for +`create-pull-request`. When the agent emits `cast_failure`, the typed job +independently validates that emitted bounded record and turns the run red. The +job is skipped when `cast_failure` is omitted, so it cannot detect that omission, +rerun the validator, inspect the discarded agent workspace, or independently +gate `create-pull-request`. It diagnoses a `cast_failure` and +`create_pull_request` emitted together, but the custom job and built-in safe +outputs run separately after the agent; this diagnosis cannot prevent a pull +request that is materialized concurrently. + +Built-in `report_incomplete` only warns and creates or updates a durable tracking +issue; it does not fail the run. The factual failure comment and, when emitted, +the red `cast_failure` job are the Cast failure signals. The generic completion +hook is deliberately neutral because pinned gh-aw chooses `run-success` from +the agent and consolidated `safe_outputs` results without considering the +custom job result. A red `cast_failure` job can therefore still select this +neutral hook; it does not select `run-failure`. Processing completion does not +indicate Cast success. Only a linked Cast PR is the success signal. ##### Step 8: Open PR