diff --git a/test/gh-aw-plan-lifecycle.test.ts b/test/gh-aw-plan-lifecycle.test.ts index 084d02b1f..a7ad38b88 100644 --- a/test/gh-aw-plan-lifecycle.test.ts +++ b/test/gh-aw-plan-lifecycle.test.ts @@ -3,7 +3,8 @@ * * Guards the long-path planning lifecycle in `workflows/squad.md` against the * three defects tracked by #1758, the Owner/Agent cast-Name binding of #1759, - * and the structural research contract of #1756. + * the structural research contract of #1756, and adversarial validation from + * #1757. * * These assertions target the workflow's structural contract (labeled sections, * ordering hints, binding rules) rather than incidental prose — a single @@ -25,11 +26,26 @@ function readText(filePath: string): string { /** Slice a `## skill: \`name\`` block out of the workflow markdown. */ function skillBlock(markdown: string, name: string): string { - const start = markdown.indexOf(`## skill: \`${name}\``); + const marker = `## skill: \`${name}\``; + const start = markdown.indexOf(marker); if (start === -1) throw new Error(`skill block "${name}" not found`); - const rest = markdown.slice(start + 1); - const nextIdx = rest.indexOf('\n## skill: `'); - return nextIdx === -1 ? markdown.slice(start) : rest.slice(0, nextIdx); + const bodyStart = start + marker.length; + const nextSkillIdx = markdown.indexOf('\n## skill: `', bodyStart); + const endMarkerIdx = markdown.indexOf(`\n## end skill: \`${name}\``, bodyStart); + const candidates = [nextSkillIdx, endMarkerIdx].filter(index => index >= 0); + const endIdx = candidates.length === 0 ? -1 : Math.min(...candidates); + return endIdx === -1 ? markdown.slice(start) : markdown.slice(start, endIdx); +} + +/** Slice a `## agent: \`name\`` block out of the workflow markdown. */ +function agentBlock(markdown: string, name: string): string { + const marker = `## agent: \`${name}\``; + const start = markdown.indexOf(marker); + if (start === -1) throw new Error(`agent block "${name}" not found`); + const bodyStart = start + marker.length; + const rest = markdown.slice(bodyStart); + const nextH2 = rest.search(/\n## /); + return nextH2 === -1 ? rest : rest.slice(0, nextH2); } const squad = readText(SQUAD_WORKFLOW); @@ -344,6 +360,147 @@ describe('#1758.3: validate precedes both accept steps', () => { }); }); +// --------------------------------------------------------------------------- +// #1757 — validation owns an adversarial gate fed by Fact Checker DA evidence +// --------------------------------------------------------------------------- + +describe('#1757: squad-plan-validate has adversarial teeth', () => { + const validation = skillBlock(squad, 'squad-plan-validate'); + const factChecker = agentBlock(squad, 'fact-checker'); + const adversarialChecks = [ + 'Opposition steelman', + 'Load-bearing assumptions', + '30-day pre-mortem', + 'Alternative approach', + 'Remaining risk acceptance', + ]; + + function numberedChecks(block: string): Map { + return new Map( + [...block.matchAll(/^\|\s*(\d+)\s*\|\s*([^|]+?)\s*\|/gm)].map(match => [ + Number(match[1]), + match[2].trim(), + ]), + ); + } + + function hasSubstantiveAdversarialValidation(artifact: string): boolean { + const requiredSections = [ + 'Steelman of the opposition', + 'Load-bearing assumptions', + '30-day pre-mortem', + 'Alternative approach', + 'Remaining risk acceptance', + 'Validator Synthesis', + ]; + + const sectionsAreSubstantive = requiredSections.every((heading, index) => { + const escaped = heading.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const next = requiredSections + .slice(index + 1) + .map(item => item.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) + .join('|'); + const boundary = next ? `(?=\\n### (?:${next})|$)` : '$'; + const body = artifact.match(new RegExp(`### ${escaped}\\n([\\s\\S]*?)${boundary}`))?.[1] ?? ''; + return /\bWHAT:/i.test(body) && /\bWHY:/i.test(body) && /\bHOW:/i.test(body); + }); + const checkNumbers = [...artifact.matchAll(/^\|\s*(\d+)\s*\|/gm)].map(match => + Number(match[1]), + ); + + return ( + sectionsAreSubstantive && + checkNumbers.join(',') === '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16' && + /### 30-day pre-mortem[\s\S]*\b30 days?\b/i.test(artifact) && + /### Remaining risk acceptance[\s\S]*\bACCEPTED\b[\s\S]*\bowner:/i.test(artifact) && + /### Validator Synthesis[\s\S]*\bvalidation decision:/i.test(artifact) + ); + } + + it('extracts the validation skill with its complete marker and no following skill', () => { + expect(validation).toMatch(/^## skill: `squad-plan-validate`/); + expect(validation).not.toContain('## skill: `squad-plan-accept-scope`'); + }); + + it('preserves structural checks 1-10 and augments them with five adversarial checks', () => { + const checks = numberedChecks(validation); + expect([...checks.keys()].slice(0, 10)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect([...checks.values()].slice(10, 15)).toEqual(adversarialChecks); + expect(checks.get(16)).toBe('Validator synthesis'); + }); + + it('uses Fact Checker DA output as advisory evidence, never as the verdict owner', () => { + expect(validation).toMatch(/Use the `fact-checker` sub-agent exactly once/); + expect(validation).toMatch(/brief is input evidence, not a verdict/i); + expect(validation).toMatch(/Validation — not Fact Checker —[\s\S]*final verdict/); + expect(factChecker).not.toMatch(/^RESULT: (?:PASS|FAIL)$/m); + expect(factChecker).toMatch(/Never emit `RESULT: PASS`, `RESULT: FAIL`/); + }); + + it('requires all five DA elements with concrete semantic thresholds', () => { + for (const section of [ + '##### Steelman of the opposition', + '##### Load-bearing assumptions', + '##### 30-day pre-mortem', + '##### Alternative approach', + '##### Remaining risk acceptance', + ]) { + expect(factChecker, `Fact Checker must emit "${section}"`).toContain(section); + } + expect(validation).toMatch(/strongest credible opposition steelman/i); + expect(validation).toMatch(/falsifiable\s+conditions/); + expect(validation).toMatch(/exactly 30 days after\s+execution begins/); + expect(validation).toMatch(/materially different alternative approach\s+sketch/); + expect(validation).toMatch(/explicitly\s+`ACCEPTED` with rationale and an accountable owner/); + }); + + it('fails closed when adversarial evidence or verdict ownership is missing', () => { + expect(validation).toMatch( + /sub-agent is unavailable, errors, returns an empty brief,[\s\S]*force `RESULT: FAIL`/, + ); + expect(validation).toMatch(/copied verdict,[\s\S]*cannot become `RESULT: PASS`/); + expect(validation).toMatch(/Structural PASS alone cannot produce overall PASS/); + }); + + it('distinguishes a neatly formatted bad plan from a genuinely validated plan', () => { + const validationHeader = [ + '## ✅ Squad Plan Validation — PASSED', + 'RESULT: PASS', + '| Check | Status | Details |', + '|---|---|---|', + ]; + const structurallyValidBadPlan = [ + ...validationHeader, + ...Array.from({ length: 10 }, (_, index) => `| ${index + 1} | ✅ | — |`), + ].join('\n'); + + const genuinelyValidatedPlan = [ + ...validationHeader, + ...Array.from({ length: 16 }, (_, index) => `| ${index + 1} | ✅ | evidence |`), + '### Steelman of the opposition', + 'WHAT: Replace the batch design with streaming. WHY: the latency target makes batching unsafe. HOW: prove the target with a prototype.', + '### Load-bearing assumptions', + 'WHAT: queue ordering is stable. WHY: reordering breaks reconciliation. HOW: add an ordering probe before implementation.', + '### 30-day pre-mortem', + 'WHAT: The rollout is reverted in 30 days. WHY: retries amplify duplicate writes. HOW: ship idempotency keys and alerts.', + '### Alternative approach', + 'WHAT: Use a durable outbox. WHY: it narrows the consistency boundary. HOW: compare operational cost before choosing.', + '### Remaining risk acceptance', + 'WHAT: delayed delivery. WHY: the queue can lag. HOW: ACCEPTED with bounded SLO; owner: EECOM.', + '### Validator Synthesis', + 'WHAT: Structural and adversarial evidence align. WHY: the identified failure modes are bounded. HOW: validation decision: proceed.', + ].join('\n'); + + expect(hasSubstantiveAdversarialValidation(structurallyValidBadPlan)).toBe(false); + expect(hasSubstantiveAdversarialValidation(genuinelyValidatedPlan)).toBe(true); + }); + + it('keeps the sub-agent outside every skill block and uses safe nested headings', () => { + expect(squad).toContain('## end skill: `squad-plan-activate`\n\n## agent: `fact-checker`'); + expect(factChecker).not.toMatch(/^## (?!agent:)/m); + }); +}); + // --------------------------------------------------------------------------- // #1756 — structural research contract replaces the >=200-char floor // --------------------------------------------------------------------------- diff --git a/workflows/squad.md b/workflows/squad.md index 77b18c292..37e1c2be6 100644 --- a/workflows/squad.md +++ b/workflows/squad.md @@ -1244,7 +1244,10 @@ Set Implementation Plan = `✅ Done`, state = Implementation planned, next = `/s description: Validate the implementation plan and emit a PASS/FAIL validation artifact. --- -Readiness gate checking plan artifacts for structural issues. +Readiness gate checking plan artifacts for structural integrity and adversarial viability. +Validation owns the final `RESULT: PASS` or `RESULT: FAIL`. Fact Checker's +Devil's Advocate mode supplies advisory evidence to this gate; it never owns or +emits the validation verdict. **Acknowledge:** `🤖 Squad is validating the plan…` @@ -1270,8 +1273,14 @@ Find the latest `program`, `implementation`, and `triage` artifacts. At minimum | 8 | Orphaned items | Both | Triage items not in program | | 9 | Milestone gaps | Program | Epics not in any milestone | | 10 | Non-roster owner/agent | Both | An `Owner`/`Agent` value is absent from the roster (see below) | +| 11 | Opposition steelman | Adversarial | The strongest credible counter-argument is absent, weakened, or unanswered | +| 12 | Load-bearing assumptions | Adversarial | Assumptions whose failure invalidates the plan are missing, unfalsifiable, or lack impact | +| 13 | 30-day pre-mortem | Adversarial | No concrete causal failure scenario explains how this plan fails within 30 days | +| 14 | Alternative approach | Adversarial | No materially different approach is sketched with trade-offs against the chosen plan | +| 15 | Remaining risk acceptance | Adversarial | A remaining risk lacks explicit acceptance with rationale and an accountable owner | +| 16 | Validator synthesis | All evidence | The verdict is copied from advisory output, lacks independent synthesis, or advisory evidence is unavailable | -Severity: ❌ Critical (blocks acceptance): 1–6, 8, 10. ⚠️ Warning: 7, 9, borderline 5. +Severity: ❌ Critical (blocks acceptance): 1–6, 8, 10–16. ⚠️ Warning: 7, 9, borderline 5. ###### Check 10 — roster binding (the `Name` column is the sole source of truth) @@ -1289,13 +1298,60 @@ Run mechanically; never accept a value because it "looks like" a teammate. Never report a value as a valid roster name unless TG-2 emitted it as a `ROSTER_MEMBER:`. +###### Checks 11–15 — Fact Checker Devil's Advocate evidence + +Use the `fact-checker` sub-agent exactly once. Give it the complete latest +program, implementation, and triage artifacts and request its Devil's Advocate +brief. The brief is input evidence, not a verdict: Fact Checker is advisory and +must not emit `RESULT: PASS`, `RESULT: FAIL`, or decide whether acceptance may +proceed. + +Fail closed when the sub-agent is unavailable, errors, returns an empty brief, +or omits any required evidence. Do not invent or silently backfill missing +evidence. Report the affected check ❌ Critical and force `RESULT: FAIL`. + +Evaluate the returned evidence against the actual plan: + +1. **Check 11:** Require the strongest credible opposition steelman, not a + strawman. It must identify what the opposition would choose instead and why. +2. **Check 12:** Require load-bearing assumptions stated as falsifiable + conditions, with the plan impact if each is untrue. +3. **Check 13:** Require a concrete failure scenario exactly 30 days after + execution begins, including a causal chain and observable warning signs. +4. **Check 14:** Require at least one materially different alternative approach + sketch with its principal trade-offs against the chosen plan. +5. **Check 15:** Name every remaining risk and assign the validation gate's + disposition. PASS requires each remaining risk to be explicitly + `ACCEPTED` with rationale and an accountable owner, or eliminated by a plan + revision. `MITIGATION REQUIRED`, `REJECTED`, or an omitted disposition is + ❌ Critical. + +Each evidence item must state WHAT the challenge is, WHY it could invalidate or +outperform the plan, and HOW the plan addresses it or consciously accepts it. +A structurally clean plan with weak, generic, or missing adversarial evidence +fails checks 11–15. + +###### Check 16 — validator synthesis and verdict ownership + +After scoring checks 1–15, independently synthesize the structural findings, +Fact Checker evidence, plan responses, and remaining-risk dispositions. Explain +why the whole plan should or should not proceed. Validation — not Fact Checker — +then determines and emits the final verdict. + +Never forward an advisory conclusion as the gate decision. A Fact Checker +verdict, a copied verdict, unavailable advisory evidence, or a verdict without +this synthesis is Check 16 ❌ Critical and cannot become `RESULT: PASS`. + ##### Step 3: Post Result `add-comment` with `data: {"squad_artifact":"validation","schema_version":"1","origin_issue":{issue_number},"phases":[]}`. Keep `RESULT: PASS` or `RESULT: FAIL` in the human-readable body. -Structure: `## ✅/❌ Squad Plan Validation — PASSED/FAILED` → Validated artifacts + timestamp → Results table (Check|Status|Details) → Issues Found (Critical ❌ then Warnings ⚠️ with fix instructions) → Summary (counts + verdict) → Next action. +Structure: `## ✅/❌ Squad Plan Validation — PASSED/FAILED` → Validated artifacts + timestamp → Results table (Check|Status|Details; all checks 1–16) → Adversarial Evidence (Steelman of the opposition; Load-bearing assumptions; 30-day pre-mortem; Alternative approach; Remaining risk acceptance, each with WHAT/WHY/HOW) → Validator Synthesis (independent reasoning across structural and adversarial evidence) → Issues Found (Critical ❌ then Warnings ⚠️ with fix instructions) → Summary (counts + validator-owned verdict) → Next action. Rules: any ❌ = FAILED heading+verdict. Warnings alone ≠ failure. +Structural PASS alone cannot produce overall PASS. Overall PASS requires checks +1–16 to have run, complete adversarial evidence, explicit remaining-risk +acceptance, and validator-owned synthesis. ##### Step 4: Update Lifecycle @@ -1515,3 +1571,49 @@ Terminal (last phase): emit `data: {"squad_artifact":"activated","schema_version Phase: `🔄 Phase {N} of {total} activated`. Next: accept/activate next phase. Full/last: `✅ Done`, state = Activated. Terminal — no next action needed. + +## end skill: `squad-plan-activate` + +## agent: `fact-checker` +--- +description: "Produces advisory Devil's Advocate evidence for plan validation" +model: inherited +--- + +Operate only in Fact Checker's Devil's Advocate mode. Review the complete +program, implementation, and triage artifacts supplied by the parent validator. +Challenge the plan rigorously but constructively. Every finding states WHAT the +challenge is, WHY it could invalidate or outperform the plan, and HOW the plan +could address it. + +Return exactly these five evidence sections: + +##### Steelman of the opposition + +Give the strongest credible counter-argument and say what its advocates would +choose instead. Never substitute an easy-to-dismiss strawman. + +##### Load-bearing assumptions + +List falsifiable assumptions whose failure would invalidate the plan, and state +the plan impact of each failure. + +##### 30-day pre-mortem + +Describe a concrete failure exactly 30 days after execution begins, including +the causal chain and observable warning signs. + +##### Alternative approach + +Sketch at least one materially different approach and compare its principal +trade-offs with the chosen plan. + +##### Remaining risk acceptance + +Name the remaining risks and the conditions, rationale, and accountable owner +needed to accept or mitigate each one. + +This brief is advisory evidence only. Never emit `RESULT: PASS`, `RESULT: FAIL`, +or a final recommendation to accept/reject the plan. If an artifact or required +analysis is unavailable, write `EVIDENCE UNAVAILABLE: {reason}` in the affected +section instead of fabricating content.