From 8d8c4f6ce068c0328a83c6fb02a171db1a0de21b Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Thu, 26 Mar 2026 05:41:48 -0700 Subject: [PATCH] feat(ci): add test count guard to prevent AI agents from deleting tests Adds a CI step that compares the current test count against a baseline stored in .github/test-baseline.json. PRs that reduce the test count below baseline are rejected. Also updates copilot-instructions.md with explicit 'never delete tests' directive for @copilot and other AI agents. Closes #39 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 17 ++++ .github/scripts/test-count-guard.mjs | 88 +++++++++++++++++++++ .github/test-baseline.json | 6 ++ .github/workflows/squad-ci.yml | 6 +- test/ci/test-count-guard.test.ts | 112 +++++++++++++++++++++++++++ 5 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/test-count-guard.mjs create mode 100644 .github/test-baseline.json create mode 100644 test/ci/test-count-guard.test.ts diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f1528d90d..a3284ada7 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -73,3 +73,20 @@ If you make a decision that affects other team members, write it to: .squad/decisions/inbox/copilot-{brief-slug}.md ``` The Scribe will merge it into the shared decisions file. + +## Test Integrity + +NEVER delete, skip, or weaken existing tests to make your code pass. + +- If a test fails, fix the CODE, not the test. +- Never add `.skip`, `xit`, or `xdescribe` to bypass failures. +- Never change `expect(x).toBe('specific')` to `expect(x).toBeTruthy()` to avoid failures. +- Never lower coverage thresholds or test count baselines. + +The only acceptable reasons to modify a test: +1. The test's expected behavior has intentionally changed (document why in commit message) +2. The test was genuinely wrong (explain in the commit message) + +If you cannot make a test pass, report the failure in your PR — do NOT suppress it. + +A CI guard will reject any PR that reduces the test count below the baseline in `.github/test-baseline.json`. diff --git a/.github/scripts/test-count-guard.mjs b/.github/scripts/test-count-guard.mjs new file mode 100644 index 000000000..0915adf88 --- /dev/null +++ b/.github/scripts/test-count-guard.mjs @@ -0,0 +1,88 @@ +#!/usr/bin/env node +// Test Count Guard — CI script to prevent AI agents from deleting tests. +// +// Usage: +// node .github/scripts/test-count-guard.mjs --actual 243 +// node .github/scripts/test-count-guard.mjs --from-json ./test-results.json +// +// Exit codes: +// 0 — actual >= baseline (pass), or baseline file missing (warning) +// 1 — actual < baseline (fail) + +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +function checkTestCount(actual, baseline) { + if (baseline === null) { + return { status: 'warning', message: 'No baseline file found. Skipping test count check (first run).' }; + } + if (actual >= baseline.count) { + return { status: 'pass', message: `Test count OK: ${actual} >= ${baseline.count}` }; + } + const delta = baseline.count - actual; + return { + status: 'fail', + message: `Test count decreased by ${delta}: expected >= ${baseline.count}, got ${actual}`, + delta, + }; +} + +function parseBaseline(raw) { + try { + const parsed = JSON.parse(raw); + if (typeof parsed.count !== 'number') throw new Error('count field missing or not a number'); + return parsed; + } catch { + return null; + } +} + +function loadBaseline(baselinePath) { + try { + const raw = readFileSync(baselinePath, 'utf8'); + return parseBaseline(raw); + } catch { + return null; + } +} + +const args = process.argv.slice(2); +const baselinePath = resolve(process.cwd(), '.github/test-baseline.json'); +const baseline = loadBaseline(baselinePath); + +let actual = null; + +const actualIdx = args.indexOf('--actual'); +if (actualIdx !== -1 && args[actualIdx + 1] !== undefined) { + actual = parseInt(args[actualIdx + 1], 10); +} + +const fromJsonIdx = args.indexOf('--from-json'); +if (fromJsonIdx !== -1 && args[fromJsonIdx + 1] !== undefined) { + try { + const jsonPath = resolve(process.cwd(), args[fromJsonIdx + 1]); + const data = JSON.parse(readFileSync(jsonPath, 'utf8')); + actual = data.numTotalTests ?? 0; + } catch { + console.error('❌ Failed to read test results JSON'); + process.exit(1); + } +} + +if (actual === null || isNaN(actual)) { + console.error('❌ No actual test count provided. Use --actual or --from-json '); + process.exit(1); +} + +const result = checkTestCount(actual, baseline); + +if (result.status === 'pass') { + console.log(`✅ ${result.message}`); + process.exit(0); +} else if (result.status === 'warning') { + console.warn(`⚠️ ${result.message}`); + process.exit(0); +} else { + console.error(`❌ ${result.message}`); + process.exit(1); +} diff --git a/.github/test-baseline.json b/.github/test-baseline.json new file mode 100644 index 000000000..1eb5df875 --- /dev/null +++ b/.github/test-baseline.json @@ -0,0 +1,6 @@ +{ + "_instructions": "To update: edit count, set updatedBy/updatedAt, document reason in commit message", + "count": 4662, + "updatedAt": "2025-07-25T00:00:00.000Z", + "updatedBy": "EECOM (initial baseline)" +} diff --git a/.github/workflows/squad-ci.yml b/.github/workflows/squad-ci.yml index 8cd153634..832d2290e 100644 --- a/.github/workflows/squad-ci.yml +++ b/.github/workflows/squad-ci.yml @@ -107,7 +107,11 @@ jobs: run: npm run build - name: Run tests - run: npm test + run: npx vitest run --reporter=default --reporter=json --outputFile=test-results.json + + - name: Test count guard + if: success() || failure() + run: node .github/scripts/test-count-guard.mjs --from-json test-results.json publish-policy: runs-on: ubuntu-latest diff --git a/test/ci/test-count-guard.test.ts b/test/ci/test-count-guard.test.ts new file mode 100644 index 000000000..188eec4be --- /dev/null +++ b/test/ci/test-count-guard.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest'; + +// Pure logic — inline so vitest (TypeScript) can test it without importing .mjs. +// The .mjs CLI script duplicates this same logic. This duplication is intentional: +// the .mjs file is a standalone CI script (pure ESM, no build step), and importing +// it here would require ESM interop complexity that isn't worth the coupling. + +type BaselineResult = + | { status: 'pass'; message: string } + | { status: 'fail'; message: string; delta: number } + | { status: 'warning'; message: string }; + +type Baseline = { count: number; updatedAt: string; updatedBy: string } | null; + +function checkTestCount(actual: number, baseline: Baseline): BaselineResult { + if (baseline === null) { + return { status: 'warning', message: 'No baseline file found. Skipping test count check (first run).' }; + } + if (actual >= baseline.count) { + return { status: 'pass', message: `Test count OK: ${actual} >= ${baseline.count}` }; + } + const delta = baseline.count - actual; + return { + status: 'fail', + message: `Test count decreased by ${delta}: expected >= ${baseline.count}, got ${actual}`, + delta, + }; +} + +function parseBaseline(raw: string): Baseline { + try { + const parsed = JSON.parse(raw); + if (typeof parsed.count !== 'number') throw new Error('count field missing or not a number'); + return parsed as Baseline; + } catch { + return null; + } +} + +const validBaseline: Baseline = { + count: 100, + updatedAt: '2025-01-01T00:00:00.000Z', + updatedBy: 'EECOM (test)', +}; + +describe('Test Count Guard', () => { + it('passes when actual count equals baseline', () => { + const result = checkTestCount(100, validBaseline); + expect(result.status).toBe('pass'); + }); + + it('passes when actual count exceeds baseline', () => { + const result = checkTestCount(150, validBaseline); + expect(result.status).toBe('pass'); + }); + + it('fails when actual count is below baseline', () => { + const result = checkTestCount(90, validBaseline); + expect(result.status).toBe('fail'); + }); + + it('returns warning (not failure) when baseline file missing', () => { + const result = checkTestCount(100, null); + expect(result.status).toBe('warning'); + }); + + it('returns failure message with delta when count decreases', () => { + const result = checkTestCount(90, validBaseline); + expect(result.status).toBe('fail'); + if (result.status === 'fail') { + expect(result.delta).toBe(10); + expect(result.message).toContain('10'); + expect(result.message).toContain('100'); + } + }); + + it('parses valid baseline JSON', () => { + const raw = JSON.stringify({ count: 42, updatedAt: '2025-01-01T00:00:00.000Z', updatedBy: 'EECOM' }); + const baseline = parseBaseline(raw); + expect(baseline).not.toBeNull(); + expect(baseline?.count).toBe(42); + }); + + it('handles malformed baseline JSON gracefully', () => { + const baseline = parseBaseline('not-valid-json{{{'); + expect(baseline).toBeNull(); + // When baseline is null, checkTestCount warns rather than fails + const result = checkTestCount(100, null); + expect(result.status).toBe('warning'); + }); + + it('handles baseline with count of 0', () => { + const zeroBaseline: Baseline = { count: 0, updatedAt: '2025-01-01T00:00:00.000Z', updatedBy: 'EECOM' }; + const result = checkTestCount(0, zeroBaseline); + expect(result.status).toBe('pass'); + }); + + it('returns null when count is a string instead of number', () => { + const baseline = parseBaseline('{"count":"fifty","updatedAt":"2025-01-01T00:00:00.000Z","updatedBy":"EECOM"}'); + expect(baseline).toBeNull(); + }); + + it('returns null when count field is missing', () => { + const baseline = parseBaseline('{"updatedAt":"2025-01-01T00:00:00.000Z","updatedBy":"EECOM"}'); + expect(baseline).toBeNull(); + }); + + it('fails when actual count is negative', () => { + const result = checkTestCount(-1, validBaseline); + expect(result.status).toBe('fail'); + }); +});