Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .github/copilot-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,20 @@ If you make a decision that affects other team members, write it to:
.squad/decisions/inbox/copilot-{brief-slug}.md
```
The Scribe will merge it into the shared decisions file.

## Test Integrity

NEVER delete, skip, or weaken existing tests to make your code pass.

- If a test fails, fix the CODE, not the test.
- Never add `.skip`, `xit`, or `xdescribe` to bypass failures.
- Never change `expect(x).toBe('specific')` to `expect(x).toBeTruthy()` to avoid failures.
- Never lower coverage thresholds or test count baselines.

The only acceptable reasons to modify a test:
1. The test's expected behavior has intentionally changed (document why in commit message)
2. The test was genuinely wrong (explain in the commit message)

If you cannot make a test pass, report the failure in your PR — do NOT suppress it.

A CI guard will reject any PR that reduces the test count below the baseline in `.github/test-baseline.json`.
88 changes: 88 additions & 0 deletions .github/scripts/test-count-guard.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#!/usr/bin/env node
// Test Count Guard — CI script to prevent AI agents from deleting tests.
//
// Usage:
// node .github/scripts/test-count-guard.mjs --actual 243
// node .github/scripts/test-count-guard.mjs --from-json ./test-results.json
//
// Exit codes:
// 0 — actual >= baseline (pass), or baseline file missing (warning)
// 1 — actual < baseline (fail)

import { readFileSync } from 'fs';
import { resolve } from 'path';

function checkTestCount(actual, baseline) {
if (baseline === null) {
return { status: 'warning', message: 'No baseline file found. Skipping test count check (first run).' };
}
if (actual >= baseline.count) {
return { status: 'pass', message: `Test count OK: ${actual} >= ${baseline.count}` };
}
const delta = baseline.count - actual;
return {
status: 'fail',
message: `Test count decreased by ${delta}: expected >= ${baseline.count}, got ${actual}`,
delta,
};
}

function parseBaseline(raw) {
try {
const parsed = JSON.parse(raw);
if (typeof parsed.count !== 'number') throw new Error('count field missing or not a number');
return parsed;
} catch {
return null;
}
}

function loadBaseline(baselinePath) {
try {
const raw = readFileSync(baselinePath, 'utf8');
return parseBaseline(raw);
} catch {
return null;
}
}

const args = process.argv.slice(2);
const baselinePath = resolve(process.cwd(), '.github/test-baseline.json');
const baseline = loadBaseline(baselinePath);

let actual = null;

const actualIdx = args.indexOf('--actual');
if (actualIdx !== -1 && args[actualIdx + 1] !== undefined) {
actual = parseInt(args[actualIdx + 1], 10);
}

const fromJsonIdx = args.indexOf('--from-json');
if (fromJsonIdx !== -1 && args[fromJsonIdx + 1] !== undefined) {
try {
const jsonPath = resolve(process.cwd(), args[fromJsonIdx + 1]);
const data = JSON.parse(readFileSync(jsonPath, 'utf8'));
actual = data.numTotalTests ?? 0;
} catch {
console.error('❌ Failed to read test results JSON');
process.exit(1);
}
}

if (actual === null || isNaN(actual)) {
console.error('❌ No actual test count provided. Use --actual <count> or --from-json <path>');
process.exit(1);
}

const result = checkTestCount(actual, baseline);

if (result.status === 'pass') {
console.log(`✅ ${result.message}`);
process.exit(0);
} else if (result.status === 'warning') {
console.warn(`⚠️ ${result.message}`);
process.exit(0);
} else {
console.error(`❌ ${result.message}`);
process.exit(1);
}
6 changes: 6 additions & 0 deletions .github/test-baseline.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"_instructions": "To update: edit count, set updatedBy/updatedAt, document reason in commit message",
"count": 4662,
"updatedAt": "2025-07-25T00:00:00.000Z",
"updatedBy": "EECOM (initial baseline)"
}
6 changes: 5 additions & 1 deletion .github/workflows/squad-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,11 @@ jobs:
run: npm run build

- name: Run tests
run: npm test
run: npx vitest run --reporter=default --reporter=json --outputFile=test-results.json

- name: Test count guard
if: success() || failure()
run: node .github/scripts/test-count-guard.mjs --from-json test-results.json

publish-policy:
runs-on: ubuntu-latest
Expand Down
112 changes: 112 additions & 0 deletions test/ci/test-count-guard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
import { describe, it, expect } from 'vitest';

// Pure logic — inline so vitest (TypeScript) can test it without importing .mjs.
// The .mjs CLI script duplicates this same logic. This duplication is intentional:
// the .mjs file is a standalone CI script (pure ESM, no build step), and importing
// it here would require ESM interop complexity that isn't worth the coupling.

type BaselineResult =
| { status: 'pass'; message: string }
| { status: 'fail'; message: string; delta: number }
| { status: 'warning'; message: string };

type Baseline = { count: number; updatedAt: string; updatedBy: string } | null;

function checkTestCount(actual: number, baseline: Baseline): BaselineResult {
if (baseline === null) {
return { status: 'warning', message: 'No baseline file found. Skipping test count check (first run).' };
}
if (actual >= baseline.count) {
return { status: 'pass', message: `Test count OK: ${actual} >= ${baseline.count}` };
}
const delta = baseline.count - actual;
return {
status: 'fail',
message: `Test count decreased by ${delta}: expected >= ${baseline.count}, got ${actual}`,
delta,
};
}

function parseBaseline(raw: string): Baseline {
try {
const parsed = JSON.parse(raw);
if (typeof parsed.count !== 'number') throw new Error('count field missing or not a number');
return parsed as Baseline;
} catch {
return null;
}
}

const validBaseline: Baseline = {
count: 100,
updatedAt: '2025-01-01T00:00:00.000Z',
updatedBy: 'EECOM (test)',
};

describe('Test Count Guard', () => {
it('passes when actual count equals baseline', () => {
const result = checkTestCount(100, validBaseline);
expect(result.status).toBe('pass');
});

it('passes when actual count exceeds baseline', () => {
const result = checkTestCount(150, validBaseline);
expect(result.status).toBe('pass');
});

it('fails when actual count is below baseline', () => {
const result = checkTestCount(90, validBaseline);
expect(result.status).toBe('fail');
});

it('returns warning (not failure) when baseline file missing', () => {
const result = checkTestCount(100, null);
expect(result.status).toBe('warning');
});

it('returns failure message with delta when count decreases', () => {
const result = checkTestCount(90, validBaseline);
expect(result.status).toBe('fail');
if (result.status === 'fail') {
expect(result.delta).toBe(10);
expect(result.message).toContain('10');
expect(result.message).toContain('100');
}
});

it('parses valid baseline JSON', () => {
const raw = JSON.stringify({ count: 42, updatedAt: '2025-01-01T00:00:00.000Z', updatedBy: 'EECOM' });
const baseline = parseBaseline(raw);
expect(baseline).not.toBeNull();
expect(baseline?.count).toBe(42);
});

it('handles malformed baseline JSON gracefully', () => {
const baseline = parseBaseline('not-valid-json{{{');
expect(baseline).toBeNull();
// When baseline is null, checkTestCount warns rather than fails
const result = checkTestCount(100, null);
expect(result.status).toBe('warning');
});

it('handles baseline with count of 0', () => {
const zeroBaseline: Baseline = { count: 0, updatedAt: '2025-01-01T00:00:00.000Z', updatedBy: 'EECOM' };
const result = checkTestCount(0, zeroBaseline);
expect(result.status).toBe('pass');
});

it('returns null when count is a string instead of number', () => {
const baseline = parseBaseline('{"count":"fifty","updatedAt":"2025-01-01T00:00:00.000Z","updatedBy":"EECOM"}');
expect(baseline).toBeNull();
});

it('returns null when count field is missing', () => {
const baseline = parseBaseline('{"updatedAt":"2025-01-01T00:00:00.000Z","updatedBy":"EECOM"}');
expect(baseline).toBeNull();
});

it('fails when actual count is negative', () => {
const result = checkTestCount(-1, validBaseline);
expect(result.status).toBe('fail');
});
});
Loading