diff --git a/.github/workflows/squad-release-verify.yml b/.github/workflows/squad-release-verify.yml index aff33930a..7045aea93 100644 --- a/.github/workflows/squad-release-verify.yml +++ b/.github/workflows/squad-release-verify.yml @@ -64,6 +64,55 @@ concurrency: cancel-in-progress: false jobs: + verify-windows-platform: + name: Verify Windows-only release tests + runs-on: windows-latest + timeout-minutes: 20 + env: + SOURCE_SHA: ${{ inputs.source_sha || github.event.pull_request.head.sha }} + EVIDENCE_DIR: ${{ github.workspace }}\evidence-windows + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7 + with: + ref: ${{ inputs.source_sha || github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Assert checkout is exactly the requested SHA + shell: pwsh + run: | + if ($env:SOURCE_SHA -notmatch '^[0-9a-f]{40}$') { throw "source_sha must be a full lowercase 40-hex SHA, got '$env:SOURCE_SHA'" } + $head = git rev-parse HEAD + if ($head -ne $env:SOURCE_SHA) { throw "checkout HEAD $head != requested $env:SOURCE_SHA" } + New-Item -ItemType Directory -Force -Path $env:EVIDENCE_DIR | Out-Null + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 #v6 + with: + node-version: 22 + + - name: Install dependencies + run: npm ci + + - name: Build workspace packages + env: + SKIP_BUILD_BUMP: '1' + run: npm run build + + - name: Run Windows platform suite with JSON report + run: npx vitest run test/mcp-root-write.test.ts --reporter=default --reporter=json --outputFile="$env:EVIDENCE_DIR\vitest-report.json" + + - name: Require the Windows-only test to execute + if: always() + run: node scripts/release/test-reconciliation.mjs --report "$env:EVIDENCE_DIR\vitest-report.json" --out "$env:EVIDENCE_DIR" --platform win32 --require-platform-executions --fail-on-unexplained + + - name: Upload Windows platform evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 #v7 + with: + name: release-windows-platform-evidence-${{ env.SOURCE_SHA }} + path: evidence-windows/ + retention-days: 90 + if-no-files-found: warn + verify: name: Verify exact SHA and build evidence runs-on: ubuntu-latest diff --git a/scripts/release/build-manifest.mjs b/scripts/release/build-manifest.mjs index d56dd0f32..b98443308 100644 --- a/scripts/release/build-manifest.mjs +++ b/scripts/release/build-manifest.mjs @@ -158,7 +158,7 @@ export function buildManifest(opts) { }, packages: { '@bradygaster/squad-sdk': pkgEntry(sdk, '@bradygaster/squad-sdk'), '@bradygaster/squad-cli': pkgEntry(cli, '@bradygaster/squad-cli') }, lockfiles: { files: lockReport.files.length, registryEntries: lockReport.entries, violations: lockReport.violations.length, deniedPatterns: lockReport.deniedPatterns, ok: lockReport.ok, externalNpmCi: npmCi }, - tests: recon ? { source: 'vitest JSON report of this run (scripts/release/test-reconciliation.mjs)', numTotalTests: recon.summary.numTotalTests, byStatus: recon.summary.byStatus, byCategory: recon.summary.byCategory, countsSum: recon.summary.countsSum, unlistedTests: recon.summary.unlistedTests, skippedFiles: recon.summary.skippedFiles, reconciliationFile: 'test-reconciliation.json' } + tests: recon ? { source: 'vitest JSON report of this run (scripts/release/test-reconciliation.mjs)', platform: recon.summary.platform, numTotalTests: recon.summary.numTotalTests, byStatus: recon.summary.byStatus, byCategory: recon.summary.byCategory, countsSum: recon.summary.countsSum, unlistedTests: recon.summary.unlistedTests, skippedFiles: recon.summary.skippedFiles, reconciliationFile: 'test-reconciliation.json' } : { source: `console summary of cited CI run ${citedRun.id} (no vitest JSON report available to this generator)`, numTotalTests: citedRun.consoleSummary.testsTotal, byStatus: { passed: citedRun.consoleSummary.testsPassed, xfail: citedRun.consoleSummary.expectedFailures, skipped: citedRun.consoleSummary.testsSkipped, todo: citedRun.consoleSummary.todos }, byCategory: {}, countsSum: citedRun.consoleSummary.testsPassed + citedRun.consoleSummary.expectedFailures + citedRun.consoleSummary.testsSkipped + citedRun.consoleSummary.todos === citedRun.consoleSummary.testsTotal, unlistedTests: citedRun.consoleSummary.testsTotal - (citedRun.consoleSummary.testsPassed + citedRun.consoleSummary.expectedFailures + citedRun.consoleSummary.testsSkipped + citedRun.consoleSummary.todos), skippedFiles: [], reconciliationFile: null }, closure, behavioralEvidence: { diff --git a/scripts/release/manifest.schema.json b/scripts/release/manifest.schema.json index fc5e3bfad..4dbd1ce58 100644 --- a/scripts/release/manifest.schema.json +++ b/scripts/release/manifest.schema.json @@ -48,7 +48,7 @@ }, "tests": { "type": "object", "additionalProperties": false, "required": ["source", "numTotalTests", "byStatus", "byCategory", "countsSum", "unlistedTests", "skippedFiles"], - "properties": { "source": { "type": "string" }, "numTotalTests": { "type": "integer" }, "byStatus": { "type": "object" }, "byCategory": { "type": "object" }, "countsSum": { "type": "boolean" }, "unlistedTests": { "type": "integer" }, "skippedFiles": { "type": "array", "items": { "type": "string" } }, "reconciliationFile": { "type": ["string", "null"] } } + "properties": { "source": { "type": "string" }, "platform": { "type": "string" }, "numTotalTests": { "type": "integer" }, "byStatus": { "type": "object" }, "byCategory": { "type": "object" }, "countsSum": { "type": "boolean" }, "unlistedTests": { "type": "integer" }, "skippedFiles": { "type": "array", "items": { "type": "string" } }, "reconciliationFile": { "type": ["string", "null"] } } }, "closure": { "type": "object", "additionalProperties": false, diff --git a/scripts/release/test-reconciliation.mjs b/scripts/release/test-reconciliation.mjs index 09ff8bc30..b5ab46ef9 100644 --- a/scripts/release/test-reconciliation.mjs +++ b/scripts/release/test-reconciliation.mjs @@ -20,7 +20,7 @@ * unexplained anything else, including tests with no terminal state * * node scripts/release/test-reconciliation.mjs --report vitest.json --out evidence [--baseline prior.json] - * [--fail-on-unexplained] + * [--platform ] [--require-platform-executions] [--fail-on-unexplained] */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; @@ -37,6 +37,20 @@ export const CATEGORY_RULES = Object.freeze([ { category: 'env-gated', file: /squad-20-real-clients-stdio\.e2e\.test\.ts$/, reason: 'SQUAD_REAL_CLI_TESTS=1 not set / host CLI binaries absent' }, { category: 'env-gated', file: /squad-20-host-feature-matrix\.test\.ts$/, reason: 'live host evidence requires the real CLI binaries' }, { category: 'env-gated', file: /mcp-spec-init\.test\.ts$/, title: /squad\.exe/, reason: 'it.runIf(win32): Windows-only PATH/.exe resolution' }, + { + category: 'env-gated', + file: 'test/mcp-root-write.test.ts', + title: '[platform: win32] removes the legacy raw uppercase Windows drive hash while preserving unrelated entries', + runsOn: 'win32', + reason: "it.runIf(process.platform === 'win32'): Windows-only legacy drive-hash reconciliation", + }, + { + category: 'env-gated', + file: 'test/mcp-root-write.test.ts', + title: '[platform: non-win32] removes the legacy raw POSIX symlink hash while preserving unrelated entries', + runsOn: 'non-win32', + reason: "it.runIf(process.platform !== 'win32'): POSIX-only legacy symlink-hash reconciliation", + }, { category: 'env-gated', file: /profile-boundary\.test\.ts$/, reason: 'host lacks 8.3 short names or symlink/junction privilege' }, { category: 'env-gated', file: /state-backend\.test\.ts$/, reason: 'platform-gated state backend case' }, { category: 'env-gated', file: /docs-build\.test\.ts$/, reason: 'docs dependencies not installed' }, @@ -58,7 +72,26 @@ function relFile(name) { return i >= 0 ? norm.slice(i + 1) : norm; } -export function reconcile(report, { baseline } = {}) { +function matches(value, expected) { + return typeof expected === 'string' ? value === expected : expected.test(value); +} + +function runsOnPlatform(rule, platform) { + return rule.runsOn === 'win32' ? platform === 'win32' : rule.runsOn === 'non-win32' ? platform !== 'win32' : true; +} + +export function resolvePlatform(claimedPlatform, actualPlatform = process.platform) { + const platform = claimedPlatform ?? actualPlatform; + if (!['aix', 'darwin', 'freebsd', 'linux', 'openbsd', 'sunos', 'win32'].includes(platform)) { + throw new Error(`unsupported reconciliation platform '${platform}'`); + } + if (claimedPlatform && claimedPlatform !== actualPlatform) { + throw new Error(`claimed reconciliation platform '${claimedPlatform}' does not match execution platform '${actualPlatform}'`); + } + return platform; +} + +export function reconcile(report, { baseline, platform = process.platform, requirePlatformExecutions = false } = {}) { const tests = []; const files = { total: report.testResults.length, byStatus: {} }; for (const tr of report.testResults) { @@ -84,10 +117,14 @@ export function reconcile(report, { baseline } = {}) { } else if (t.status === 'todo') { category = TODO_BLOCKERS.includes(t.title) ? 'todo-blocking' : 'todo'; reason = 'it.todo placeholder'; } else { for (const rule of CATEGORY_RULES) { - if (!rule.file.test(t.file)) continue; + if (!matches(t.file, rule.file)) continue; if (rule.status && rule.status !== t.status) continue; - if (rule.title && !rule.title.test(t.fullName) && !rule.title.test(t.title)) continue; + if (rule.title && !matches(t.title, rule.title) && !matches(t.fullName, rule.title)) continue; if (!rule.status && t.status === 'xfail') continue; + if (rule.runsOn && runsOnPlatform(rule, platform)) { + reason = `expected to execute on ${rule.runsOn}, but report status was '${t.status}'`; + break; + } category = rule.category; reason = rule.reason; break; } if (t.status === 'failed') { category = 'failed'; reason = t.failureMessages[0]?.split('\n')[0] ?? 'failed'; } @@ -100,13 +137,26 @@ export function reconcile(report, { baseline } = {}) { const bucketSum = Object.values(byStatus).reduce((a, b) => a + b, 0); const listed = tests.length; const unexplainedTotal = report.numTotalTests - listed; + const requiredExecutions = requirePlatformExecutions + ? CATEGORY_RULES.filter((rule) => rule.runsOn && runsOnPlatform(rule, platform)).map((rule) => { + const test = tests.find((candidate) => matches(candidate.file, rule.file) && matches(candidate.title, rule.title)); + return { + file: rule.file, + title: rule.title, + platform: rule.runsOn, + status: test?.status ?? 'missing', + ok: test?.status === 'passed', + }; + }) + : []; const summary = { - numTotalTests: report.numTotalTests, listed, bucketSum, byStatus, byCategory, + platform, numTotalTests: report.numTotalTests, listed, bucketSum, byStatus, byCategory, countsSum: bucketSum === listed && listed === report.numTotalTests, unlistedTests: unexplainedTotal, + requiredExecutions, files, skippedFiles: report.testResults.filter((tr) => tr.assertionResults.length > 0 && tr.assertionResults.every((a) => a.status === 'skipped' || a.status === 'todo')).map((tr) => relFile(tr.name)), - ok: report.numFailedTests === 0 && (byCategory.unexplained ?? 0) === 0 && (byCategory['todo-blocking'] ?? 0) === 0 && unexplainedTotal === 0 && bucketSum === report.numTotalTests, + ok: report.numFailedTests === 0 && (byCategory.unexplained ?? 0) === 0 && (byCategory['todo-blocking'] ?? 0) === 0 && unexplainedTotal === 0 && bucketSum === report.numTotalTests && requiredExecutions.every((entry) => entry.ok), }; let diff = null; if (baseline) { @@ -118,6 +168,11 @@ export function reconcile(report, { baseline } = {}) { return { generatedAt: new Date().toISOString(), summary, nonPassing: categorized.sort((a, b) => a.file.localeCompare(b.file) || a.fullName.localeCompare(b.fullName)), diffVsBaseline: diff }; } +export function shouldFail(result, { failOnUnexplained = false } = {}) { + const s = result.summary; + return failOnUnexplained ? !s.ok : (s.numTotalTests > 0 && (s.byStatus.failed ?? 0) > 0); +} + if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const arg = (k) => { const i = process.argv.indexOf(k); return i >= 0 ? process.argv[i + 1] : undefined; }; const reportPath = path.resolve(arg('--report') ?? 'vitest-report.json'); @@ -125,13 +180,25 @@ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.me const outDir = path.resolve(arg('--out') ?? 'evidence'); mkdirSync(outDir, { recursive: true }); const baseline = arg('--baseline') ? JSON.parse(readFileSync(path.resolve(arg('--baseline')), 'utf8')) : undefined; - const result = reconcile(JSON.parse(readFileSync(reportPath, 'utf8')), { baseline }); + let platform; + try { + platform = resolvePlatform(arg('--platform')); + } catch (error) { + console.error(`::error::${error.message}`); + process.exit(2); + } + const result = reconcile(JSON.parse(readFileSync(reportPath, 'utf8')), { + baseline, + platform, + requirePlatformExecutions: process.argv.includes('--require-platform-executions'), + }); writeFileSync(path.join(outDir, 'test-reconciliation.json'), JSON.stringify(result, null, 2) + '\n'); const s = result.summary; - console.log(`tests=${s.numTotalTests} listed=${s.listed} byStatus=${JSON.stringify(s.byStatus)} byCategory=${JSON.stringify(s.byCategory)} unlisted=${s.unlistedTests} skippedFiles=${s.skippedFiles.join(',') || '-'}`); + console.log(`platform=${s.platform} tests=${s.numTotalTests} listed=${s.listed} byStatus=${JSON.stringify(s.byStatus)} byCategory=${JSON.stringify(s.byCategory)} unlisted=${s.unlistedTests} skippedFiles=${s.skippedFiles.join(',') || '-'}`); for (const t of result.nonPassing.filter((t) => t.category === 'unexplained' || t.category === 'failed' || t.category === 'todo-blocking')) console.error(`::warning::${t.category}: ${t.file} :: ${t.fullName} [${t.status}]`); + for (const required of s.requiredExecutions.filter((entry) => !entry.ok)) console.error(`::error::required platform test did not pass: ${required.file} :: ${required.title} [${required.status}]`); if (!s.countsSum || s.unlistedTests !== 0) console.error(`::error::${s.unlistedTests} test(s) are counted in numTotalTests but have no listed result (never reached a terminal state); see ${path.join(outDir, 'test-reconciliation.json')}`); - const fail = process.argv.includes('--fail-on-unexplained') ? !s.ok : (s.numTotalTests > 0 && (s.byStatus.failed ?? 0) > 0); + const fail = shouldFail(result, { failOnUnexplained: process.argv.includes('--fail-on-unexplained') }); console.log(`${s.ok ? '✅' : '⚠️'} reconciliation written to ${outDir}`); process.exit(fail ? 1 : 0); } diff --git a/test/mcp-root-write.test.ts b/test/mcp-root-write.test.ts index b63a9265a..d0eabf2c8 100644 --- a/test/mcp-root-write.test.ts +++ b/test/mcp-root-write.test.ts @@ -327,7 +327,7 @@ describe('iter-8 mcp-root: repo-root .mcp.json writer + project tombstone', () = }); it.runIf(process.platform === 'win32')( - 'removes the legacy raw uppercase Windows drive hash while preserving unrelated entries', + '[platform: win32] removes the legacy raw uppercase Windows drive hash while preserving unrelated entries', () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), 'squad-home-win-legacy-')); try { @@ -363,7 +363,7 @@ describe('iter-8 mcp-root: repo-root .mcp.json writer + project tombstone', () = ); it.runIf(process.platform !== 'win32')( - 'removes the legacy raw POSIX symlink hash while preserving unrelated entries', + '[platform: non-win32] removes the legacy raw POSIX symlink hash while preserving unrelated entries', () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), 'squad-home-posix-legacy-')); const links = fs.mkdtempSync(path.join(os.tmpdir(), 'squad-links-posix-legacy-')); diff --git a/test/test-reconciliation.test.ts b/test/test-reconciliation.test.ts new file mode 100644 index 000000000..db4f51e52 --- /dev/null +++ b/test/test-reconciliation.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest'; +import { reconcile, resolvePlatform, shouldFail } from '../scripts/release/test-reconciliation.mjs'; + +function reportFor( + file: string, + title: string, + status: 'passed' | 'skipped', +) { + return { + numTotalTests: 1, + numFailedTests: 0, + testResults: [{ + name: file, + status: 'passed', + assertionResults: [{ + title, + fullName: `iter-8 mcp-root: repo-root .mcp.json writer + project tombstone ${title}`, + status, + failureMessages: [], + }], + }], + }; +} + +describe('release test reconciliation', () => { + it('classifies the self-described Windows-only skip in a Linux-shaped report', () => { + const title = '[platform: win32] removes the legacy raw uppercase Windows drive hash while preserving unrelated entries'; + const result = reconcile(reportFor('/home/runner/work/squad/squad/test/mcp-root-write.test.ts', title, 'skipped'), { platform: 'linux' }); + + expect(result.summary).toMatchObject({ + numTotalTests: 1, + listed: 1, + byStatus: { skipped: 1 }, + byCategory: { 'env-gated': 1 }, + countsSum: true, + unlistedTests: 0, + platform: 'linux', + ok: true, + }); + expect(result.nonPassing).toEqual([ + expect.objectContaining({ + file: 'test/mcp-root-write.test.ts', + title, + status: 'skipped', + category: 'env-gated', + reason: "it.runIf(process.platform === 'win32'): Windows-only legacy drive-hash reconciliation", + }), + ]); + expect(shouldFail(result, { failOnUnexplained: true })).toBe(false); + }); + + it('fails when the Windows-only test is skipped on Windows', () => { + const title = '[platform: win32] removes the legacy raw uppercase Windows drive hash while preserving unrelated entries'; + const result = reconcile(reportFor('D:\\a\\squad\\squad\\test\\mcp-root-write.test.ts', title, 'skipped'), { platform: 'win32' }); + + expect(result.summary).toMatchObject({ + platform: 'win32', + byCategory: { unexplained: 1 }, + ok: false, + }); + expect(result.nonPassing[0]).toMatchObject({ + category: 'unexplained', + reason: "expected to execute on win32, but report status was 'skipped'", + }); + expect(shouldFail(result, { failOnUnexplained: true })).toBe(true); + }); + + it('requires the Windows-only test to be present and passed in Windows evidence', () => { + const title = '[platform: win32] removes the legacy raw uppercase Windows drive hash while preserving unrelated entries'; + const result = reconcile(reportFor('D:\\a\\squad\\squad\\test\\mcp-root-write.test.ts', title, 'passed'), { + platform: 'win32', + requirePlatformExecutions: true, + }); + + expect(result.summary.requiredExecutions).toEqual([{ + file: 'test/mcp-root-write.test.ts', + title, + platform: 'win32', + status: 'passed', + ok: true, + }]); + expect(result.summary.ok).toBe(true); + expect(shouldFail(result, { failOnUnexplained: true })).toBe(false); + }); + + it('fails Windows evidence when the required Windows-only test is absent', () => { + const result = reconcile(reportFor('D:\\a\\squad\\squad\\test\\other.test.ts', 'ordinary passing test', 'passed'), { + platform: 'win32', + requirePlatformExecutions: true, + }); + + expect(result.summary.requiredExecutions).toEqual([ + expect.objectContaining({ platform: 'win32', status: 'missing', ok: false }), + ]); + expect(result.summary.ok).toBe(false); + expect(shouldFail(result, { failOnUnexplained: true })).toBe(true); + }); + + it('fails when the non-Windows-only test is skipped on Linux', () => { + const title = '[platform: non-win32] removes the legacy raw POSIX symlink hash while preserving unrelated entries'; + const result = reconcile(reportFor('/home/runner/work/squad/squad/test/mcp-root-write.test.ts', title, 'skipped'), { platform: 'linux' }); + + expect(result.summary).toMatchObject({ + byCategory: { unexplained: 1 }, + ok: false, + }); + }); + + it('classifies the non-Windows-only skip as env-gated on Windows', () => { + const title = '[platform: non-win32] removes the legacy raw POSIX symlink hash while preserving unrelated entries'; + const result = reconcile(reportFor('D:\\a\\squad\\squad\\test\\mcp-root-write.test.ts', title, 'skipped'), { platform: 'win32' }); + + expect(result.summary).toMatchObject({ + byCategory: { 'env-gated': 1 }, + ok: true, + }); + }); + + it('keeps a truly unexplained skip release-blocking', () => { + const result = reconcile(reportFor('/home/runner/work/squad/squad/test/other.test.ts', 'silently skips work', 'skipped'), { platform: 'linux' }); + + expect(result.summary).toMatchObject({ + byCategory: { unexplained: 1 }, + ok: false, + }); + expect(result.nonPassing[0]).toMatchObject({ + category: 'unexplained', + reason: 'no reviewed rule matches this test', + }); + expect(shouldFail(result, { failOnUnexplained: true })).toBe(true); + }); + + it('fails a wrong-platform title claim instead of accepting a near match', () => { + const wrongTitle = '[platform: non-win32] removes the legacy raw uppercase Windows drive hash while preserving unrelated entries'; + const result = reconcile(reportFor('/home/runner/work/squad/squad/test/mcp-root-write.test.ts', wrongTitle, 'skipped'), { platform: 'linux' }); + + expect(result.nonPassing[0]).toMatchObject({ + category: 'unexplained', + reason: 'no reviewed rule matches this test', + }); + expect(result.summary.ok).toBe(false); + }); + + it('rejects a platform claim that differs from the execution host', () => { + expect(() => resolvePlatform('linux', 'win32')).toThrow( + "claimed reconciliation platform 'linux' does not match execution platform 'win32'", + ); + }); +});