ci: faster test feedback — split API tests, parallelise common-utils, shard integration

.github/workflows/knip.yml

@@ -26,7 +26,7 @@ jobs:
           cache: 'yarn'
 
       - name: Install dependencies
-        run: yarn install
+        run: yarn install --immutable
 
       - name: Run Knip
         run:

.github/workflows/main.yml

@@ -41,14 +41,19 @@ jobs:
           cache-dependency-path: 'yarn.lock'
           cache: 'yarn'
       - name: Install root dependencies
-        run: yarn install
+        run: yarn install --immutable
       - name: Build dependencies
         run: make ci-build
       - name: Run unit tests
         run: make ci-unit
-  integration:
+  integration-shards:
+    name: Integration - Shard ${{ matrix.shard }}
     timeout-minutes: 16
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -59,17 +64,46 @@ jobs:
           cache-dependency-path: 'yarn.lock'
           cache: 'yarn'
       - name: Install root dependencies
-        run: yarn install
+        run: yarn install --immutable
       - name: Expose GitHub Runtime
         uses: crazy-max/ghaction-github-runtime@v4
       - name: Spin up docker services
         run: |
           docker buildx create --use --driver=docker-container
-          docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha" --set *.cache-from="type=gha" --load
+          docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha,mode=max,scope=ci-integration" --set *.cache-from="type=gha,scope=ci-integration" --load
       - name: Build dependencies
         run: make ci-build
+      # Integration tests boot a MockServer on fixed ports and share a single
+      # MongoDB/ClickHouse database, so they must stay serial (--runInBand)
+      # within a runner. We shard ACROSS runners instead: each shard is its own
+      # VM with its own Docker stack, and jest's --shard splits the test files
+      # so the shards run in parallel. --shard is forwarded through nx (args
+      # after `--` reach the underlying jest invocations).
+      #
+      # Runs inline (not `make ci-int`) so the compose project stays `hdx-ci`
+      # (matching the bake step above); make ci-int's `-p int-<slot>` is local
+      # multi-worktree isolation that CI doesn't need, and routing through it
+      # would reintroduce a project-name mismatch against bake. Teardown and
+      # log-archival are omitted intentionally — the runner is ephemeral and
+      # Actions streams stdout.
       - name: Run integration tests
-        run: make ci-int
+        run: |
+          docker compose -f ./docker-compose.ci.yml up -d --quiet-pull
+          npx nx run-many -t ci:int --parallel=false --output-style=stream -- --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+  # Stable aggregator context: `integration` is green only when all four shards
+  # pass. Merge-queue / required-status checks target this single context, since
+  # the per-shard contexts (Integration - Shard N) are not stable check names.
+  integration:
+    needs: [integration-shards]
+    if: always()
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - name: Verify all integration shards passed
+        if: needs.integration-shards.result != 'success'
+        run: |
+          echo "::error::One or more integration shards failed"
+          exit 1
   otel-unit-test:
     timeout-minutes: 8
     runs-on: ubuntu-24.04
@@ -264,7 +298,7 @@ jobs:
           cache: 'yarn'
 
       - name: Install dependencies
-        run: yarn install
+        run: yarn install --immutable
 
       - name: Build App
         run: yarn build:clickhouse

AGENTS.md

@@ -113,11 +113,14 @@ yarn ci:unit           # Run unit tests
 yarn dev:unit          # Watch mode for unit tests
 ```
 
-**packages/api** (integration tests only):
+**packages/api** (unit and integration tests):
 
 ```bash
-make dev-int-build                  # Build dependencies (run once before tests)
-make dev-int FILE=<TEST_FILE_NAME>  # Spins up Docker services and runs tests.
+cd packages/api
+yarn ci:unit                        # Run unit tests (no services needed)
+
+make dev-int-build                  # Build dependencies (run once before integration tests)
+make dev-int FILE=<TEST_FILE_NAME>  # Spins up Docker services and runs integration tests.
                                     # Ctrl-C to stop and wait for all services to tear down.
 ```
 

packages/api/jest.config.js

@@ -17,6 +17,7 @@ module.exports = {
   verbose: true,
   rootDir: './src',
   testMatch: ['**/__tests__/**/*.test.ts?(x)'],
+  testPathIgnorePatterns: ['<rootDir>/node_modules/', '.*\\.int\\.test\\.tsx?$'],
   testTimeout: 30000,
   moduleNameMapper: {
     '@/(.*)$': '<rootDir>/$1',

packages/api/jest.int.config.js

@@ -0,0 +1,10 @@
+const base = require('./jest.config.js');
+
+/** @type {import("jest").Config} **/
+module.exports = {
+  ...base,
+  testMatch: ['**/__tests__/**/*.int.test.ts?(x)'],
+  // Override the unit config's ignore list: it excludes `.int.test.ts`, which
+  // would otherwise hide every integration test from this suite.
+  testPathIgnorePatterns: ['<rootDir>/node_modules/'],
+};

packages/api/package.json

@@ -92,8 +92,10 @@
     "lint": "npx eslint --quiet . --ext .ts",
     "lint:fix": "npx eslint . --ext .ts --fix",
     "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:openapi",
-    "ci:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --runInBand --ci --forceExit",
-    "dev:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --runInBand --coverage --forceExit",
+    "ci:unit": "jest --ci --coverage",
+    "dev:unit": "jest --watch",
+    "ci:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --config jest.int.config.js --runInBand --ci --forceExit --passWithNoTests",
+    "dev:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --config jest.int.config.js --runInBand --coverage --forceExit",
     "dev:migrate-db-create": "ts-node node_modules/.bin/migrate-mongo create -f migrate-mongo-config.ts",
     "dev:migrate-db": "ts-node node_modules/.bin/migrate-mongo up -f migrate-mongo-config.ts",
     "dev:migrate-ch-create": "migrate create -ext sql -dir ./migrations/ch -seq",

packages/common-utils/package.json

@@ -46,9 +46,9 @@
     "lint": "npx eslint --quiet . --ext .ts",
     "lint:fix": "npx eslint . --ext .ts --fix",
     "ci:lint": "yarn lint && yarn tsc --noEmit",
-    "ci:unit": "jest --runInBand --ci --coverage",
-    "dev:unit": "jest --watchAll --runInBand",
-    "ci:int": "DOTENV_CONFIG_PATH=.env.test jest --config jest.int.config.js --runInBand --ci --coverage",
+    "ci:unit": "jest --ci --coverage",
+    "dev:unit": "jest --watchAll",
+    "ci:int": "DOTENV_CONFIG_PATH=.env.test jest --config jest.int.config.js --runInBand --ci --coverage --passWithNoTests",
     "dev:int": "DOTENV_CONFIG_PATH=.env.test jest --config jest.int.config.js --watchAll --runInBand"
   }
 }