diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0f779d9fc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,41 @@ +# Dependencies (installed inside container) +node_modules + +# Build outputs +dist +coverage +ctrf + +# Git +.git +.gitignore + +# Worktrees +.worktrees + +# IDE / Editor +.vscode +.idea +*.swp +*.swo + +# Environment files (secrets ā€” not needed in build context) +.env +.env.* +!.env.example + +# Documentation (not needed in images) +docs + +# Deployment manifests (not part of app images) +deploy +infra + +# Scripts (except dev-entrypoint.sh which is COPY'd explicitly) +# scripts/ is kept since dev-entrypoint.sh lives there + +# Misc +tmp +*.md +!README.md +LICENSE diff --git a/.env.base b/.env.base new file mode 100644 index 000000000..4869466ee --- /dev/null +++ b/.env.base @@ -0,0 +1,27 @@ +# ============================================================================= +# Base environment for Docker Compose services +# ============================================================================= +# These are the default values used by docker-compose.yml. +# In worktrees, worktree-env adds a per-worktree offset (1-99) to each +# numeric port and appends - to string values, allowing +# multiple compositions to run simultaneously. +# +# Requirements for base ports: +# - Must end in 00 (2+ trailing zeros) for full 100-offset range +# - Ending in 0 (1 trailing zero) triggers a warning (only 10 offsets) +# - No trailing zeros causes an error (no room for offsets) +# ============================================================================= + +COMPOSE_PROJECT_NAME=scope-mt-app + +MONGODB_PORT=27000 +REDIS_PORT=6300 +LOWKEY_VAULT_PORT=8400 +AZURITE_BLOB_PORT=10100 +AZURITE_QUEUE_PORT=10200 +AZURITE_TABLE_PORT=10300 +API_PORT=3100 +JUDGE_PORT=3200 +TOKEN_MANAGER_PORT=3300 +PORTAL_PORT=5100 +DEVPROXY_API_PORT=18800 diff --git a/.env.example b/.env.example index 37e33c98f..734aef25d 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,13 @@ # # When services run INSIDE Docker Compose, the compose file overrides these # with Docker-internal hostnames (e.g. mongodb instead of localhost). +# +# WORKTREE PORT OFFSETS: +# When working in a git worktree, run `source scripts/worktree-env.sh` (or any +# `pnpm docker:*` command) to auto-generate a managed section at the bottom +# of .env with offset ports. Base ports are defined in .env.base. +# The offset is persisted in .port-offset (gitignored, auto-cleaned on +# `git worktree remove`). Do NOT edit the managed section manually. # ============================================================================= # --------------------------------------------------------------------------- @@ -21,26 +28,26 @@ SCOPE_MT_API_URL=http://localhost:3100 # Infrastructure connections (for host-native runs) # --------------------------------------------------------------------------- -# MongoDB (host port 27117 matches docker-compose mapping) -MONGO_CONNECTION_STRING=mongodb://localhost:27117 +# MongoDB (host port 27000 matches docker-compose mapping) +MONGO_CONNECTION_STRING=mongodb://localhost:27000 MONGO_DATABASE=requests-db MONGO_COLLECTION=requests # Redis REDIS_HOST=localhost -REDIS_PORT=6379 +REDIS_PORT=6300 REDIS_PASSWORD= # Azure Storage (Azurite) AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1 -STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;QueueEndpoint=http://127.0.0.1:10101/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10100/devstoreaccount1; +STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;QueueEndpoint=http://127.0.0.1:10200/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10100/devstoreaccount1; # Queue names (used by API to route to workers) AZURE_STORAGE_QUEUE_WORKER_1=queue-coder-acp-claude-code AZURE_STORAGE_QUEUE_WORKER_2=queue-coder-acp-copilot # Judge service URL (when running a worker natively while judge is in Docker) -JUDGE_SERVICE_URL=http://localhost:3001 +JUDGE_SERVICE_URL=http://localhost:3200 # --------------------------------------------------------------------------- # Per-worker queue name (set when running a specific worker natively) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..c1965c216 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.github/workflows/*.lock.yml linguist-generated=true merge=ours \ No newline at end of file diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index c000179ef..ec0869e8b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -7,11 +7,27 @@ on: description: 'Comma-separated list of images to build (or "all")' required: false default: 'all' + tag: + description: 'Image tag to use (default: dev)' + required: false + default: 'dev' push: branches: [main, dev] paths: - - 'packages/**' - - 'Dockerfile.*' + - 'apps/**' + - 'packages/shared/**' + - 'packages/model-scanning/**' + - 'packages/db-migrations/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'tsconfig.base.json' + pull_request: + branches: [dev] + paths: + - 'apps/**' + - 'packages/shared/**' + - 'packages/model-scanning/**' + - 'packages/db-migrations/**' - 'package.json' - 'pnpm-lock.yaml' - 'tsconfig.base.json' @@ -19,12 +35,13 @@ on: permissions: id-token: write contents: read + pull-requests: read env: jobs: detect-changes: - if: github.event_name == 'push' + if: github.event_name == 'push' || github.event_name == 'pull_request' runs-on: ubuntu-latest outputs: api: ${{ steps.changes.outputs.api }} @@ -32,52 +49,105 @@ jobs: portal: ${{ steps.changes.outputs.portal }} coder-acp-copilot: ${{ steps.changes.outputs.coder-acp-copilot }} coder-acp-claude-code: ${{ steps.changes.outputs.coder-acp-claude-code }} + token-manager: ${{ steps.changes.outputs.token-manager }} + model-scanner-copilot: ${{ steps.changes.outputs.model-scanner-copilot }} + model-scanner-anthropic: ${{ steps.changes.outputs.model-scanner-anthropic }} + report-generator: ${{ steps.changes.outputs.report-generator }} + db-migrations: ${{ steps.changes.outputs.db-migrations }} shared: ${{ steps.changes.outputs.shared }} + model-scanning: ${{ steps.changes.outputs.model-scanning }} steps: - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: changes with: - filters: | - shared: - - 'packages/shared/**' - - 'package.json' - - 'pnpm-lock.yaml' - - 'tsconfig.base.json' - api: - - 'packages/api/**' - - 'Dockerfile.api' - judge: - - 'packages/judge/**' - - 'Dockerfile.judge' - portal: - - 'packages/portal/**' - - 'Dockerfile.portal' - coder-acp-copilot: - - 'packages/workers/coder-acp-copilot/**' - - 'Dockerfile.coder-acp-copilot' - coder-acp-claude-code: - - 'packages/workers/coder-acp-claude-code/**' - - 'Dockerfile.coder-acp-claude-code' + fetch-depth: 0 + - name: Detect changed paths + id: changes + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + else + BASE_SHA="${{ github.event.before }}" + fi + CHANGED=$(git diff --name-only "$BASE_SHA" "${{ github.sha }}" || true) + + check_paths() { + local name="$1" + shift + for pattern in "$@"; do + if echo "$CHANGED" | grep -qE "$pattern"; then + echo "${name}=true" >> "$GITHUB_OUTPUT" + return + fi + done + echo "${name}=false" >> "$GITHUB_OUTPUT" + } + + check_paths shared '^packages/shared/' '^package\.json$' '^pnpm-lock\.yaml$' '^tsconfig\.base\.json$' + check_paths model-scanning '^packages/model-scanning/' + check_paths api '^apps/api/' + check_paths judge '^apps/judge/' + check_paths portal '^apps/portal/' + check_paths coder-acp-copilot '^apps/workers/coder-acp-copilot/' + check_paths coder-acp-claude-code '^apps/workers/coder-acp-claude-code/' + check_paths token-manager '^apps/token-manager/' + check_paths model-scanner-copilot '^apps/model-scanners/copilot/' + check_paths model-scanner-anthropic '^apps/model-scanners/anthropic/' + check_paths report-generator '^apps/workers/report-generator/' + check_paths db-migrations '^packages/db-migrations/' build-on-push: - if: github.event_name == 'push' + if: >- + github.event_name == 'push' && ( + needs.detect-changes.outputs.api == 'true' || + needs.detect-changes.outputs.judge == 'true' || + needs.detect-changes.outputs.portal == 'true' || + needs.detect-changes.outputs.coder-acp-copilot == 'true' || + needs.detect-changes.outputs.coder-acp-claude-code == 'true' || + needs.detect-changes.outputs.token-manager == 'true' || + needs.detect-changes.outputs.model-scanner-copilot == 'true' || + needs.detect-changes.outputs.model-scanner-anthropic == 'true' || + needs.detect-changes.outputs.report-generator == 'true' || + needs.detect-changes.outputs.db-migrations == 'true' || + needs.detect-changes.outputs.shared == 'true' || + needs.detect-changes.outputs.model-scanning == 'true' + ) needs: detect-changes runs-on: ubuntu-latest + environment: ${{ github.ref_name == 'main' && 'production' || 'integration' }} strategy: fail-fast: false matrix: image: - name: api - changed: ${{ needs.detect-changes.outputs.api == 'true' || needs.detect-changes.outputs.shared == 'true' }} + dockerfile: apps/api/Dockerfile + changed: ${{ (needs.detect-changes.outputs.api == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} - name: judge - changed: ${{ needs.detect-changes.outputs.judge == 'true' || needs.detect-changes.outputs.shared == 'true' }} + dockerfile: apps/judge/Dockerfile + changed: ${{ (needs.detect-changes.outputs.judge == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} - name: portal - changed: ${{ needs.detect-changes.outputs.portal == 'true' || needs.detect-changes.outputs.shared == 'true' }} + dockerfile: apps/portal/Dockerfile + changed: ${{ (needs.detect-changes.outputs.portal == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} - name: coder-acp-copilot - changed: ${{ needs.detect-changes.outputs.coder-acp-copilot == 'true' || needs.detect-changes.outputs.shared == 'true' }} + dockerfile: apps/workers/coder-acp-copilot/Dockerfile + changed: ${{ (needs.detect-changes.outputs.coder-acp-copilot == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} - name: coder-acp-claude-code - changed: ${{ needs.detect-changes.outputs.coder-acp-claude-code == 'true' || needs.detect-changes.outputs.shared == 'true' }} + dockerfile: apps/workers/coder-acp-claude-code/Dockerfile + changed: ${{ (needs.detect-changes.outputs.coder-acp-claude-code == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: token-manager + dockerfile: apps/token-manager/Dockerfile + changed: ${{ (needs.detect-changes.outputs.token-manager == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: model-scanner-copilot + dockerfile: apps/model-scanners/copilot/Dockerfile + changed: ${{ (needs.detect-changes.outputs.model-scanner-copilot == 'true' || needs.detect-changes.outputs.shared == 'true' || needs.detect-changes.outputs.model-scanning == 'true') && 'true' || 'false' }} + - name: model-scanner-anthropic + dockerfile: apps/model-scanners/anthropic/Dockerfile + changed: ${{ (needs.detect-changes.outputs.model-scanner-anthropic == 'true' || needs.detect-changes.outputs.shared == 'true' || needs.detect-changes.outputs.model-scanning == 'true') && 'true' || 'false' }} + - name: report-generator + dockerfile: apps/workers/report-generator/Dockerfile + changed: ${{ (needs.detect-changes.outputs.report-generator == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: db-migrations + dockerfile: packages/db-migrations/Dockerfile + changed: ${{ (needs.detect-changes.outputs.db-migrations == 'true') && 'true' || 'false' }} exclude: - image: changed: 'false' @@ -91,19 +161,150 @@ jobs: tenant-id: ${{ vars.AZURE_TENANT_ID }} subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Determine environment tag + id: env-tag + run: | + if [[ "${{ github.ref_name }}" == "main" ]]; then + echo "tag=prod" >> "$GITHUB_OUTPUT" + echo "overlay=prod" >> "$GITHUB_OUTPUT" + else + echo "tag=dev" >> "$GITHUB_OUTPUT" + echo "overlay=integration" >> "$GITHUB_OUTPUT" + fi + + - name: Load pinned versions + id: versions + run: | + EXTRA_ARGS="" + for f in apps/workers/${{ matrix.image.name }}/versions.env apps/${{ matrix.image.name }}/versions.env; do + if [ -f "$f" ]; then + while IFS='=' read -r key value; do + [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue + EXTRA_ARGS="${EXTRA_ARGS} --build-arg ${key}=${value}" + done < "$f" + break + fi + done + echo "extra_args=${EXTRA_ARGS}" >> $GITHUB_OUTPUT + - name: Build and push to ACR run: | + SHORT_SHA="${{ github.sha }}" + SHORT_SHA="${SHORT_SHA:0:7}" + ENV_TAG="${{ steps.env-tag.outputs.tag }}" + TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ) + BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) az acr build \ --registry "${{ vars.ACR_NAME }}" \ --resource-group "${{ vars.ACR_RESOURCE_GROUP }}" \ --image "scoped/${{ matrix.image.name }}:${{ github.sha }}" \ - --image "scoped/${{ matrix.image.name }}:latest" \ - --file "Dockerfile.${{ matrix.image.name }}" \ + --image "scoped/${{ matrix.image.name }}:${ENV_TAG}-latest" \ + --image "scoped/${{ matrix.image.name }}:${ENV_TAG}-${TIMESTAMP}-${SHORT_SHA}" \ + --build-arg GIT_COMMIT=${{ github.sha }} \ + --build-arg BUILD_TIME=${BUILD_TIME} \ + ${{ steps.versions.outputs.extra_args }} \ + --file "${{ matrix.image.dockerfile }}" \ + . + + build-on-pr: + if: >- + github.event_name == 'pull_request' && ( + needs.detect-changes.outputs.api == 'true' || + needs.detect-changes.outputs.judge == 'true' || + needs.detect-changes.outputs.portal == 'true' || + needs.detect-changes.outputs.coder-acp-copilot == 'true' || + needs.detect-changes.outputs.coder-acp-claude-code == 'true' || + needs.detect-changes.outputs.token-manager == 'true' || + needs.detect-changes.outputs.model-scanner-copilot == 'true' || + needs.detect-changes.outputs.model-scanner-anthropic == 'true' || + needs.detect-changes.outputs.report-generator == 'true' || + needs.detect-changes.outputs.db-migrations == 'true' || + needs.detect-changes.outputs.shared == 'true' || + needs.detect-changes.outputs.model-scanning == 'true' + ) + needs: detect-changes + runs-on: ubuntu-latest + environment: integration + strategy: + fail-fast: false + matrix: + image: + - name: api + dockerfile: apps/api/Dockerfile + changed: ${{ (needs.detect-changes.outputs.api == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: judge + dockerfile: apps/judge/Dockerfile + changed: ${{ (needs.detect-changes.outputs.judge == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: portal + dockerfile: apps/portal/Dockerfile + changed: ${{ (needs.detect-changes.outputs.portal == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: coder-acp-copilot + dockerfile: apps/workers/coder-acp-copilot/Dockerfile + changed: ${{ (needs.detect-changes.outputs.coder-acp-copilot == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: coder-acp-claude-code + dockerfile: apps/workers/coder-acp-claude-code/Dockerfile + changed: ${{ (needs.detect-changes.outputs.coder-acp-claude-code == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: token-manager + dockerfile: apps/token-manager/Dockerfile + changed: ${{ (needs.detect-changes.outputs.token-manager == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: model-scanner-copilot + dockerfile: apps/model-scanners/copilot/Dockerfile + changed: ${{ (needs.detect-changes.outputs.model-scanner-copilot == 'true' || needs.detect-changes.outputs.shared == 'true' || needs.detect-changes.outputs.model-scanning == 'true') && 'true' || 'false' }} + - name: model-scanner-anthropic + dockerfile: apps/model-scanners/anthropic/Dockerfile + changed: ${{ (needs.detect-changes.outputs.model-scanner-anthropic == 'true' || needs.detect-changes.outputs.shared == 'true' || needs.detect-changes.outputs.model-scanning == 'true') && 'true' || 'false' }} + - name: report-generator + dockerfile: apps/workers/report-generator/Dockerfile + changed: ${{ (needs.detect-changes.outputs.report-generator == 'true' || needs.detect-changes.outputs.shared == 'true') && 'true' || 'false' }} + - name: db-migrations + dockerfile: packages/db-migrations/Dockerfile + changed: ${{ (needs.detect-changes.outputs.db-migrations == 'true') && 'true' || 'false' }} + exclude: + - image: + changed: 'false' + steps: + - uses: actions/checkout@v4 + + - name: Log in to Azure + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Load pinned versions + id: versions + run: | + EXTRA_ARGS="" + for f in apps/workers/${{ matrix.image.name }}/versions.env apps/${{ matrix.image.name }}/versions.env; do + if [ -f "$f" ]; then + while IFS='=' read -r key value; do + [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue + EXTRA_ARGS="${EXTRA_ARGS} --build-arg ${key}=${value}" + done < "$f" + break + fi + done + echo "extra_args=${EXTRA_ARGS}" >> $GITHUB_OUTPUT + + - name: Build and push to ACR + run: | + BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) + az acr build \ + --registry "${{ vars.ACR_NAME }}" \ + --resource-group "${{ vars.ACR_RESOURCE_GROUP }}" \ + --image "scoped/${{ matrix.image.name }}:${{ github.event.pull_request.head.sha }}" \ + --image "scoped/${{ matrix.image.name }}:pr-${{ github.event.pull_request.number }}-latest" \ + --build-arg GIT_COMMIT=${{ github.event.pull_request.head.sha }} \ + --build-arg BUILD_TIME=${BUILD_TIME} \ + ${{ steps.versions.outputs.extra_args }} \ + --file "${{ matrix.image.dockerfile }}" \ . build-manual: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest + environment: ${{ github.ref_name == 'main' && 'production' || 'integration' }} strategy: fail-fast: false matrix: @@ -120,6 +321,21 @@ jobs: echo "build=false" >> "$GITHUB_OUTPUT" fi + - name: Compute Dockerfile path + id: dockerfile + if: steps.check.outputs.build == 'true' + run: | + case "${{ matrix.image }}" in + api|judge|portal) DOCKERFILE="apps/${{ matrix.image }}/Dockerfile" ;; + coder-*) DOCKERFILE="apps/workers/${{ matrix.image }}/Dockerfile" ;; + report-generator) DOCKERFILE="apps/workers/${{ matrix.image }}/Dockerfile" ;; + model-scanner-copilot) DOCKERFILE="apps/model-scanners/copilot/Dockerfile" ;; + model-scanner-anthropic) DOCKERFILE="apps/model-scanners/anthropic/Dockerfile" ;; + db-migrations) DOCKERFILE="packages/db-migrations/Dockerfile" ;; + *) DOCKERFILE="apps/${{ matrix.image }}/Dockerfile" ;; + esac + echo "path=$DOCKERFILE" >> "$GITHUB_OUTPUT" + - name: Log in to Azure if: steps.check.outputs.build == 'true' uses: azure/login@v2 @@ -128,13 +344,37 @@ jobs: tenant-id: ${{ vars.AZURE_TENANT_ID }} subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Load pinned versions + id: versions + run: | + EXTRA_ARGS="" + for f in apps/workers/${{ matrix.image }}/versions.env apps/${{ matrix.image }}/versions.env; do + if [ -f "$f" ]; then + while IFS='=' read -r key value; do + [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue + EXTRA_ARGS="${EXTRA_ARGS} --build-arg ${key}=${value}" + done < "$f" + break + fi + done + echo "extra_args=${EXTRA_ARGS}" >> $GITHUB_OUTPUT + - name: Build and push to ACR if: steps.check.outputs.build == 'true' run: | + SHORT_SHA="${{ github.sha }}" + SHORT_SHA="${SHORT_SHA:0:7}" + TAG="${{ github.event.inputs.tag }}" + TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ) + BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) az acr build \ --registry "${{ vars.ACR_NAME }}" \ --resource-group "${{ vars.ACR_RESOURCE_GROUP }}" \ --image "scoped/${{ matrix.image }}:${{ github.sha }}" \ - --image "scoped/${{ matrix.image }}:latest" \ - --file "Dockerfile.${{ matrix.image }}" \ + --image "scoped/${{ matrix.image }}:${TAG}-latest" \ + --image "scoped/${{ matrix.image }}:${TAG}-${TIMESTAMP}-${SHORT_SHA}" \ + --build-arg GIT_COMMIT=${{ github.sha }} \ + --build-arg BUILD_TIME=${BUILD_TIME} \ + ${{ steps.versions.outputs.extra_args }} \ + --file "${{ steps.dockerfile.outputs.path }}" \ . diff --git a/.github/workflows/check-versions.yml b/.github/workflows/check-versions.yml new file mode 100644 index 000000000..61811d715 --- /dev/null +++ b/.github/workflows/check-versions.yml @@ -0,0 +1,158 @@ +name: Check VS Code Worker Versions + +on: + schedule: + # Weekdays at 8am SF time (3pm UTC during PDT, 4pm during PST) + - cron: '0 15 * * 1-5' + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + check-versions: + name: Check for version updates + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # pnpm/action-setup@v4 is not allowed on EMU + - name: Install pnpm + run: | + curl -fsSL https://get.pnpm.io/install.sh | sh - + echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile --filter version-checker-vscode-web + + - name: Check for new versions + id: check + run: | + set +e + pnpm --silent --filter version-checker-vscode-web check > /tmp/versions.json 2>/tmp/versions.log + EXIT_CODE=$? + set -e + + echo "::group::check-versions stderr" + cat /tmp/versions.log + echo "::endgroup::" + echo "::group::check-versions stdout" + cat /tmp/versions.json + echo "::endgroup::" + echo "Exit code: $EXIT_CODE" + + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + + if [ "$EXIT_CODE" -eq 1 ]; then + echo "updates_available=true" >> $GITHUB_OUTPUT + + # Parse the JSON output ā€” stdout only, no pnpm noise + node -e " + const fs = require('fs'); + const r = JSON.parse(fs.readFileSync('/tmp/versions.json', 'utf8')); + const out = fs.createWriteStream(process.env.GITHUB_OUTPUT, { flags: 'a' }); + out.write('vscode_current=' + r.vscode.current + '\n'); + out.write('vscode_latest=' + r.vscode.latest + '\n'); + out.write('copilot_current=' + r.copilotChat.current + '\n'); + out.write('copilot_latest=' + r.copilotChat.latest + '\n'); + const parts = []; + if (r.vscode.current !== r.vscode.latest) parts.push('VS Code ' + r.vscode.latest); + if (r.copilotChat.current !== r.copilotChat.latest) parts.push('Copilot Chat ' + r.copilotChat.latest); + out.end(); + " + else + echo "updates_available=false" >> $GITHUB_OUTPUT + echo "No updates available (exit code: $EXIT_CODE)" + fi + + - name: Check for existing open issue + if: steps.check.outputs.updates_available == 'true' + id: existing + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # List open issues with the dependencies label and filter by title prefix. + # Using list + jq instead of --search to avoid GitHub search index lag + # which can cause duplicate issues when runs happen close together. + ISSUE_URL=$(gh issue list --state open --label dependencies --json title,url \ + if [ -n "$ISSUE_URL" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + echo "issue_url=$ISSUE_URL" >> $GITHUB_OUTPUT + echo "Existing open issue found: $ISSUE_URL" + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Create issue + id: create_issue + if: steps.check.outputs.updates_available == 'true' && steps.existing.outputs.exists == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + + | Component | Current | Latest | + |-----------|---------|--------| + | VS Code | ${{ steps.check.outputs.vscode_current }} | ${{ steps.check.outputs.vscode_latest }} | + | Copilot Chat | ${{ steps.check.outputs.copilot_current }} | ${{ steps.check.outputs.copilot_latest }} | + + ### How to update + + + \`\`\`env + VSCODE_VERSION=${{ steps.check.outputs.vscode_latest }} + COPILOT_CHAT_VERSION=${{ steps.check.outputs.copilot_latest }} + \`\`\` + + Then run integration tests: + \`\`\`bash + pnpm test:integration + \`\`\` + + > [!IMPORTANT] + + ### Links + + - [VS Code Release Notes](https://code.visualstudio.com/updates) + - [Copilot Chat on Marketplace](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat) + " + + ISSUE_URL=$(gh issue create \ + --title "${{ steps.check.outputs.title }}" \ + --body "$BODY" \ + --label "dependencies") + echo "issue_url=$ISSUE_URL" >> $GITHUB_OUTPUT + + - name: Write job summary + if: always() && steps.check.outputs.exit_code != '2' + run: | + if [ "${{ steps.check.outputs.updates_available }}" != "true" ]; then + cat >> $GITHUB_STEP_SUMMARY <<'EOF' + ## āœ… All versions up to date + + EOF + else + cat >> $GITHUB_STEP_SUMMARY <> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.existing.outputs.exists }}" == "true" ]; then + echo "ā„¹ļø **Issue already exists:** ${{ steps.existing.outputs.issue_url }}" >> $GITHUB_STEP_SUMMARY + fi + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..a13ade299 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,229 @@ +name: CI + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + test: + name: Unit Tests + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [22] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # pnpm/action-setup@v4 is not allowed on EMU + - name: Install pnpm + run: | + curl -fsSL https://get.pnpm.io/install.sh | sh - + echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build shared package + run: pnpm --filter shared build + + - name: Run tests with coverage + run: pnpm test:coverage + + - name: Generate GitHub test summary from CTRF + if: always() + run: | + pnpm exec github-actions-ctrf "ctrf/*-ctrf-report.json" --title "Test Results (Node.js ${{ matrix.node-version }})" + + - name: Upload coverage reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-reports-node-${{ matrix.node-version }} + path: coverage/ + + - name: Post coverage summary to Pull Request + if: always() && github.event_name == 'pull_request' + run: | + node -e " + const fs = require('fs'); + const summary = JSON.parse(fs.readFileSync('coverage/coverage-summary.json', 'utf8')); + const t = summary.total; + const pct = (v) => v.pct !== undefined && v.pct !== null ? v.pct + '%' : 'N/A'; + const lines = [ + '## šŸ“Š Coverage Summary', + '', + '| Category | Coverage | Covered | Total |', + '|----------|----------|---------|-------|', + '| **Statements** | ' + pct(t.statements) + ' | ' + t.statements.covered + ' | ' + t.statements.total + ' |', + '| **Branches** | ' + pct(t.branches) + ' | ' + t.branches.covered + ' | ' + t.branches.total + ' |', + '| **Functions** | ' + pct(t.functions) + ' | ' + t.functions.covered + ' | ' + t.functions.total + ' |', + '| **Lines** | ' + pct(t.lines) + ' | ' + t.lines.covered + ' | ' + t.lines.total + ' |', + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); + " + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Post test results to Pull Request + if: always() && github.event_name == 'pull_request' + run: | + pnpm exec github-actions-ctrf pull-request "ctrf/*-ctrf-report.json" --title "Test Results (Node.js ${{ matrix.node-version }})" || { + echo "āš ļø Failed to post PR comment (may be due to permissions)" + echo "āœ… Test results are still available in the Actions summary above" + exit 0 + } + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload CTRF test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ctrf-test-results-node-${{ matrix.node-version }} + path: ctrf/ + + integration-test: + name: Integration Tests (shard ${{ matrix.shard }}/${{ strategy.job-total }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + shard: [1, 2] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # pnpm/action-setup@v4 is not allowed on EMU + - name: Install pnpm + run: | + curl -fsSL https://get.pnpm.io/install.sh | sh - + echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build shared package + run: pnpm --filter shared build + + - name: Run integration tests + run: npx vitest run --config vitest.integration.config.ts --silent=false --shard=${{ matrix.shard }}/${{ strategy.job-total }} + + - name: Generate GitHub test summary from CTRF + if: always() + run: | + pnpm exec github-actions-ctrf "ctrf/*-integration-ctrf-report.json" --title "Integration Test Results (shard ${{ matrix.shard }})" || true + + - name: Upload CTRF integration test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ctrf-integration-test-results-shard-${{ matrix.shard }} + path: ctrf/ + + lint: + name: Lint Code + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # pnpm/action-setup@v4 is not allowed on EMU + - name: Install pnpm + run: | + curl -fsSL https://get.pnpm.io/install.sh | sh - + echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Lint + # TODO: remove continue-on-error once ESLint is added as a dependency + continue-on-error: true + run: pnpm lint + + build: + name: Build Applications + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # pnpm/action-setup@v4 is not allowed on EMU + - name: Install pnpm + run: | + curl -fsSL https://get.pnpm.io/install.sh | sh - + echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build + run: pnpm build + + # Summary job that depends on all other jobs + ci-summary: + name: CI Summary + runs-on: ubuntu-latest + needs: [test, integration-test, lint, build] + if: always() + + steps: + - name: Check overall status + run: | + if [ "${{ needs.test.result }}" = "success" ] && \ + [ "${{ needs.integration-test.result }}" = "success" ] && \ + [ "${{ needs.lint.result }}" = "success" ] && \ + [ "${{ needs.build.result }}" = "success" ]; then + echo "āœ… All checks passed!" + exit 0 + else + echo "āŒ Some checks failed:" + echo " Unit Tests: ${{ needs.test.result }}" + echo " Integration Tests: ${{ needs.integration-test.result }}" + echo " Lint: ${{ needs.lint.result }}" + echo " Build: ${{ needs.build.result }}" + exit 1 + fi diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml new file mode 100644 index 000000000..1e63312ca --- /dev/null +++ b/.github/workflows/daily-repo-status.lock.yml @@ -0,0 +1,1124 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.57.1). DO NOT EDIT. +# +# To update this file, edit githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# This workflow creates daily repo status reports. It gathers recent repository +# activity (issues, PRs, discussions, releases, code changes) and generates +# engaging GitHub issues with productivity insights, community highlights, +# and project recommendations. +# +# Source: githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f +# +# gh-aw-metadata: {"schema_version":"v2","frontmatter_hash":"a1e9bd63a75233b0f3627611cbb157d74caf6e41e465b0dabefa8e3fa9cb790a","compiler_version":"v0.57.1","strict":true} + +name: "Daily Repo Status" +"on": + schedule: + - cron: "47 23 * * *" + # Friendly format: daily (scattered) + workflow_dispatch: + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Daily Repo Status" + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@36e5751f7c3d40ec9df8f44c0252b7bfabfc4dd6 # v0.57.1 + with: + destination: /opt/gh-aw/actions + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_INFO_VERSION: "" + GH_AW_INFO_AGENT_VERSION: "latest" + GH_AW_INFO_CLI_VERSION: "v0.57.1" + GH_AW_INFO_WORKFLOW_NAME: "Daily Repo Status" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.23.0" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { main } = require('/opt/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "daily-repo-status.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + { + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/xpia.md" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" + cat "/opt/gh-aw/prompts/markdown.md" + cat "/opt/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_EOF' + + Tools: create_issue, missing_tool, missing_data, noop + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + {{#runtime-import .github/workflows/daily-repo-status.md}} + GH_AW_PROMPT_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: activation + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + retention-days: 1 + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_WORKFLOW_ID_SANITIZED: dailyrepostatus + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + inference_access_error: ${{ steps.detect-inference-error.outputs.inference_access_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@36e5751f7c3d40ec9df8f44c0252b7bfabfc4dd6 # v0.57.1 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + (github.event.pull_request) || (github.event.issue.pull_request) + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh latest + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.23.0 + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.23.0 ghcr.io/github/gh-aw-firewall/api-proxy:0.23.0 ghcr.io/github/gh-aw-firewall/squid:0.23.0 ghcr.io/github/gh-aw-mcpg:v0.1.8 ghcr.io/github/github-mcp-server:v0.32.0 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"create_issue":{"max":1},"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a new GitHub issue for tracking bugs, feature requests, or tasks. Use this for actionable work items that need assignment, labeling, and status tracking. For reports, announcements, or status updates that don't require task tracking, use create_discussion instead. CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [\"report\" \"daily-status\"] will be automatically added.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed issue description in Markdown. Do NOT repeat the title as a heading since it already appears as the issue's h1. Include context, reproduction steps, or acceptance criteria as appropriate.", + "type": "string" + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "labels": { + "description": "Labels to categorize the issue (e.g., 'bug', 'enhancement'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "parent": { + "description": "Parent issue number for creating sub-issues. This is the numeric ID from the GitHub URL (e.g., 42 in github.com/owner/repo/issues/42). Can also be a temporary_id (e.g., 'aw_abc123', 'aw_Test123') from a previously created issue in the same workflow run.", + "type": [ + "number", + "string" + ] + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + }, + "temporary_id": { + "description": "Unique temporary identifier for referencing this issue before it's created. Format: 'aw_' followed by 3 to 12 alphanumeric characters (e.g., 'aw_abc1', 'aw_Test123'). Use '#aw_ID' in body text to reference other issues by their temporary_id; these are replaced with actual issue numbers after creation.", + "pattern": "^aw_[A-Za-z0-9]{3,12}$", + "type": "string" + }, + "title": { + "description": "Concise issue title summarizing the bug, feature, or task. The title appears as the main heading, so keep it brief and descriptive.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_issue" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "create_issue": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "parent": { + "issueOrPRNumber": true + }, + "repo": { + "type": "string", + "maxLength": 256 + }, + "temporary_id": { + "type": "string" + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.8' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.32.0", + "env": { + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Download activation artifact + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 + with: + name: activation + path: /tmp/gh-aw + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + touch /tmp/gh-aw/agent-step-summary.md + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.23.0 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Detect inference access error + id: detect-inference-error + if: always() + continue-on-error: true + run: bash /opt/gh-aw/actions/detect_inference_access_error.sh + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Append agent step summary + if: always() + run: bash /opt/gh-aw/actions/append_agent_step_summary.sh + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_ALLOWED_GITHUB_REFS: "" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) + if command -v awf &> /dev/null; then + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + else + echo 'AWF binary not installed, skipping firewall log summary' + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + if-no-files-found: ignore + # --- Threat Detection (inline) --- + - name: Check if detection needed + id: detection_guard + if: always() + env: + OUTPUT_TYPES: ${{ steps.collect_output.outputs.output_types }} + HAS_PATCH: ${{ steps.collect_output.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP configuration for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f /tmp/gh-aw/mcp-config/mcp-servers.json + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + WORKFLOW_NAME: "Daily Repo Status" + WORKFLOW_DESCRIPTION: "This workflow creates daily repo status reports. It gathers recent repository\nactivity (issues, PRs, discussions, releases, code changes) and generates\nengaging GitHub issues with productivity insights, community highlights,\nand project recommendations." + HAS_PATCH: ${{ steps.collect_output.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + id: detection_agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + touch /tmp/gh-aw/agent-step-summary.md + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.23.0 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(wc)'\'' --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_detection_results + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Set detection conclusion + id: detection_conclusion + if: always() + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_SUCCESS: ${{ steps.parse_detection_results.outputs.success }} + run: | + if [[ "$RUN_DETECTION" != "true" ]]; then + echo "conclusion=skipped" >> "$GITHUB_OUTPUT" + echo "success=true" >> "$GITHUB_OUTPUT" + echo "Detection was not needed, marking as skipped" + elif [[ "$DETECTION_SUCCESS" == "true" ]]; then + echo "conclusion=success" >> "$GITHUB_OUTPUT" + echo "success=true" >> "$GITHUB_OUTPUT" + echo "Detection passed successfully" + else + echo "conclusion=failure" >> "$GITHUB_OUTPUT" + echo "success=false" >> "$GITHUB_OUTPUT" + echo "Detection found issues" + fi + + conclusion: + needs: + - activation + - agent + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: read + issues: write + concurrency: + group: "gh-aw-conclusion-daily-repo-status" + cancel-in-progress: false + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@36e5751f7c3d40ec9df8f44c0252b7bfabfc4dd6 # v0.57.1 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Daily Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/346204513ecfa08b81566450d7d599556807389f/workflows/daily-repo-status.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Daily Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/346204513ecfa08b81566450d7d599556807389f/workflows/daily-repo-status.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Daily Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/346204513ecfa08b81566450d7d599556807389f/workflows/daily-repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "daily-repo-status" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_TIMEOUT_MINUTES: "20" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Daily Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/346204513ecfa08b81566450d7d599556807389f/workflows/daily-repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + + safe_outputs: + needs: agent + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.agent.outputs.detection_success == 'true') + runs-on: ubuntu-slim + permissions: + contents: read + issues: write + timeout-minutes: 15 + env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/daily-repo-status" + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "daily-repo-status" + GH_AW_WORKFLOW_NAME: "Daily Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/346204513ecfa08b81566450d7d599556807389f/workflows/daily-repo-status.md" + outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + created_issue_number: ${{ steps.process_safe_outputs.outputs.created_issue_number }} + created_issue_url: ${{ steps.process_safe_outputs.outputs.created_issue_url }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@36e5751f7c3d40ec9df8f44c0252b7bfabfc4dd6 # v0.57.1 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"missing_data\":{},\"missing_tool\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + - name: Upload safe output items manifest + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: safe-output-items + path: /tmp/safe-output-items.jsonl + if-no-files-found: warn + diff --git a/.github/workflows/daily-repo-status.md b/.github/workflows/daily-repo-status.md new file mode 100644 index 000000000..f556ac4b6 --- /dev/null +++ b/.github/workflows/daily-repo-status.md @@ -0,0 +1,57 @@ +--- +description: | + This workflow creates daily repo status reports. It gathers recent repository + activity (issues, PRs, discussions, releases, code changes) and generates + engaging GitHub issues with productivity insights, community highlights, + and project recommendations. + +on: + schedule: daily + workflow_dispatch: + +permissions: + contents: read + issues: read + pull-requests: read + +network: defaults + +tools: + github: + # If in a public repo, setting `lockdown: false` allows + # reading issues, pull requests and comments from 3rd-parties + # If in a private repo this has no particular effect. + lockdown: false + +safe-outputs: + mentions: false + allowed-github-references: [] + create-issue: + title-prefix: "[repo-status] " + labels: [report, daily-status] + close-older-issues: true +source: githubnext/agentics/workflows/daily-repo-status.md@346204513ecfa08b81566450d7d599556807389f +--- + +# Daily Repo Status + +Create an upbeat daily status report for the repo as a GitHub issue. + +## What to include + +- Recent repository activity (issues, PRs, discussions, releases, code changes) +- Progress tracking, goal reminders and highlights +- Project status and recommendations +- Actionable next steps for maintainers + +## Style + +- Be positive, encouraging, and helpful šŸŒŸ +- Use emojis moderately for engagement +- Keep it concise - adjust length based on actual activity + +## Process + +1. Gather recent activity from the repository +2. Study the repository, its issues and its pull requests +3. Create a new GitHub issue with your findings and insights diff --git a/.gitignore b/.gitignore index 295415758..c36e49689 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,12 @@ node_modules/ dist/ .env .env.local +.port-offset *.log .DS_Store .azure/ .venv/ downloads/ +ctrf/ +coverage/ +.auth/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..5be09730c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,22 @@ +{ + "workbench.colorCustomizations": { + "activityBar.activeBackground": "#93e6fc", + "activityBar.background": "#93e6fc", + "activityBar.foreground": "#15202b", + "activityBar.inactiveForeground": "#15202b99", + "activityBarBadge.background": "#fa45d4", + "activityBarBadge.foreground": "#15202b", + "commandCenter.border": "#15202b99", + "sash.hoverBorder": "#93e6fc", + "statusBar.background": "#61dafb", + "statusBar.foreground": "#15202b", + "statusBarItem.hoverBackground": "#2fcefa", + "statusBarItem.remoteBackground": "#61dafb", + "statusBarItem.remoteForeground": "#15202b", + "titleBar.activeBackground": "#61dafb", + "titleBar.activeForeground": "#15202b", + "titleBar.inactiveBackground": "#61dafb99", + "titleBar.inactiveForeground": "#15202b99" + }, + "peacock.color": "#61dafb" +} \ No newline at end of file diff --git a/Dockerfile.api b/Dockerfile.api deleted file mode 100644 index 45eb81aac..000000000 --- a/Dockerfile.api +++ /dev/null @@ -1,37 +0,0 @@ -FROM node:22.22.0-slim AS builder - -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -WORKDIR /app - -COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ -COPY tsconfig.base.json ./ -COPY packages/api/package.json ./packages/api/ - -RUN pnpm install --frozen-lockfile - -COPY packages/api/tsconfig.json ./packages/api/ -COPY packages/api/src ./packages/api/src/ - -RUN pnpm --filter api build - -FROM node:22.22.0-slim - -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -WORKDIR /app - -COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ -COPY packages/api/package.json ./packages/api/ - -RUN pnpm install --frozen-lockfile --prod - -COPY --from=builder /app/packages/api/dist ./packages/api/dist/ - -WORKDIR /app/packages/api - -EXPOSE 80 - -ENV PORT=80 - -CMD ["node", "dist/index.js"] diff --git a/Dockerfile.coder-acp-claude-code b/Dockerfile.coder-acp-claude-code deleted file mode 100644 index c5a858712..000000000 --- a/Dockerfile.coder-acp-claude-code +++ /dev/null @@ -1,58 +0,0 @@ -FROM node:22.22.0-slim AS builder - -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -WORKDIR /app - -COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ -COPY tsconfig.base.json ./ -COPY packages/shared/package.json ./packages/shared/ -COPY packages/workers/coder-acp-claude-code/package.json ./packages/workers/coder-acp-claude-code/ - -RUN pnpm install --frozen-lockfile - -COPY packages/shared/tsconfig.json ./packages/shared/ -COPY packages/shared/src ./packages/shared/src/ -COPY packages/workers/coder-acp-claude-code/tsconfig.json ./packages/workers/coder-acp-claude-code/ -COPY packages/workers/coder-acp-claude-code/src ./packages/workers/coder-acp-claude-code/src/ - -RUN pnpm --filter shared build && pnpm --filter coder-acp-claude-code build - -FROM node:22.22.0-slim - -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -# Install Python and uv for Python-based MCP servers -RUN apt-get update && apt-get install -y \ - python3 \ - python3-pip \ - curl \ - git \ - && curl -LsSf https://astral.sh/uv/0.10.0/install.sh | sh \ - && rm -rf /var/lib/apt/lists/* - -# Add uv to PATH -ENV PATH="/root/.local/bin:$PATH" - -# Install Claude Code ACP globally -RUN npm install -g @zed-industries/claude-code-acp@0.16.0 - -WORKDIR /app - -COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ -COPY packages/shared/package.json ./packages/shared/ -COPY packages/workers/coder-acp-claude-code/package.json ./packages/workers/coder-acp-claude-code/ - -RUN pnpm install --frozen-lockfile --prod - -COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ -COPY --from=builder /app/packages/workers/coder-acp-claude-code/dist ./packages/workers/coder-acp-claude-code/dist/ - -# Create workspace directory -RUN mkdir -p /workspace - -WORKDIR /app/packages/workers/coder-acp-claude-code - -ENV WORKER_NAME=coder-acp-claude-code - -CMD ["node", "dist/index.js"] diff --git a/Dockerfile.coder-acp-copilot b/Dockerfile.coder-acp-copilot deleted file mode 100644 index 1de5d371a..000000000 --- a/Dockerfile.coder-acp-copilot +++ /dev/null @@ -1,58 +0,0 @@ -FROM node:22.22.0-slim AS builder - -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -WORKDIR /app - -COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ -COPY tsconfig.base.json ./ -COPY packages/shared/package.json ./packages/shared/ -COPY packages/workers/coder-acp-copilot/package.json ./packages/workers/coder-acp-copilot/ - -RUN pnpm install --frozen-lockfile - -COPY packages/shared/tsconfig.json ./packages/shared/ -COPY packages/shared/src ./packages/shared/src/ -COPY packages/workers/coder-acp-copilot/tsconfig.json ./packages/workers/coder-acp-copilot/ -COPY packages/workers/coder-acp-copilot/src ./packages/workers/coder-acp-copilot/src/ - -RUN pnpm --filter shared build && pnpm --filter coder-acp-copilot build - -FROM node:22.22.0-slim - -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -# Install Python and uv for Python-based MCP servers -RUN apt-get update && apt-get install -y \ - python3 \ - python3-pip \ - curl \ - git \ - && curl -LsSf https://astral.sh/uv/0.10.0/install.sh | sh \ - && rm -rf /var/lib/apt/lists/* - -# Add uv to PATH -ENV PATH="/root/.local/bin:$PATH" - -# Install GitHub Copilot CLI globally -RUN npm install -g @anthropic-ai/claude-code@2.1.37 @github/copilot@0.0.406 - -WORKDIR /app - -COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ -COPY packages/shared/package.json ./packages/shared/ -COPY packages/workers/coder-acp-copilot/package.json ./packages/workers/coder-acp-copilot/ - -RUN pnpm install --frozen-lockfile --prod - -COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ -COPY --from=builder /app/packages/workers/coder-acp-copilot/dist ./packages/workers/coder-acp-copilot/dist/ - -# Create workspace directory -RUN mkdir -p /workspace - -WORKDIR /app/packages/workers/coder-acp-copilot - -ENV WORKER_NAME=coder-acp-copilot - -CMD ["node", "dist/index.js"] diff --git a/Dockerfile.portal b/Dockerfile.portal deleted file mode 100644 index e1571dc1d..000000000 --- a/Dockerfile.portal +++ /dev/null @@ -1,18 +0,0 @@ -# Stage 1: Build -FROM node:20-alpine AS builder -RUN corepack enable && corepack prepare pnpm@10.29.1 --activate - -WORKDIR /app -COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./ -COPY packages/portal/package.json packages/portal/ -RUN pnpm install --frozen-lockfile --filter portal - -COPY packages/portal/ packages/portal/ -RUN pnpm --filter portal build - -# Stage 2: Serve with nginx -FROM nginx:alpine -COPY --from=builder /app/packages/portal/dist /usr/share/nginx/html -COPY packages/portal/nginx.conf /etc/nginx/conf.d/default.conf -EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md index dae95942f..6a144812c 100644 --- a/ENV_VARIABLES.md +++ b/ENV_VARIABLES.md @@ -108,3 +108,102 @@ JUDGE_MAX_PARALLELISM=5 FEEDBACK_MAX_CRITERIA=3 FEEDBACK_DESCENDANT_GUARD=false ``` + +## Report Generator Configuration + +### AZURE_STORAGE_QUEUE_REPORT +**Default:** `report-queue` +**Type:** string + +Azure Storage Queue name for report generation jobs. The API enqueues messages here when a report is requested; the report-generator worker polls this queue. + +### REPORT_MODEL +**Default:** `gpt-4.1` +**Type:** string + +The LLM model used by the report-generator worker (via the Copilot SDK) to generate run analysis reports. Examples: `gpt-4.1`, `gpt-4o`, `claude-sonnet-4`. + +### SCOPE_MT_API_URL +**Default:** `http://localhost:3001` (local), `http://api:80` (Docker) +**Type:** URL string + +Base URL of the Scope MT API. The report-generator worker calls this to fetch run data (summary, turns, criteria trajectory) via REST tools during report generation. + +### SESSION_TIMEOUT_MS +**Default:** `300000` (5 minutes) +**Type:** integer (milliseconds) + +Timeout for the Copilot SDK session used by the report-generator worker. If the LLM takes longer than this to generate a report, the session will be terminated and the report marked as failed. + +### GIT_COMMIT +**Default:** `development` +**Type:** string + +Git commit hash embedded in reporter metadata. Automatically set during CI/CD builds. Used to track which version of the report-generator produced a given report. + +## Token Manager Configuration + +### TOKEN_MANAGER_URL +**Default:** (not set) +**Type:** URL string + +Base URL of the Token Manager service. Workers, judge, and report-generator use the `TokenManagerClient` to dynamically acquire tokens via `POST /api/v1/tokens/acquire` (round-robin across enabled tokens). + +In Kubernetes, no static token secrets (`GITHUB_TOKEN`, `ANTHROPIC_API_KEY`) are injected into pods ā€” all tokens are acquired from the Token Manager at runtime. In local dev / Docker Compose, env vars can still be set as a fallback (the `TokenManagerClient` checks env vars first before calling the Token Manager HTTP API). + +- **Docker Compose:** `http://token-manager:80` +- **Kubernetes:** `http://token-manager-service.scoped.svc.cluster.local:80` +- **Local dev:** Leave unset to use env var fallback (`GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, etc.) + +### AZURE_KEYVAULT_URI +**Type:** URL string (token-manager only) ā€” **Required** + +Azure KeyVault URI for storing token secret values. The Token Manager uses `KeyVaultTokenStore` with `DefaultAzureCredential`. + +- **Docker Compose:** Provided automatically via Lowkey Vault (Azure KV emulator): `https://lowkey-vault:8443` +- **Kubernetes:** Azure Key Vault URI (e.g., `https://my-vault.vault.azure.net`) +- **Local dev (no Docker):** Not supported without a vault; use Docker Compose + +### VALIDATION_INTERVAL_MS +**Default:** `300000` (5 minutes) +**Type:** integer (milliseconds) + +How often the Token Manager's scheduler validates all active tokens against their provider APIs. Each token is tested (e.g., GitHub PAT ā†’ `GET /user`, Anthropic ā†’ `GET /v1/models`) and its `lastValidationStatus` is updated in MongoDB. + +### TOKEN_MANAGER_PORT +**Default:** `3102` +**Type:** integer (Docker Compose only) + +Host port mapping for the token-manager service in Docker Compose. + +## DevProxy Configuration (HAR Capture) + +### DEV_PROXY_ENABLED +**Default:** `false` +**Type:** boolean (`true` | `false`) + +Enables DevProxy integration for capturing HTTP traffic as HAR files. When `true`, the worker starts/stops DevProxy recording around each coding agent session, extracts tool calls from the HAR, and uploads the HAR to blob storage. + +- **Docker Compose:** Set via `DEV_PROXY_ENABLED=true` in `.env` or inline +- **Kubernetes:** Set in the deployment manifest env vars (auto-set when sidecar is present) + +### DEV_PROXY_API_URL +**Default:** `http://localhost:18897` +**Type:** URL string + +URL of the DevProxy REST API. The `DevProxyClient` uses this to start/stop recording, check status, and download the CA certificate. + +- **Docker Compose:** `http://devproxy-copilot:18897` (separate service) +- **Kubernetes:** `http://localhost:18897` (sidecar in same pod) + +### DEV_PROXY_HAR_DIR +**Default:** `/har-output` +**Type:** path + +Directory where DevProxy writes HAR files. Shared between the DevProxy process and the worker via a volume mount. + +### DEVPROXY_API_PORT +**Default:** `18897` +**Type:** integer (Docker Compose only) + +Host port mapping for the DevProxy REST API in Docker Compose. diff --git a/README.md b/README.md index 7dba8f2c9..228780d3f 100644 --- a/README.md +++ b/README.md @@ -1,233 +1,212 @@ -# Coding Agent Workers with Real-Time Logging (MongoDB + Azure Managed Redis) +# Scope MT -This experiment demonstrates a TypeScript-based API with **three coding agent workers** and **real-time log streaming** using Azure Container Apps: +**Scope MT** is a Kubernetes-native platform for benchmarking AI coding agents. It orchestrates coding tasks across multiple agent workers, evaluates results using a criteria DAG, and provides real-time log streaming ā€” all backed by MongoDB, Redis, and Azure Storage Queues. The application is deployed via FluxCD GitOps with Kustomize overlays and runs on AKS. -- **API**: Express.js REST API that routes coding requests to different agent workers and streams logs via SSE -- **coder-acp-claude-code**: Claude Code agent using Agent Client Protocol (ACP) -- **coder-acp-copilot**: GitHub Copilot agent using Agent Client Protocol (ACP) -- **Storage Queues**: Separate queue per worker for message routing -- **CosmosDB (MongoDB API)**: Shared database for request status tracking and log persistence -- **Azure Managed Redis**: Real-time log publishing via Pub/Sub -- **Azure Key Vault**: Secure storage for API keys and auth state +## Components + +| Component | Description | +|-----------|-------------| +| **API** | Express.js REST API ā€” routes requests to workers, streams logs via SSE | +| **Judge** | Evaluates completed runs against a criteria DAG using Copilot SDK | +| **Portal** | React web UI for managing and inspecting runs | +| **CLI** | Command-line interface for submitting runs, managing criteria, and more | +| **coder-acp-claude-code** | Claude Code agent worker (ACP) | +| **coder-acp-copilot** | GitHub Copilot agent worker (ACP) | +| **report-generator** | Generates evaluation reports from completed runs | ## Architecture +```mermaid +flowchart LR + Client([Client / CLI]) + Portal([Portal]) + API[API] + Judge[Judge] + + subgraph Queues + Q1[claude-code] + Q2[copilot] + Q3[vscode-web] + end + + subgraph Workers + W1[coder-acp-claude-code] + W2[coder-acp-copilot] + end + + MongoDB[(MongoDB)] + Redis[(Redis Pub/Sub)] + Blob[(Blob Storage)] + + Client -->|REST| API + Portal -->|REST| API + API -->|SSE| Client + API --> Q1 & Q2 & Q3 + Q1 --> W1 + Q2 --> W2 + Q3 --> W3 + W1 & W2 & W3 -->|status + logs| MongoDB + W1 & W2 & W3 -->|real-time logs| Redis + W1 & W2 & W3 -->|workspace snapshots| Blob + Redis -->|subscribe| API + Judge -->|evaluate| MongoDB ``` - ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” - ā”Œā”€ā”€ā–¶ā”‚ Queue claude ā”‚ā”€ā”€ā”€ā”€ā–¶ā”‚ coder-acp-claude-code ā”‚ā”€ā”€ā”€ā” -ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ (Claude Code ACP) ā”‚ ā”‚ -ā”‚ Client ā”‚ā”€ā”€ā”€ā”€ā–¶ā”‚ API (ACA) ā”‚ā”€ā”€ā”€ā”¤ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ -ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ - ā–² ā”‚ ā”‚ ā”œā”€ā”€ā–¶ā”‚ Queue copilot ā”‚ā”€ā”€ā”€ā”€ā–¶ā”‚ coder-acp-copilot ā”‚ā”€ā”€ā”€ā”¤ - ā”‚ ā”‚ SSE ā—€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”¤ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ (Copilot ACP) ā”‚ ā”‚ - ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ - ā”‚ ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ - ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ (Playwright + VS Code) ā”‚ ā”‚ - ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ - ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ - ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”‚ Redis Pub/Sub ā”‚ā—€ā”€ā”€ā”€ā”€ā”‚ Log Events ā”‚ā—€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ - ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ - ā”‚ - ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ - ā”‚ CosmosDB ā”‚ā—€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ - ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ -``` - -## Features - -### Coding Agent Workers -| Worker | Agent | Protocol | Use Case | -|--------|-------|----------|----------| -| coder-acp-claude-code | Claude Code | ACP (stdio) | AI coding with Claude | -| coder-acp-copilot | GitHub Copilot | ACP (stdio) | AI coding with Copilot CLI | +## Quick Start -### Real-Time Log Streaming -Workers publish step-by-step progress logs that clients can stream in real-time: -- **Redis Pub/Sub** for instant log delivery (milliseconds latency) -- **Server-Sent Events (SSE)** for browser and CLI compatibility -- **CosmosDB persistence** for log replay and history +### Prerequisites -## CLI Usage +- Node.js 22+ +- [pnpm](https://pnpm.io/) +- Docker & Docker Compose ā€” on macOS, [OrbStack](https://orbstack.dev/) is recommended over Docker Desktop (faster, lighter) +- [GitHub CLI](https://cli.github.com/) (`gh`) with authentication -The CLI provides an easy way to submit requests and stream logs. +### Run locally ```bash -export $(azd env get-values | xargs) - -# Run demo TUI with concurrent requests to all coders -pnpm cli demo -m "create a hello world app" -c 2 -u $API_ENDPOINT - -# Submit to Copilot and stream logs -pnpm cli submit -m "Explain this code: const x = 42;" -w coder-acp-copilot -u $API_ENDPOINT - -# Submit to Claude Code -pnpm cli submit -m "Create a hello world function in Python" -w coder-acp-claude-code -u $API_ENDPOINT - -# Submit to VS Code Web +pnpm install +GITHUB_TOKEN=$(gh auth token) pnpm docker:dev:copilot +``` -# Submit without streaming logs -pnpm cli submit -m "Say hello" -w coder-acp-copilot -u $API_ENDPOINT --no-stream +This starts all core services (MongoDB, Redis, Azurite, API, Judge, Token Manager) plus the Copilot worker, report generator, and **portal** ā€” with hot reload. Edit any source file and the running service restarts automatically. -# Check request status -pnpm cli status -u $API_ENDPOINT +### Accessing the Portal -# Stream logs for existing request -pnpm cli logs -u $API_ENDPOINT --from-start +Once the services are running, open the portal in your browser: -# List all requests -pnpm cli list -u $API_ENDPOINT +``` +http://localhost:5100 ``` -## API Endpoints +Or use the convenience command to open it automatically: -### Health Check ```bash -export $(azd env get-values | xargs) && curl $API_ENDPOINT/health +pnpm open:portal ``` -### Get Request Status -```bash -curl $API_ENDPOINT/api/v1/requests/:id -``` +> **Worktree note:** In a [git worktree](#git-worktree-support), ports are offset for isolation so the portal may run on a different port (e.g. `5103`). Check `PORTAL_PORT` in your `.env` file for the actual port, or just run `pnpm open:portal` ā€” it reads `.env` and opens the correct URL. -### Stream Logs (SSE) -```bash -curl -N "$API_ENDPOINT/api/v1/requests/:id/logs?fromStart=true" -``` +### Other useful commands -## Local Development +```bash +# Start only infrastructure services (MongoDB, Redis, Azurite) +pnpm docker:up:infra -### Prerequisites +# Run individual services locally (after starting infra) +pnpm dev:api +pnpm dev:portal +pnpm dev:coder-acp-copilot +pnpm dev:coder-acp-claude-code +``` -- Node.js 22+ -- pnpm -- Docker & Docker Compose -- Azure CLI (for Azure deployment) +### Hot Reload with Docker Compose -### Local Development with Docker Compose +The `docker:dev:*` commands use [Docker Compose Watch](https://docs.docker.com/compose/how-tos/file-watch/) to sync source files into containers: ```bash -# Start local services (MongoDB, Redis, Azurite) -pnpm docker:up - -# Copy environment template -cp .env.example .env +pnpm docker:dev:claude-code # Core + Claude Code + Report Generator + Portal +pnpm docker:dev:copilot # Core + Copilot + Report Generator + Portal +pnpm docker:dev:portal # Core + Portal (Vite HMR) +pnpm docker:dev:all # All services +``` -# Set required credentials in .env: -# - ANTHROPIC_API_KEY (for coder-acp-claude-code) -# - GITHUB_TOKEN (for coder-acp-copilot) +| Change | Reload Time | +|--------|-------------| +| Service source (`apps/*/src`) | ~1-2s (tsx watch restart) | +| Shared package (`packages/shared/src`) | ~2-3s (tsc recompile + tsx restart) | +| Portal source (`apps/portal/src`) | Instant (Vite HMR) | +| Config files (`config/`) | ~1-2s (tsx watch restart) | +| Dependencies (`package.json`, lockfile) | Full rebuild (~30-60s) | -# Install dependencies -pnpm install +### Git Worktree Support -# Build all packages -pnpm build +Docker Compose works seamlessly in [git worktrees](https://git-scm.com/docs/git-worktree). The `scripts/worktree-env.ts` script runs automatically before every `docker:*` / `pnpm docker:*` command and assigns each worktree a unique port offset so multiple stacks can run in parallel without conflicts. -# Run API locally -pnpm dev:api +- **Main repo**: offset 0 ā€” base ports used as-is (e.g. API on 3100, MongoDB on 27100) +- **Worktrees**: offset 1ā€“99 ā€” ports are shifted (e.g. offset 1 ā†’ API on 3101, MongoDB on 27101) +- The offset is persisted in a `.port-offset` file inside each worktree and reused across restarts. -# Run workers locally (separate terminals) -pnpm dev:coder-acp-claude-code -pnpm dev:coder-acp-copilot -``` +No manual configuration is needed ā€” just run `pnpm docker:dev:copilot` from any worktree. ### Environment Variables -See `.env.example` for all required variables. +See [ENV_VARIABLES.md](ENV_VARIABLES.md) for a full reference of configurable environment variables. -## Deployment - -```bash -# Login to Azure -azd auth login +## CLI -# Create environment -azd env new myenv +The CLI is the primary interface for interacting with Scope MT. Show available commands with: -# Deploy -azd up +```bash +pnpm cli --help ``` -### Configure Worker Credentials - -After deployment, set the required credentials for each worker: +It is organized into subcommand groups (`run`, `criteria`, ā€¦). Use `--help` at any level to discover options: ```bash -# Set Anthropic API key for Claude Code worker -azd env get-values | xargs -I {} sh -c 'export {}' && \ -az containerapp update --name coder-acp-claude-code \ - --resource-group $(azd env get-values | grep AZURE_RESOURCE_GROUP | cut -d= -f2 | tr -d '"') \ - --set-env-vars "ANTHROPIC_API_KEY=" - -# Set GitHub token for Copilot worker -az containerapp update --name coder-acp-copilot \ - --resource-group $(azd env get-values | grep AZURE_RESOURCE_GROUP | cut -d= -f2 | tr -d '"') \ - --set-env-vars "GITHUB_TOKEN=" +pnpm cli run --help +pnpm cli run submit --help +pnpm cli criteria --help ``` -#### VS Code Web Auth State - -The VS Code Web worker requires GitHub authentication via Playwright browser state stored in Key Vault: +## Configuration -```bash -# Upload auth state (write-only, stored in Key Vault) -curl -X PUT "$(azd env get-values | grep API_ENDPOINT | cut -d= -f2 | tr -d '"')/api/v1/auth/vscode-web" \ - -H "Content-Type: application/json" \ - -d @github-storage.json - -# Check if auth state exists -curl "$(azd env get-values | grep API_ENDPOINT | cut -d= -f2 | tr -d '"')/api/v1/auth/vscode-web/status" +The `config/` directory contains YAML-based configuration for the evaluation system: -# Delete auth state -curl -X DELETE "$(azd env get-values | grep API_ENDPOINT | cut -d= -f2 | tr -d '"')/api/v1/auth/vscode-web" -``` +| Directory | Purpose | +|-----------|---------| +| `config/criteria/` | Evaluation criteria definitions (used by the Judge DAG) | +| `config/personas/` | Judge personas (e.g. `demanding-senior`, `friendly-senior`) | +| `config/scenarios/` | Benchmark scenario definitions | +| `config/traits.yaml` | Trait dimensions (personality, experience, verbosity, type) | -### Deploy Individual Services +## Documentation -Use the postprovision script to build and deploy specific services: +The [`docs/`](docs/README.md) directory contains architecture and research documentation: -```bash -# Deploy all services -./infra/hooks/postprovision.sh +| Path | Contents | +|------|----------| +| `docs/architecture/` | System design ā€” app design, criteria provider, DB migrations, token manager, skills | +| `docs/research/` | Research spikes ā€” delta storage, real-time data flow | -# Deploy only a specific service -./infra/hooks/postprovision.sh api -./infra/hooks/postprovision.sh coder-acp-claude-code -./infra/hooks/postprovision.sh coder-acp-copilot -``` +## Deployment -## Azure Resources +Scope MT is a Kubernetes-native application deployed via [FluxCD](https://fluxcd.io/) GitOps. The `deploy/` directory contains Kustomize base manifests and environment overlays that FluxCD reconciles automatically. -- **Azure Container Apps Environment**: Hosting for API and workers -- **Azure Container Registry**: Container image storage -- **Azure Cosmos DB (MongoDB API)**: Request and log persistence -- **Azure Storage Account**: Queue-based message routing -- **Azure Managed Redis**: Real-time log pub/sub -- **Azure Key Vault**: API keys, auth state storage +Infrastructure provisioning (AKS cluster, Azure resources) is managed in the [scope-mt-infra](https://github.com/growth-ecosystems/scope-mt-infra) repository. ## Project Structure ``` -ā”œā”€ā”€ azure.yaml -ā”œā”€ā”€ docker-compose.yml -ā”œā”€ā”€ .env.example -ā”œā”€ā”€ Dockerfile.api -ā”œā”€ā”€ Dockerfile.coder-acp-claude-code -ā”œā”€ā”€ Dockerfile.coder-acp-copilot -ā”œā”€ā”€ infra/ -ā”‚ ā”œā”€ā”€ main.bicep -ā”‚ ā”œā”€ā”€ resources.bicep -ā”‚ ā””ā”€ā”€ modules/ -ā”œā”€ā”€ packages/ -ā”‚ ā”œā”€ā”€ api/src/index.ts -ā”‚ ā”œā”€ā”€ shared/src/ +scope-mt-app/ +ā”œā”€ā”€ apps/ +ā”‚ ā”œā”€ā”€ api/ # REST API + SSE +ā”‚ ā”œā”€ā”€ cli/ # CLI (commander + ink TUI) +ā”‚ ā”œā”€ā”€ judge/ # Criteria DAG evaluator +ā”‚ ā”œā”€ā”€ portal/ # React + Vite web UI +ā”‚ ā”œā”€ā”€ model-scanners/ +ā”‚ ā”‚ ā”œā”€ā”€ anthropic/ # Anthropic model scanner +ā”‚ ā”‚ ā””ā”€ā”€ copilot/ # Copilot model scanner +ā”‚ ā”œā”€ā”€ token-manager/ # Token management service ā”‚ ā””ā”€ā”€ workers/ -ā”‚ ā”œā”€ā”€ coder-acp-claude-code/ -ā”‚ ā”œā”€ā”€ coder-acp-copilot/ +ā”‚ ā”œā”€ā”€ coder-acp-claude-code/ # Claude Code worker (ACP) +ā”‚ ā”œā”€ā”€ coder-acp-copilot/ # Copilot worker (ACP) +ā”‚ ā””ā”€ā”€ report-generator/ # Report generation worker +ā”œā”€ā”€ packages/ +ā”‚ ā”œā”€ā”€ shared/ # Shared library (criteria graph, MongoDB, Redis, etc.) +ā”‚ ā”œā”€ā”€ db-migrations/ # Database migration scripts +ā”‚ ā””ā”€ā”€ model-scanning/ # Model scanning library +ā”œā”€ā”€ config/ +ā”‚ ā”œā”€ā”€ criteria/ # Evaluation criteria YAML definitions +ā”‚ ā”œā”€ā”€ personas/ # Judge persona configurations +ā”‚ ā”œā”€ā”€ scenarios/ # Benchmark scenario definitions +ā”‚ ā””ā”€ā”€ traits.yaml # Trait dimensions +ā”œā”€ā”€ deploy/ +ā”‚ ā”œā”€ā”€ base/ # Kustomize base manifests +ā”‚ ā”œā”€ā”€ overlays/ # Environment overlays (integration, preview, prod) +ā”‚ ā”œā”€ā”€ image-automation/ # FluxCD image automation +ā”‚ ā””ā”€ā”€ pr-envs/ # PR preview environments +ā”œā”€ā”€ docs/ # Architecture & research documentation +ā”œā”€ā”€ scripts/ # Utility scripts +ā”œā”€ā”€ docker-compose.yml ā””ā”€ā”€ package.json ``` - -## License - -MIT diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile new file mode 100644 index 000000000..a7b8bb753 --- /dev/null +++ b/apps/api/Dockerfile @@ -0,0 +1,68 @@ +FROM node:22.22.0-slim AS builder + +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY tsconfig.base.json ./ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/db-migrations/package.json ./packages/db-migrations/ +COPY apps/api/package.json ./apps/api/ + +RUN pnpm install --frozen-lockfile + +COPY packages/shared/tsconfig.json ./packages/shared/ +COPY packages/shared/src ./packages/shared/src/ +RUN pnpm --filter shared build + +COPY packages/db-migrations/tsconfig.json ./packages/db-migrations/ +COPY packages/db-migrations/src ./packages/db-migrations/src/ +RUN pnpm --filter db-migrations build + +COPY apps/api/tsconfig.json ./apps/api/ +COPY apps/api/src ./apps/api/src/ + +RUN pnpm --filter api build + +# --- Dev stage: hot reload with tsx watch --- +FROM builder AS dev +ENV NODE_ENV=development +ENV SERVICE_DIR=api +COPY scripts/dev-entrypoint.sh /app/scripts/dev-entrypoint.sh +COPY config /app/config/ +EXPOSE 80 +ENV PORT=80 +CMD ["/app/scripts/dev-entrypoint.sh"] + +# --- Production stage --- +FROM node:22.22.0-slim + +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/db-migrations/package.json ./packages/db-migrations/ +COPY apps/api/package.json ./apps/api/ + +RUN pnpm install --frozen-lockfile --prod + +COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ +COPY --from=builder /app/packages/db-migrations/dist ./packages/db-migrations/dist/ +COPY --from=builder /app/apps/api/dist ./apps/api/dist/ + +WORKDIR /app/apps/api + +# Version information (passed as build args) +ARG GIT_COMMIT=unknown +ARG BUILD_TIME=unknown +ENV GIT_COMMIT=$GIT_COMMIT +ENV BUILD_TIME=$BUILD_TIME + +EXPOSE 80 + +ENV PORT=80 + +CMD ["node", "dist/index.js"] diff --git a/packages/api/package.json b/apps/api/package.json similarity index 87% rename from packages/api/package.json rename to apps/api/package.json index 2177de567..98c05ab47 100644 --- a/packages/api/package.json +++ b/apps/api/package.json @@ -10,18 +10,21 @@ "lint": "eslint src/" }, "dependencies": { + "@azure-rest/ai-inference": "^1.0.0-beta.6", "@azure/core-auth": "^1.9.0", "@azure/identity": "^4.5.0", "@azure/keyvault-secrets": "^4.8.0", "@azure/storage-blob": "^12.26.0", "@azure/storage-queue": "^12.24.0", - "@azure-rest/ai-inference": "^1.0.0-beta.6", "cors": "^2.8.5", + "db-migrations": "workspace:*", "dotenv": "^16.4.5", "express": "^4.21.0", "ioredis": "^5.4.1", "mongodb": "^6.12.0", "multer": "^1.4.5-lts.1", + "shared": "workspace:*", + "tar-stream": "^3.1.7", "uuid": "^10.0.0", "yaml": "^2.6.0" }, @@ -30,6 +33,7 @@ "@types/express": "^4.17.21", "@types/multer": "^1.4.12", "@types/node": "^22.7.0", + "@types/tar-stream": "^3.1.4", "@types/uuid": "^10.0.0", "tsx": "^4.19.0", "typescript": "^5.6.2" diff --git a/apps/api/src/analysis.test.ts b/apps/api/src/analysis.test.ts new file mode 100644 index 000000000..9fe91660e --- /dev/null +++ b/apps/api/src/analysis.test.ts @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect } from "vitest"; +import { computeAnalysis, type AnalyzableRun } from "./analysis.js"; +import { computeTaskPromptId } from "shared"; + +describe("computeAnalysis ā€“ taskPromptId grouping", () => { + const kValues = [1, 5]; + + function makeRun(overrides: Partial & { task: string; workerType: string }): AnalyzableRun { + const { task, workerType, ...rest } = overrides; + return { + scenario: { task, criteria: ["c1"] }, + workerType, + status: "completed", + turns: [{ iteration: 1, passed: true, criteriaResults: [] }], + ...rest, + }; + } + + it("groups runs by taskPromptId when present", () => { + const tpId = computeTaskPromptId("Create an API"); + const runs: AnalyzableRun[] = [ + makeRun({ task: "Create an API", workerType: "agent-a", taskPromptId: tpId }), + makeRun({ task: "Create an API", workerType: "agent-a", taskPromptId: tpId }), + ]; + + const result = computeAnalysis(runs, kValues); + expect(result.groups).toHaveLength(1); + expect(result.groups[0].taskPromptId).toBe(tpId); + expect(result.groups[0].task).toBe("Create an API"); + expect(result.groups[0].total).toBe(2); + }); + + it("falls back to computing ID from task text for legacy runs without taskPromptId", () => { + const runs: AnalyzableRun[] = [ + makeRun({ task: "Build a CLI", workerType: "agent-a" }), // no taskPromptId + makeRun({ task: "Build a CLI", workerType: "agent-a" }), // no taskPromptId + ]; + + const result = computeAnalysis(runs, kValues); + expect(result.groups).toHaveLength(1); + expect(result.groups[0].taskPromptId).toBe(computeTaskPromptId("Build a CLI")); + }); + + it("groups mixed runs (with and without taskPromptId) by same computed ID", () => { + const tpId = computeTaskPromptId("Deploy to Azure"); + const runs: AnalyzableRun[] = [ + makeRun({ task: "Deploy to Azure", workerType: "agent-a", taskPromptId: tpId }), + makeRun({ task: "Deploy to Azure", workerType: "agent-a" }), // legacy ā€” no taskPromptId + ]; + + const result = computeAnalysis(runs, kValues); + expect(result.groups).toHaveLength(1); + expect(result.groups[0].total).toBe(2); + }); + + it("separates different tasks into different groups", () => { + const runs: AnalyzableRun[] = [ + makeRun({ task: "Task A", workerType: "agent-a", taskPromptId: computeTaskPromptId("Task A") }), + makeRun({ task: "Task B", workerType: "agent-a", taskPromptId: computeTaskPromptId("Task B") }), + ]; + + const result = computeAnalysis(runs, kValues); + expect(result.groups).toHaveLength(2); + }); + + it("separates same task with different workers into different groups", () => { + const tpId = computeTaskPromptId("Same task"); + const runs: AnalyzableRun[] = [ + makeRun({ task: "Same task", workerType: "agent-a", taskPromptId: tpId }), + makeRun({ task: "Same task", workerType: "agent-b", taskPromptId: tpId }), + ]; + + const result = computeAnalysis(runs, kValues); + expect(result.groups).toHaveLength(2); + const workers = result.groups.map((g) => g.workerType).sort(); + expect(workers).toEqual(["agent-a", "agent-b"]); + }); +}); diff --git a/packages/api/src/analysis.ts b/apps/api/src/analysis.ts similarity index 94% rename from packages/api/src/analysis.ts rename to apps/api/src/analysis.ts index b0e1b1575..c1870eedd 100644 --- a/packages/api/src/analysis.ts +++ b/apps/api/src/analysis.ts @@ -11,6 +11,7 @@ // Types for analysis response export interface TaskWorkerGroup { task: string; + taskPromptId: string; workerType: string; total: number; completed: number; @@ -51,9 +52,12 @@ export interface CriterionResult { evaluated: boolean; } +import { computeTaskPromptId } from 'shared'; + // Run data needed for analysis (subset of RequestDocument) export interface AnalyzableRun { scenario: { task: string; criteria?: string[] }; + taskPromptId?: string; workerType: string; status: string; turns?: Array<{ @@ -188,11 +192,12 @@ export function computeAnalysis( }) : allValidRuns; - // Group by task + workerType + // Group by taskPromptId + workerType (falls back to computing ID from text for legacy runs) const groupMap = new Map(); for (const run of validRuns) { - const key = `${run.scenario.task}|||${run.workerType}`; + const tpId = run.taskPromptId || computeTaskPromptId(run.scenario.task); + const key = `${tpId}|||${run.workerType}`; if (!groupMap.has(key)) { groupMap.set(key, []); } @@ -204,7 +209,8 @@ export function computeAnalysis( let globalMaxT = 1; for (const [key, groupRuns] of groupMap) { - const [task, workerType] = key.split('|||'); + const [taskPromptId, workerType] = key.split('|||'); + const task = groupRuns[0].scenario.task; // Use task text from first run in group const completed = groupRuns.filter(r => r.status === 'completed'); const passedRuns = completed.filter(r => isPassedRun(r, selectedCriteria)); @@ -243,6 +249,7 @@ export function computeAnalysis( groups.push({ task, + taskPromptId, workerType, total: groupRuns.length, completed: completed.length, @@ -256,7 +263,8 @@ export function computeAnalysis( // Now fill in Success@ā‰¤T for all groups using globalMaxT for (const group of groups) { - const groupRuns = groupMap.get(`${group.task}|||${group.workerType}`)!; + const tpId = group.taskPromptId; + const groupRuns = groupMap.get(`${tpId}|||${group.workerType}`)!; const passedRuns = groupRuns.filter(r => r.status === 'completed').filter(r => isPassedRun(r, selectedCriteria)); const passedIterations = passedRuns .map(r => getPassedIteration(r, selectedCriteria)) diff --git a/apps/api/src/criteria-mdp.test.ts b/apps/api/src/criteria-mdp.test.ts new file mode 100644 index 000000000..1c3845c78 --- /dev/null +++ b/apps/api/src/criteria-mdp.test.ts @@ -0,0 +1,602 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect } from "vitest"; +import { + computeMdp, + mergeMdpResponses, + parseStateKey, + type MdpAnalyzableRun, +} from "./criteria-mdp.js"; + +// ā”€ā”€ā”€ Helpers ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +function makeRun( + criteria: string[], + turns: Array>, + status = "completed" +): MdpAnalyzableRun { + return { + scenario: { criteria }, + status, + turns: turns.map((turnState, i) => ({ + iteration: i + 1, + criteriaResults: Object.entries(turnState).map(([id, passed]) => ({ + criterionId: id, + passed, + evaluated: true, + })), + })), + }; +} + +function makeRunWithFeatures( + criteria: string[], + turns: Array>, + features: Array<{ featureId: string; detected: boolean }>, + status = "completed" +): MdpAnalyzableRun { + return { + ...makeRun(criteria, turns, status), + promptFeatures: features.map((f) => ({ ...f, evaluated: true })), + }; +} + +// ā”€ā”€ā”€ Tests ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("computeMdp", () => { + it("returns empty graph for no runs", () => { + const result = computeMdp([]); + expect(result.episodeCount).toBe(0); + // No runs ā†’ no initial node created (nothing to anchor it to) + expect(result.edges.length).toBe(0); + }); + + it("builds correct MDP for a single run with 2 turns", () => { + const run = makeRun(["a", "b"], [ + { a: true, b: false }, // Turn 1: a passed, b failed + { a: true, b: true }, // Turn 2: both passed + ]); + + const result = computeMdp([run]); + + expect(result.episodeCount).toBe(1); + // Nodes: initial (a:0|b:0) + turn1 (a:1|b:0) + turn2 (a:1|b:1) = 3 + expect(result.nodes.length).toBe(3); + + // Initial node + const initial = result.nodes.find((n) => n.isInitial); + expect(initial).toBeDefined(); + expect(initial!.criteria.every((c) => !c.passed)).toBe(true); + expect(initial!.visits).toBe(1); + + // Terminal node (a:1|b:1 ā€” all passed) + const terminal = result.nodes.find((n) => n.isTerminal); + expect(terminal).toBeDefined(); + expect(terminal!.criteria.every((c) => c.passed)).toBe(true); + + // 2 edges: initialā†’turn1, turn1ā†’turn2 + expect(result.edges.length).toBe(2); + for (const edge of result.edges) { + expect(edge.probability).toBe(1.0); + expect(edge.count).toBe(1); + } + }); + + it("merges identical states from different episodes", () => { + const run1 = makeRun(["a", "b"], [ + { a: true, b: false }, + { a: true, b: true }, + ]); + const run2 = makeRun(["a", "b"], [ + { a: true, b: false }, // Same state as run1 turn1 + { a: true, b: true }, + ]); + + const result = computeMdp([run1, run2]); + + expect(result.episodeCount).toBe(2); + // Same 3 unique states as a single run + expect(result.nodes.length).toBe(3); + + // Initial node visited twice + const initial = result.nodes.find((n) => n.isInitial); + expect(initial!.visits).toBe(2); + + // The intermediate state visited twice + const mid = result.nodes.find( + (n) => !n.isInitial && !n.isTerminal + ); + expect(mid).toBeDefined(); + expect(mid!.visits).toBe(2); + + // Edges: all have count=2, probability=1.0 + for (const edge of result.edges) { + expect(edge.count).toBe(2); + expect(edge.probability).toBe(1.0); + } + }); + + it("computes correct probabilities for branching transitions", () => { + // Run 1: initial ā†’ (a:1,b:0) ā†’ (a:1,b:1) + const run1 = makeRun(["a", "b"], [ + { a: true, b: false }, + { a: true, b: true }, + ]); + // Run 2: initial ā†’ (a:0,b:1) ā†’ (a:1,b:1) + const run2 = makeRun(["a", "b"], [ + { a: false, b: true }, + { a: true, b: true }, + ]); + + const result = computeMdp([run1, run2]); + + expect(result.episodeCount).toBe(2); + // Nodes: initial + (a:1,b:0) + (a:0,b:1) + (a:1,b:1) = 4 + expect(result.nodes.length).toBe(4); + + // Initial should have 2 outgoing edges, each with probability 0.5 + const initialEdges = result.edges.filter((e) => { + const initial = result.nodes.find((n) => n.isInitial); + return e.source === initial!.id; + }); + expect(initialEdges.length).toBe(2); + for (const edge of initialEdges) { + expect(edge.probability).toBeCloseTo(0.5); + expect(edge.count).toBe(1); + } + }); + + it("handles self-loops (state doesn't change between turns)", () => { + const run = makeRun(["a"], [ + { a: false }, // Turn 1 + { a: false }, // Turn 2 ā€” same state + { a: true }, // Turn 3 + ]); + + const result = computeMdp([run]); + + expect(result.episodeCount).toBe(1); + // Nodes: initial (a:0) + (a:1) = 2 (initial and turn1/2 are the same state a:0) + expect(result.nodes.length).toBe(2); + + // There should be a self-loop on the initial/a:0 state + // Count=2: initialā†’turn1(a:0) + turn1(a:0)ā†’turn2(a:0) + const selfLoop = result.edges.find((e) => e.source === e.target); + expect(selfLoop).toBeDefined(); + expect(selfLoop!.count).toBe(2); + }); + + it("projects to selected criteria subset", () => { + // Run has 3 criteria but we project to only [a, b] + const run = makeRun(["a", "b", "c"], [ + { a: true, b: false, c: true }, + { a: true, b: true, c: false }, + ]); + + const result = computeMdp([run], ["a", "b"]); + + expect(result.selectedCriteria).toEqual(["a", "b"]); + // All state vectors should only have a and b + for (const node of result.nodes) { + const ids = node.criteria.map((c) => c.id); + expect(ids).toContain("a"); + expect(ids).toContain("b"); + expect(ids).not.toContain("c"); + } + }); + + it("collapses states that differ only in excluded criteria", () => { + // Run 1: turns with c=true + const run1 = makeRun(["a", "b", "c"], [ + { a: true, b: false, c: true }, + ]); + // Run 2: turns with c=false but same a,b + const run2 = makeRun(["a", "b", "c"], [ + { a: true, b: false, c: false }, + ]); + + // Without projection: 2 distinct turn1 states + const full = computeMdp([run1, run2]); + const nonInitialFull = full.nodes.filter((n) => !n.isInitial); + expect(nonInitialFull.length).toBe(2); + + // With projection to [a, b]: turn1 states collapse into 1 + const projected = computeMdp([run1, run2], ["a", "b"]); + const nonInitialProjected = projected.nodes.filter((n) => !n.isInitial); + expect(nonInitialProjected.length).toBe(1); + expect(nonInitialProjected[0].visits).toBe(2); + }); + + it("handles single-turn episodes", () => { + const run = makeRun(["a"], [{ a: true }]); + const result = computeMdp([run]); + + expect(result.episodeCount).toBe(1); + // initial + terminal = 2 nodes + expect(result.nodes.length).toBe(2); + // 1 edge: initial ā†’ terminal + expect(result.edges.length).toBe(1); + expect(result.edges[0].probability).toBe(1.0); + }); + + it("skips runs with no turns", () => { + const run: MdpAnalyzableRun = { + scenario: { criteria: ["a"] }, + status: "completed", + turns: [], + }; + const result = computeMdp([run]); + expect(result.episodeCount).toBe(0); + }); + + it("treats unevaluated criteria as failed", () => { + const run: MdpAnalyzableRun = { + scenario: { criteria: ["a", "b"] }, + status: "completed", + turns: [ + { + iteration: 1, + criteriaResults: [ + { criterionId: "a", passed: true, evaluated: true }, + { criterionId: "b", passed: false, evaluated: false }, // Skipped + ], + }, + ], + }; + const result = computeMdp([run]); + const terminal = result.nodes.find((n) => n.isTerminal); + const bState = terminal!.criteria.find((c) => c.id === "b"); + expect(bState!.passed).toBe(false); + }); + + it("includes availableCriteria from all runs", () => { + const run1 = makeRun(["a", "b"], [{ a: true, b: true }]); + const run2 = makeRun(["b", "c"], [{ b: true, c: false }]); + const result = computeMdp([run1, run2]); + expect(result.availableCriteria).toEqual(["a", "b", "c"]); + }); + + it("probabilities sum to 1.0 for each source node", () => { + const run1 = makeRun(["a", "b"], [ + { a: true, b: false }, + { a: true, b: true }, + ]); + const run2 = makeRun(["a", "b"], [ + { a: false, b: true }, + { a: true, b: true }, + ]); + const run3 = makeRun(["a", "b"], [ + { a: true, b: false }, + { a: false, b: true }, + ]); + + const result = computeMdp([run1, run2, run3]); + + // Group edges by source and check probabilities sum to ~1.0 + const bySource = new Map(); + for (const edge of result.edges) { + bySource.set( + edge.source, + (bySource.get(edge.source) || 0) + edge.probability + ); + } + for (const [, total] of bySource) { + expect(total).toBeCloseTo(1.0); + } + }); +}); + +describe("mergeMdpResponses", () => { + it("merges nodes by summing visits", () => { + const base = computeMdp([ + makeRun(["a"], [{ a: true }]), + ]); + const delta = computeMdp([ + makeRun(["a"], [{ a: true }]), + ]); + + const merged = mergeMdpResponses(base, delta); + + expect(merged.episodeCount).toBe(2); + const terminal = merged.nodes.find((n) => n.isTerminal); + expect(terminal!.visits).toBe(2); + }); + + it("adds new nodes from delta", () => { + const base = computeMdp([ + makeRun(["a"], [{ a: true }]), + ]); + const delta = computeMdp([ + makeRun(["a"], [{ a: false }]), + ]); + + const merged = mergeMdpResponses(base, delta); + + // base has: initial (a:0), terminal (a:1) + // delta has: initial (a:0), terminal (a:0) ā€” but a:0 is the initial state + // Actually delta terminal is a:0 which is the same as initial + // So merged should have 2 nodes: a:0 (initial) and a:1: + expect(merged.nodes.length).toBe(2); + }); + + it("recomputes probabilities after merge", () => { + // Base: initial ā†’ a:1 (100%) + const base = computeMdp([ + makeRun(["a"], [{ a: true }]), + ]); + // Delta: initial ā†’ a:0 (self-loop, then a:1) + const delta = computeMdp([ + makeRun(["a"], [{ a: false }, { a: true }]), + ]); + + const merged = mergeMdpResponses(base, delta); + + // From initial (a:0), there should be 2 outgoing edges: + // initial ā†’ a:1 (from base) and initial ā†’ a:0 (self-loop from delta) + const initial = merged.nodes.find((n) => n.isInitial)!; + const fromInitial = merged.edges.filter((e) => e.source === initial.id); + const totalProb = fromInitial.reduce((sum, e) => sum + e.probability, 0); + expect(totalProb).toBeCloseTo(1.0); + }); + + it("returns existing with updated timestamp when delta is empty", () => { + const base = computeMdp([ + makeRun(["a"], [{ a: true }]), + ]); + const emptyDelta: typeof base = { + ...base, + nodes: [], + edges: [], + episodeCount: 0, + computedAt: "2026-01-01T00:00:00.000Z", + }; + + const merged = mergeMdpResponses(base, emptyDelta); + expect(merged.episodeCount).toBe(base.episodeCount); + expect(merged.nodes.length).toBe(base.nodes.length); + expect(merged.computedAt).toBe("2026-01-01T00:00:00.000Z"); + }); + + it("merges availableCriteria from both", () => { + const base = computeMdp([makeRun(["a"], [{ a: true }])]); + const delta = computeMdp([makeRun(["b"], [{ b: false }])]); + + const merged = mergeMdpResponses(base, delta); + expect(merged.availableCriteria).toContain("a"); + expect(merged.availableCriteria).toContain("b"); + }); + + it("merges availablePromptFeatures from both", () => { + const base = computeMdp([ + makeRunWithFeatures(["a"], [{ a: true }], [{ featureId: "f1", detected: true }]), + ]); + const delta = computeMdp([ + makeRunWithFeatures(["a"], [{ a: false }], [{ featureId: "f2", detected: false }]), + ]); + + const merged = mergeMdpResponses(base, delta); + expect(merged.availablePromptFeatures).toContain("f1"); + expect(merged.availablePromptFeatures).toContain("f2"); + }); +}); + +// ā”€ā”€ā”€ Prompt Feature Start Nodes ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("computeMdp with prompt features", () => { + it("uses prompt features as start nodes instead of synthetic initial", () => { + const run = makeRunWithFeatures( + ["a", "b"], + [{ a: true, b: false }, { a: true, b: true }], + [{ featureId: "azure", detected: true }, { featureId: "docker", detected: false }] + ); + + const result = computeMdp([run]); + + // Start node should be a prompt-features node + const startNode = result.nodes.find((n) => n.isInitial); + expect(startNode).toBeDefined(); + expect(startNode!.type).toBe("prompt-features"); + expect(startNode!.features).toBeDefined(); + expect(startNode!.features!.length).toBe(2); + expect(startNode!.features!.find((f) => f.id === "azure")?.detected).toBe(true); + expect(startNode!.features!.find((f) => f.id === "docker")?.detected).toBe(false); + // Criteria should be empty on feature nodes + expect(startNode!.criteria.length).toBe(0); + + // Criteria state nodes should have type "criteria" + const criteriaNodes = result.nodes.filter((n) => n.type === "criteria"); + expect(criteriaNodes.length).toBe(2); // turn1 + turn2 states + }); + + it("groups runs with same feature vector into same start node", () => { + const run1 = makeRunWithFeatures( + ["a"], + [{ a: true }], + [{ featureId: "azure", detected: true }] + ); + const run2 = makeRunWithFeatures( + ["a"], + [{ a: false }], + [{ featureId: "azure", detected: true }] + ); + + const result = computeMdp([run1, run2]); + + // Both runs share the same feature start node + const startNodes = result.nodes.filter((n) => n.isInitial); + expect(startNodes.length).toBe(1); + expect(startNodes[0].visits).toBe(2); + expect(startNodes[0].type).toBe("prompt-features"); + }); + + it("creates separate start nodes for different feature vectors", () => { + const run1 = makeRunWithFeatures( + ["a"], + [{ a: true }], + [{ featureId: "azure", detected: true }] + ); + const run2 = makeRunWithFeatures( + ["a"], + [{ a: false }], + [{ featureId: "azure", detected: false }] + ); + + const result = computeMdp([run1, run2]); + + const startNodes = result.nodes.filter((n) => n.isInitial); + expect(startNodes.length).toBe(2); + expect(startNodes.every((n) => n.type === "prompt-features")).toBe(true); + }); + + it("creates 'Unknown' start node for runs without features", () => { + const runWithFeatures = makeRunWithFeatures( + ["a"], + [{ a: true }], + [{ featureId: "azure", detected: true }] + ); + const runWithoutFeatures = makeRun(["a"], [{ a: false }]); + + const result = computeMdp([runWithFeatures, runWithoutFeatures]); + + const startNodes = result.nodes.filter((n) => n.isInitial); + expect(startNodes.length).toBe(2); + + // One known, one unknown + const unknownNode = startNodes.find((n) => n.id === "F||unknown"); + expect(unknownNode).toBeDefined(); + expect(unknownNode!.features).toEqual([]); + expect(unknownNode!.visits).toBe(1); + }); + + it("includes availablePromptFeatures in response", () => { + const run = makeRunWithFeatures( + ["a"], + [{ a: true }], + [{ featureId: "azure", detected: true }, { featureId: "docker", detected: false }] + ); + + const result = computeMdp([run]); + expect(result.availablePromptFeatures).toEqual(["azure", "docker"]); + }); + + it("filters runs by selectedFeatures (AND logic)", () => { + const run1 = makeRunWithFeatures( + ["a"], + [{ a: true }], + [{ featureId: "azure", detected: true }, { featureId: "docker", detected: true }] + ); + const run2 = makeRunWithFeatures( + ["a"], + [{ a: false }], + [{ featureId: "azure", detected: true }] // missing docker + ); + + const result = computeMdp([run1, run2], undefined, ["azure", "docker"]); + + // Only run1 should be included (run2 doesn't have docker feature evaluated) + expect(result.episodeCount).toBe(1); + expect(result.selectedFeatures).toEqual(["azure", "docker"]); + }); + + it("projects feature start nodes to selected features only", () => { + const run = makeRunWithFeatures( + ["a"], + [{ a: true }], + [ + { featureId: "azure", detected: true }, + { featureId: "docker", detected: false }, + { featureId: "k8s", detected: true }, + ] + ); + + const result = computeMdp([run], undefined, ["azure", "docker"]); + + const startNode = result.nodes.find((n) => n.isInitial); + expect(startNode!.features!.length).toBe(2); + const featureIds = startNode!.features!.map((f) => f.id); + expect(featureIds).toContain("azure"); + expect(featureIds).toContain("docker"); + expect(featureIds).not.toContain("k8s"); + }); + + it("applies both criteria AND features filters", () => { + // Run1: has criteria a,b and feature azure + const run1 = makeRunWithFeatures( + ["a", "b"], + [{ a: true, b: true }], + [{ featureId: "azure", detected: true }] + ); + // Run2: has criteria a (missing b) and feature azure + const run2 = makeRunWithFeatures( + ["a"], + [{ a: true }], + [{ featureId: "azure", detected: true }] + ); + // Run3: has criteria a,b but no features + const run3 = makeRun(["a", "b"], [{ a: true, b: false }]); + + const result = computeMdp([run1, run2, run3], ["a", "b"], ["azure"]); + + // Only run1 passes both filters + expect(result.episodeCount).toBe(1); + }); + + it("falls back to synthetic initial when no runs have features", () => { + const run = makeRun(["a"], [{ a: true }]); + const result = computeMdp([run]); + + const startNode = result.nodes.find((n) => n.isInitial); + expect(startNode).toBeDefined(); + // No type set (backward compat) or undefined + expect(startNode!.criteria.length).toBeGreaterThan(0); + expect(result.availablePromptFeatures).toEqual([]); + }); +}); + +// ā”€ā”€ā”€ parseStateKey ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("parseStateKey", () => { + it("parses a single criterion", () => { + expect(parseStateKey("has_azure:1")).toEqual([ + { id: "has_azure", passed: true }, + ]); + }); + + it("parses multiple criteria", () => { + expect(parseStateKey("has_azure:0|has_cloud:1")).toEqual([ + { id: "has_azure", passed: false }, + { id: "has_cloud", passed: true }, + ]); + }); + + it("returns empty array for empty string", () => { + expect(parseStateKey("")).toEqual([]); + }); + + it("returns empty array for whitespace-only string", () => { + expect(parseStateKey(" ")).toEqual([]); + }); + + it("handles criteria with colons in the ID", () => { + // e.g. a namespaced criterion "ns:foo:1" ā†’ id="ns:foo", passed=true + expect(parseStateKey("ns:foo:1")).toEqual([ + { id: "ns:foo", passed: true }, + ]); + }); + + it("round-trips with computeMdp stateKey format", () => { + // Verify parseStateKey can parse output from computeMdp nodes + const run = makeRun(["b", "a"], [{ a: true, b: false }]); + const result = computeMdp([run]); + const criteriaNode = result.nodes.find( + (n) => n.type !== "prompt-features" && !n.isInitial && n.criteria.length > 0 + ); + if (criteriaNode) { + const parsed = parseStateKey(criteriaNode.id); + expect(parsed).toEqual( + criteriaNode.criteria.map((c) => ({ id: c.id, passed: c.passed })) + ); + } + }); +}); diff --git a/apps/api/src/criteria-mdp.ts b/apps/api/src/criteria-mdp.ts new file mode 100644 index 000000000..de34344d9 --- /dev/null +++ b/apps/api/src/criteria-mdp.ts @@ -0,0 +1,489 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * MDP (Markov Decision Process) state transition module + * + * Computes a state-transition graph where: + * - Each node is a unique composite criteria state vector (pass/fail per criterion) + * - Each edge is a transition between states across consecutive iterations, + * labeled with a count and probability + * - An initial "all-failed" state is prepended to every episode + */ + +// ā”€ā”€ā”€ Types ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** Per-criterion state within a composite state vector */ +export interface CriterionState { + id: string; + passed: boolean; +} + +/** Per-feature state within a prompt-feature start node */ +export interface FeatureState { + id: string; + detected: boolean; +} + +/** Node type discriminator */ +export type MdpNodeType = "prompt-features" | "criteria"; + +/** A node in the MDP graph ā€” a unique composite state vector */ +export interface MdpStateNode { + /** Canonical string key (e.g. "has_azure:0|has_cloud:1|has_iac:0") */ + id: string; + /** Sorted criteria states (present on criteria nodes) */ + criteria: CriterionState[]; + /** Sorted feature states (present on prompt-feature start nodes) */ + features?: FeatureState[]; + /** Node type: "prompt-features" for start nodes, "criteria" for state nodes */ + type?: MdpNodeType; + /** How many times any episode visited this state */ + visits: number; + /** True for the start state (prompt-features node) */ + isInitial?: boolean; + /** True if no outgoing transitions exist (final state of some episodes) */ + isTerminal?: boolean; +} + +/** An edge in the MDP graph ā€” a transition between two states */ +export interface MdpTransitionEdge { + source: string; + target: string; + /** How many times this specific transition was observed */ + count: number; + /** Probability: count / total outgoing from source */ + probability: number; +} + +/** Full MDP response */ +export interface MdpResponse { + nodes: MdpStateNode[]; + edges: MdpTransitionEdge[]; + /** Total number of episodes (runs) that contributed */ + episodeCount: number; + /** All criteria IDs found across all runs (before filtering) */ + availableCriteria: string[]; + /** Criteria IDs used for projection (empty = all) */ + selectedCriteria: string[]; + /** All prompt feature IDs found across all runs */ + availablePromptFeatures: string[]; + /** Prompt feature IDs used for filtering (empty = all) */ + selectedFeatures: string[]; + /** ISO timestamp of when this was computed ā€” used for incremental polling */ + computedAt: string; +} + +/** Minimal run shape needed for MDP computation (subset of RequestDocument) */ +export interface MdpAnalyzableRun { + scenario: { criteria?: string[] }; + status: string; + updatedAt?: Date | string; + /** Prompt features from the associated task prompt */ + promptFeatures?: Array<{ + featureId: string; + detected: boolean; + evaluated: boolean; + }>; + turns?: Array<{ + iteration: number; + criteriaResults?: Array<{ + criterionId: string; + passed: boolean; + evaluated: boolean; + }>; + }>; +} + +// ā”€ā”€ā”€ Helpers ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Build a canonical state key from a criteria state vector. + * Criteria are sorted alphabetically by ID so the key is deterministic. + */ +function stateKey(criteria: CriterionState[]): string { + return criteria + .slice() + .sort((a, b) => a.id.localeCompare(b.id)) + .map((c) => `${c.id}:${c.passed ? 1 : 0}`) + .join("|"); +} + +/** + * Parse a state key string back into a CriterionState array. + * Input: "has_azure:0|has_cloud:1" + * Output: [{id: "has_azure", passed: false}, {id: "has_cloud", passed: true}] + */ +export function parseStateKey(key: string): CriterionState[] { + if (!key || key.trim() === "") return []; + return key.split("|").map((part) => { + const lastColon = part.lastIndexOf(":"); + if (lastColon === -1) return { id: part, passed: false }; + const id = part.slice(0, lastColon); + const passed = part.slice(lastColon + 1) === "1"; + return { id, passed }; + }); +} + +/** + * Build a canonical key for a prompt-feature start node. + * Prefixed with "F||" to avoid collision with criteria state keys. + */ +function featureNodeKey(features: FeatureState[]): string { + const sorted = features.slice().sort((a, b) => a.id.localeCompare(b.id)); + return "F||" + sorted.map((f) => `${f.id}:${f.detected ? 1 : 0}`).join("|"); +} + +/** + * Build a FeatureState[] from a run's prompt features, projected to selectedFeatures. + * If the run has no prompt features, returns null (for the "Unknown" start node). + */ +function buildFeatureVector( + run: MdpAnalyzableRun, + featureIds: string[] +): FeatureState[] | null { + if (!run.promptFeatures || run.promptFeatures.length === 0) return null; + const map = new Map(run.promptFeatures.map((f) => [f.featureId, f])); + return featureIds.map((id) => { + const f = map.get(id); + return { id, detected: f ? f.detected : false }; + }); +} + +/** + * Build a criteria state vector for a given turn. + * `evaluated: false` is treated as `passed: false` (skipped = unmet). + * Only criteria in `criteriaIds` are included. + */ +function buildStateVector( + turn: NonNullable[number], + criteriaIds: string[] +): CriterionState[] { + const resultsMap = new Map( + (turn.criteriaResults || []).map((r) => [r.criterionId, r]) + ); + return criteriaIds.map((id) => { + const r = resultsMap.get(id); + return { id, passed: r ? r.evaluated && r.passed : false }; + }); +} + +// ā”€ā”€ā”€ Core computation ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Compute the MDP state-transition graph from a set of runs. + * + * @param runs Completed/failed/exhausted runs with turns + criteriaResults + * @param selectedCriteria Optional subset of criteria to project to. If provided, + * only these criteria are included in the state vectors (producing a sub-MDP where + * states that only differed in excluded criteria are merged). + * @param selectedFeatures Optional subset of prompt features to filter by. If provided, + * only runs whose task prompt has ALL selected features evaluated are included, + * and start nodes are built from these features. + */ +export function computeMdp( + runs: MdpAnalyzableRun[], + selectedCriteria?: string[], + selectedFeatures?: string[] +): MdpResponse { + // Collect all criteria IDs across all runs + const allCriteriaSet = new Set(); + for (const run of runs) { + if (run.scenario?.criteria) { + for (const c of run.scenario.criteria) { + allCriteriaSet.add(c); + } + } + // Also collect from turns in case scenario.criteria is incomplete + if (run.turns) { + for (const turn of run.turns) { + if (turn.criteriaResults) { + for (const r of turn.criteriaResults) { + allCriteriaSet.add(r.criterionId); + } + } + } + } + } + const availableCriteria = Array.from(allCriteriaSet).sort(); + + // Collect all prompt feature IDs across all runs + const allFeaturesSet = new Set(); + for (const run of runs) { + if (run.promptFeatures) { + for (const f of run.promptFeatures) { + allFeaturesSet.add(f.featureId); + } + } + } + const availablePromptFeatures = Array.from(allFeaturesSet).sort(); + + // Determine which criteria to include in state vectors + const projectionCriteria = + selectedCriteria && selectedCriteria.length > 0 + ? selectedCriteria.slice().sort() + : availableCriteria; + + // Determine which features to include in start nodes + const projectionFeatures = + selectedFeatures && selectedFeatures.length > 0 + ? selectedFeatures.slice().sort() + : availablePromptFeatures; + + // Filter runs: if selectedCriteria is provided, only include runs that have + // all of the selected criteria (so the state vector is complete and meaningful) + let filteredRuns = runs; + if (selectedCriteria && selectedCriteria.length > 0) { + filteredRuns = filteredRuns.filter((run) => { + const runCriteria = new Set(); + if (run.scenario?.criteria) { + for (const c of run.scenario.criteria) runCriteria.add(c); + } + if (run.turns) { + for (const turn of run.turns) { + if (turn.criteriaResults) { + for (const r of turn.criteriaResults) runCriteria.add(r.criterionId); + } + } + } + return selectedCriteria.every((c) => runCriteria.has(c)); + }); + } + + // Filter runs: if selectedFeatures is provided, only include runs that have + // all of the selected features evaluated in their prompt features + if (selectedFeatures && selectedFeatures.length > 0) { + filteredRuns = filteredRuns.filter((run) => { + if (!run.promptFeatures) return false; + const runFeatures = new Set(run.promptFeatures.map((f) => f.featureId)); + return selectedFeatures.every((f) => runFeatures.has(f)); + }); + } + + // The "Unknown" start node key ā€” for runs without prompt features + const UNKNOWN_KEY = "F||unknown"; + + // Accumulators + const nodesMap = new Map(); + const edgesMap = new Map(); + const outgoingCounts = new Map(); // source -> total outgoing transitions + + // Build synthetic initial state (all criteria failed) ā€” fallback when no features exist + const initialCriteria: CriterionState[] = projectionCriteria.map((id) => ({ + id, + passed: false, + })); + const syntheticInitialKey = stateKey(initialCriteria); + + let episodeCount = 0; + + for (const run of filteredRuns) { + if (!run.turns || run.turns.length === 0) continue; + + // Determine start node: prompt-features if available, else synthetic all-failed + let startKey: string; + if (projectionFeatures.length > 0) { + const featureVec = buildFeatureVector(run, projectionFeatures); + if (featureVec) { + startKey = featureNodeKey(featureVec); + if (!nodesMap.has(startKey)) { + nodesMap.set(startKey, { + id: startKey, + criteria: [], + features: featureVec.slice().sort((a, b) => a.id.localeCompare(b.id)), + type: "prompt-features", + visits: 0, + isInitial: true, + }); + } + } else { + // Run has no prompt features ā†’ "Unknown" start node + startKey = UNKNOWN_KEY; + if (!nodesMap.has(startKey)) { + nodesMap.set(startKey, { + id: startKey, + criteria: [], + features: [], + type: "prompt-features", + visits: 0, + isInitial: true, + }); + } + } + } else { + // No features available at all ā†’ use synthetic all-failed initial state + startKey = syntheticInitialKey; + if (!nodesMap.has(startKey)) { + nodesMap.set(startKey, { + id: startKey, + criteria: initialCriteria, + visits: 0, + isInitial: true, + }); + } + } + + // Sort turns by iteration + const sortedTurns = run.turns.slice().sort((a, b) => a.iteration - b.iteration); + + // Build the sequence of state keys for this episode + const states: string[] = [startKey]; + + for (const turn of sortedTurns) { + const vec = buildStateVector(turn, projectionCriteria); + const key = stateKey(vec); + states.push(key); + + // Ensure criteria node exists + if (!nodesMap.has(key)) { + nodesMap.set(key, { + id: key, + criteria: vec.slice().sort((a, b) => a.id.localeCompare(b.id)), + type: "criteria", + visits: 0, + }); + } + } + + // Record visits and transitions + for (let i = 0; i < states.length; i++) { + const node = nodesMap.get(states[i])!; + node.visits++; + + if (i < states.length - 1) { + const edgeKey = `${states[i]}||${states[i + 1]}`; + const existing = edgesMap.get(edgeKey); + if (existing) { + existing.count++; + } else { + edgesMap.set(edgeKey, { + source: states[i], + target: states[i + 1], + count: 1, + }); + } + outgoingCounts.set( + states[i], + (outgoingCounts.get(states[i]) || 0) + 1 + ); + } + } + + episodeCount++; + } + + // Mark terminal nodes (no outgoing transitions) + for (const node of nodesMap.values()) { + if (!outgoingCounts.has(node.id)) { + node.isTerminal = true; + } + } + + // Build edges with probabilities + const edges: MdpTransitionEdge[] = []; + for (const edge of edgesMap.values()) { + const totalOutgoing = outgoingCounts.get(edge.source) || 1; + edges.push({ + source: edge.source, + target: edge.target, + count: edge.count, + probability: edge.count / totalOutgoing, + }); + } + + return { + nodes: Array.from(nodesMap.values()), + edges, + episodeCount, + availableCriteria, + selectedCriteria: selectedCriteria || [], + availablePromptFeatures, + selectedFeatures: selectedFeatures || [], + computedAt: new Date().toISOString(), + }; +} + +// ā”€ā”€ā”€ Merge utility (for incremental updates) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Merge a delta MDP response into an existing one. + * Nodes are merged by ID (visits summed), edges by source+target (counts summed), + * probabilities recomputed. + */ +export function mergeMdpResponses( + existing: MdpResponse, + delta: MdpResponse +): MdpResponse { + // Merge nodes + const nodesMap = new Map(); + for (const node of existing.nodes) { + nodesMap.set(node.id, { ...node }); + } + for (const node of delta.nodes) { + const prev = nodesMap.get(node.id); + if (prev) { + prev.visits += node.visits; + // Keep isInitial if either has it + if (node.isInitial) prev.isInitial = true; + } else { + nodesMap.set(node.id, { ...node }); + } + } + + // Merge edges (sum counts) + const edgesMap = new Map(); + for (const edge of existing.edges) { + const key = `${edge.source}||${edge.target}`; + edgesMap.set(key, { ...edge }); + } + for (const edge of delta.edges) { + const key = `${edge.source}||${edge.target}`; + const prev = edgesMap.get(key); + if (prev) { + prev.count += edge.count; + } else { + edgesMap.set(key, { ...edge }); + } + } + + // Recompute probabilities from merged counts + const outgoingCounts = new Map(); + for (const edge of edgesMap.values()) { + outgoingCounts.set( + edge.source, + (outgoingCounts.get(edge.source) || 0) + edge.count + ); + } + for (const edge of edgesMap.values()) { + const total = outgoingCounts.get(edge.source) || 1; + edge.probability = edge.count / total; + } + + // Recompute terminal status + for (const node of nodesMap.values()) { + node.isTerminal = !outgoingCounts.has(node.id); + } + + // Merge available criteria + const allCriteria = new Set([ + ...existing.availableCriteria, + ...delta.availableCriteria, + ]); + + // Merge available prompt features + const allFeatures = new Set([ + ...(existing.availablePromptFeatures || []), + ...(delta.availablePromptFeatures || []), + ]); + + return { + nodes: Array.from(nodesMap.values()), + edges: Array.from(edgesMap.values()), + episodeCount: existing.episodeCount + delta.episodeCount, + availableCriteria: Array.from(allCriteria).sort(), + selectedCriteria: delta.selectedCriteria, // Use the latest selection + availablePromptFeatures: Array.from(allFeatures).sort(), + selectedFeatures: delta.selectedFeatures || [], + computedAt: delta.computedAt, + }; +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts new file mode 100644 index 000000000..76bba69f6 --- /dev/null +++ b/apps/api/src/index.ts @@ -0,0 +1,4709 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import express, { Request, Response, NextFunction } from "express"; +import cors from "cors"; +import { MongoClient, Db, Collection } from "mongodb"; +import { QueueClient } from "@azure/storage-queue"; +import { DefaultAzureCredential } from "@azure/identity"; +import { BlobServiceClient, RestError } from "@azure/storage-blob"; +import { v4 as uuidv4 } from "uuid"; +import { createRequire } from "module"; +import dotenv from "dotenv"; +import multer from "multer"; +import { execSync } from "child_process"; +import { mkdtempSync, rmSync, existsSync, readdirSync, statSync } from "fs"; +import { readFile } from "fs/promises"; +import { tmpdir } from "os"; +import { join, basename } from "path"; +import { createHash } from "crypto"; +import { parse as yamlParse, stringify as yamlStringify } from "yaml"; +import { pack as tarPack } from "tar-stream"; +import { createGzip } from "zlib"; +import { pipeline } from "stream/promises"; +import { isLlmAvailable, generateCriteriaPrompt } from "./llm.js"; +import { + isLlmAvailable as isPromptFeatureLlmAvailable, + generatePromptFeaturePrompt, + extractPromptFeatures, +} from "./prompt-feature-llm.js"; +import { computeAnalysis, AnalysisResponse, AnalyzableRun } from "./analysis.js"; +import { computeMdp, parseStateKey, type MdpAnalyzableRun } from "./criteria-mdp.js"; +import { TaskPromptStore, computeTaskPromptId, type TaskPromptDocument, SkillRevisionStore, SkillResolver, type SkillDocument, type SkillRevisionDocument, type SkillSearchResult } from "shared"; +import { evaluateTrigger } from "shared"; +import { checkMigrations } from "db-migrations/check-migrations"; + +const require = createRequire(import.meta.url); +const Redis = require("ioredis"); + +dotenv.config(); + +const app = express(); +app.use(cors()); +app.use(express.json()); + +// Multer configuration for file uploads (stored in temp directory) +const upload = multer({ dest: tmpdir() }); + +// Configuration from environment +// K8s: MONGO_CONNECTION_STRING from secret, STORAGE_CONNECTION_STRING from secret +const mongoUri = process.env.MONGO_CONNECTION_STRING || process.env.MONGO_URI || "mongodb://localhost:27000"; +const mongoDatabase = process.env.MONGO_DATABASE || "requests-db"; +const mongoCollection = process.env.MONGO_COLLECTION || "requests"; +const storageAccountName = process.env.AZURE_STORAGE_ACCOUNT_NAME || ""; +const storageConnectionString = process.env.STORAGE_CONNECTION_STRING || process.env.AZURE_STORAGE_CONNECTION_STRING || ""; +const queueWorker1 = process.env.AZURE_STORAGE_QUEUE_WORKER_1 || "queue-coder-acp-claude-code"; +const queueWorker2 = process.env.AZURE_STORAGE_QUEUE_WORKER_2 || "queue-coder-acp-copilot"; +const queueReport = process.env.AZURE_STORAGE_QUEUE_REPORT || "report-queue"; +const redisHost = process.env.REDIS_HOST || ""; +const redisPort = parseInt(process.env.REDIS_PORT || "6300", 10); +const redisPassword = process.env.REDIS_PASSWORD || ""; +const port = parseInt(process.env.PORT || "3000", 10); + +// Version information (injected at build time) +const GIT_COMMIT = process.env.GIT_COMMIT || "development"; +const BUILD_TIME = process.env.BUILD_TIME || new Date().toISOString(); + +// Valid worker types +const VALID_WORKERS = ["coder-acp-claude-code", "coder-acp-copilot"] as const; +type WorkerType = (typeof VALID_WORKERS)[number]; + +// MongoDB clients +let mongoClient: MongoClient; +let db: Db; +let collection: Collection; +let criteriaCollection: Collection; +let promptFeatureCollection: Collection; +let promptFeatureExtractionCollection: Collection; +let reportCollection: Collection; +let agentCollection: Collection; +let modelCollection: Collection; +let mcpServerCollection: Collection; +let insightsCollection: Collection; +let taskPromptCollection: Collection; +let taskPromptStore: TaskPromptStore; +let featureFlagCollection: Collection; +let reportTemplateCollection: Collection; +let skillCollection: Collection; +let skillRevisionCollection: Collection; +let skillRevisionStore: SkillRevisionStore; +let skillResolver: SkillResolver; +const queueClients: Map = new Map(); +let reportQueueClient: QueueClient; + +// Criteria document interface +interface CriteriaDocument { + id: string; + prompt: string; + dependsOn?: string[]; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +// Prompt feature document interfaces +interface PromptFeatureDocument { + id: string; + prompt: string; + dependsOn?: string[]; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +interface PromptFeatureExtractionDocument { + _id?: string; + taskText: string; + taskTextHash: string; + promptFeatureResults: Array<{ featureId: string; detected: boolean; evaluated: boolean }>; + suggestedFeatures?: Array<{ suggestedId: string; behavior: string; prompt: string }>; + extractedAt: Date; + model?: string; +} + +// Insight reference interface (embedded in ReportDocument) +interface InsightReference { + insightId: string; + referencedAt: Date; + isNew: boolean; +} + +// Report document interface +interface ReportDocument { + _id: string; + requestId: string; + templateId?: string; // FK ā†’ report-templates.id (which template generated this report) + reporter?: { + id: string; + name: string; + gitHash: string; + model: string; + agentId: string; + agentVersion: string; + }; + content?: string; + status: "pending" | "generating" | "completed" | "failed"; + error?: string; + logs: LogEvent[]; + insightReferences?: InsightReference[]; + createdAt: Date; + updatedAt?: Date; +} + +// Report trigger types (discriminated union) +type ReportTrigger = + | { type: "always" } + | { type: "criteria"; criteriaIds: string[]; match?: "any" | "all" } + | { type: "taskPrompt"; taskPromptIds: string[] } + | { type: "promptFeature"; featureIds: string[]; match?: "any" | "all" }; + +// Report template document interface +interface ReportTemplateDocument { + id: string; // Human-readable slug (e.g. "default", "failure-analysis") + name: string; // Display name + description?: string; + userPrompt: string; // REQUIRED ā€” the agent instruction + systemPrompt?: { // OPTIONAL ā€” customize base system prompt + mode: "append" | "override"; + content: string; + }; + trigger?: ReportTrigger; // OPTIONAL ā€” omit = "always" + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +// Insight document interface +interface InsightDocument { + _id: string; + title: string; + /** Markdown-formatted detailed observation */ + description: string; + category?: string; + tags?: string[]; + upvotes: number; + downvotes: number; + blocked: boolean; + referenceCount: number; + createdBy: "agent" | "user"; + sourceReportId?: string; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +// Log event interface +interface LogEvent { + timestamp: string; + level: "info" | "warn" | "error" | "debug"; + source?: string; + message: string; + data?: Record; +} + +// Request document interface +interface RequestDocument { + _id: string; + scenario: { task: string; criteria: string[]; version?: 'v1' | 'v2' }; + workerType: WorkerType; + model?: string; + status: "pending" | "processing" | "iterating" | "completed" | "failed" | "exhausted"; + result?: string; + error?: string; + logs?: LogEvent[]; + maxIterations?: number; + turns?: Array<{ + iteration: number; + codingAgentResponse: string; + judgeFeedback: string; + snapshotUrl: string; + passed: boolean; + timestamp: Date; + harUrl?: string; + videoUrls?: string[]; + }>; + personaInstructions?: string; + persona?: { personality: string; experience: string; verbosity: string; type: string }; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; + taskPromptId?: string; // Materialized UUIDv5 of scenario.task (FK ā†’ task-prompts._id) + mcpServers?: string[]; // MCP server slugs selected for this run + skillRevisions?: string[]; // Human-readable skill revision refs (source/skillName@commitHash) + harUrl?: string; + videoUrls?: string[]; +} + +// Coding agent document interface +interface CodingAgentDocument { + _id: string; + name: string; + description?: string; + supportedModels: string[]; + defaultModel?: string; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +// Model document interface ā€” tracks lifecycle of scanned models +interface ModelDocument { + _id: string; // Compound: "{agentId}:{modelId}" + modelId: string; // Model identifier (e.g. "gpt-4.1") + provider: string; // Provider identifier (e.g. "github-copilot", "anthropic") + agentId: string; // Which coding agent this model was discovered for + firstSeenAt: Date; // First time our scanner discovered this model + lastSeenAt: Date; // Last scan where this model was still present + disappearedAt?: Date; // Set when model no longer returned by provider + providerAvailableFrom?: Date; // Provider-reported availability date + providerEndOfLife?: Date; // Provider-reported planned end-of-life + metadata?: Record; // Additional provider-specific metadata +} + +// MCP server document interface +interface McpServerDocument { + _id: string; // Slug identifier + name: string; + type: "sse" | "http"; + url: string; + headers?: Array<{ name: string; value: string }>; + description?: string; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +// Feature flag document interface +interface FeatureFlagDocument { + key: string; + label: string; + enabled: boolean; + updatedAt: Date; +} + +// Queue message interface +interface QueueMessage { + requestId: string; +} + +// SSE client management for connection pooling +type SSEClient = { + res: Response; + cleanup: () => void; + onActivity?: () => void; +}; +const sseClients: Map> = new Map(); +let sharedSubscriber: InstanceType | null = null; +const subscribedChannels: Set = new Set(); + +// Determine if we should use TLS (Azure Redis uses port 6380 with TLS) +const useRedisTls = redisPassword && redisPort !== 6379; + +// Track if Redis error has been logged to avoid spam +let redisErrorLogged = false; + +function getOrCreateSubscriber(): InstanceType { + if (!sharedSubscriber && redisHost) { + sharedSubscriber = new Redis({ + host: redisHost, + port: redisPort, + password: redisPassword || undefined, + ...(useRedisTls ? { tls: { servername: redisHost } } : {}), + // Limit reconnection attempts to avoid log spam + maxRetriesPerRequest: 3, + retryStrategy: (times: number) => { + if (times > 3) { + return null; // Stop retrying after 3 attempts + } + return Math.min(times * 1000, 3000); + }, + }); + + sharedSubscriber.on("message", (channel: string, message: string) => { + const requestId = channel.replace("logs:", ""); + const clients = sseClients.get(requestId); + if (clients) { + for (const client of clients) { + client.res.write(`data: ${message}\n\n`); + client.onActivity?.(); + + // Check for completion + try { + const logEvent = JSON.parse(message) as LogEvent; + if (logEvent.data?.final === true) { + client.res.write(`event: done\ndata: {"status":"completed"}\n\n`); + client.cleanup(); + } + } catch { + // Ignore parse errors + } + } + } + }); + + sharedSubscriber.on("error", (err: Error) => { + // Only log once to avoid spam + if (!redisErrorLogged) { + console.error("Shared Redis subscriber error:", err.message); + redisErrorLogged = true; + } + }); + + sharedSubscriber.on("connect", () => { + redisErrorLogged = false; // Reset on successful connect + console.log("Redis subscriber connected"); + }); + + console.log(`Initialized shared Redis subscriber (TLS: ${useRedisTls})`); + } + return sharedSubscriber!; +} + +async function subscribeClient(requestId: string, client: SSEClient): Promise { + const channelName = `logs:${requestId}`; + + // Add client to the set + if (!sseClients.has(requestId)) { + sseClients.set(requestId, new Set()); + } + sseClients.get(requestId)!.add(client); + + // Subscribe to channel if not already subscribed + if (!subscribedChannels.has(channelName)) { + const subscriber = getOrCreateSubscriber(); + await subscriber.subscribe(channelName); + subscribedChannels.add(channelName); + console.log(`Subscribed to channel: ${channelName}`); + } +} + +function unsubscribeClient(requestId: string, client: SSEClient): void { + const clients = sseClients.get(requestId); + if (clients) { + clients.delete(client); + + // If no more clients for this request, unsubscribe from channel + if (clients.size === 0) { + sseClients.delete(requestId); + const channelName = `logs:${requestId}`; + if (sharedSubscriber && subscribedChannels.has(channelName)) { + sharedSubscriber.unsubscribe(channelName); + subscribedChannels.delete(channelName); + console.log(`Unsubscribed from channel: ${channelName}`); + } + } + } +} + +async function initializeClients(): Promise { + // Connect to MongoDB + mongoClient = new MongoClient(mongoUri); + await mongoClient.connect(); + db = mongoClient.db(mongoDatabase); + collection = db.collection(mongoCollection); + criteriaCollection = db.collection("criteria"); + promptFeatureCollection = db.collection("prompt-features"); + promptFeatureExtractionCollection = db.collection("prompt-feature-extractions"); + reportCollection = db.collection("reports"); + agentCollection = db.collection("agents"); + modelCollection = db.collection("models"); + mcpServerCollection = db.collection("mcp-servers"); + insightsCollection = db.collection("insights"); + taskPromptCollection = db.collection("task-prompts"); + taskPromptStore = new TaskPromptStore(taskPromptCollection); + featureFlagCollection = db.collection("feature-flags"); + reportTemplateCollection = db.collection("report-templates"); + skillCollection = db.collection("skills"); + skillRevisionCollection = db.collection("skill-revisions"); + skillRevisionStore = new SkillRevisionStore(skillRevisionCollection); + skillResolver = new SkillResolver({ + githubToken: process.env.GITHUB_TOKEN, + }); + + // Note: Collection indexes are managed by db-migrations (see 002-create-indexes.ts). + // Run `pnpm migrate:up` to apply pending migrations. + + const criteriaCount = await criteriaCollection.countDocuments({ deletedAt: { $exists: false } }); + console.log(`Criteria collection has ${criteriaCount} documents`); + + const promptFeatureCount = await promptFeatureCollection.countDocuments({ deletedAt: { $exists: false } }); + console.log(`Prompt features collection has ${promptFeatureCount} documents`); + + const agentCount = await agentCollection.countDocuments({ deletedAt: { $exists: false } }); + console.log(`Agents collection has ${agentCount} documents`); + + const modelCount = await modelCollection.countDocuments(); + console.log(`Models collection has ${modelCount} documents`); + + const mcpServerCount = await mcpServerCollection.countDocuments({ deletedAt: { $exists: false } }); + console.log(`MCP servers collection has ${mcpServerCount} documents`); + + const skillCount = await skillCollection.countDocuments({ deletedAt: { $exists: false } }); + console.log(`Skills collection has ${skillCount} documents`); + + // Seed default feature flags (upsert ā€” won't overwrite existing enabled state) + const defaultFlags: Array<{ key: string; label: string }> = [ + { key: "mcp", label: "MCP Servers" }, + { key: "models", label: "Models" }, + { key: "agents", label: "Agents" }, + { key: "tokens", label: "Tokens" }, + ]; + for (const flag of defaultFlags) { + await featureFlagCollection.updateOne( + { key: flag.key }, + { $setOnInsert: { key: flag.key, label: flag.label, enabled: true, updatedAt: new Date() } }, + { upsert: true } + ); + } + const featureFlagCount = await featureFlagCollection.countDocuments(); + console.log(`Feature flags collection has ${featureFlagCount} documents`); + + const insightCount = await insightsCollection.countDocuments({ deletedAt: { $exists: false } }); + console.log(`Insights collection has ${insightCount} documents`); + + console.log(`Connected to MongoDB: ${mongoUri.replace(/\/\/[^:]+:[^@]+@/, "//***:***@")}`); + + // Initialize queue clients + if (storageConnectionString) { + // Connection string auth (local Azurite or Azure with connection string) + queueClients.set("coder-acp-claude-code", new QueueClient(storageConnectionString, queueWorker1)); + queueClients.set("coder-acp-copilot", new QueueClient(storageConnectionString, queueWorker2)); + reportQueueClient = new QueueClient(storageConnectionString, queueReport); + } else { + // Azure with DefaultAzureCredential + const credential = new DefaultAzureCredential(); + const queueUrl = `https://${storageAccountName}.queue.core.windows.net`; + queueClients.set("coder-acp-claude-code", new QueueClient(`${queueUrl}/${queueWorker1}`, credential)); + queueClients.set("coder-acp-copilot", new QueueClient(`${queueUrl}/${queueWorker2}`, credential)); + reportQueueClient = new QueueClient(`${queueUrl}/${queueReport}`, credential); + } + + // Ensure queues exist (creates them in Azurite on first run) + for (const [name, client] of queueClients) { + await client.createIfNotExists(); + console.log(`Ensured queue exists: ${name}`); + } + await reportQueueClient.createIfNotExists(); + console.log(`Ensured queue exists: ${queueReport}`); + + console.log(`Initialized Queue clients for workers: ${Array.from(queueClients.keys()).join(", ")}, report`); +} + +// Health check endpoint (liveness probe ā€” always returns 200) +app.get("/health", (_req: Request, res: Response) => { + res.json({ status: "healthy", version: GIT_COMMIT }); +}); + +// Readiness probe ā€” returns 200 only when all required DB migrations have +// been applied. Kubernetes will withhold traffic until this returns 200. +app.get("/ready", async (_req: Request, res: Response) => { + try { + const result = await checkMigrations(db); + if (result.ready) { + res.json({ status: "ready", migrations: result }); + } else { + res.status(503).json({ status: "not-ready", migrations: result }); + } + } catch (err: any) { + res.status(503).json({ + status: "not-ready", + error: err.message ?? String(err), + }); + } +}); + +// About endpoint +app.get("/about", (_req: Request, res: Response) => { + res.json({ + name: "Multi-Worker API (MongoDB)", + version: GIT_COMMIT, + buildTime: BUILD_TIME, + description: "API that routes requests to multiple workers via separate queues", + workers: VALID_WORKERS, + }); +}); + +// Version endpoint +app.get("/api/v1/version", (_req: Request, res: Response) => { + res.json({ + commit: GIT_COMMIT, + buildTime: BUILD_TIME, + }); +}); + +// Submit a request +app.post("/api/v1/requests", async (req: Request, res: Response, next: NextFunction) => { + try { + const { scenario: scenarioObj, persona: personaObj, maxIterations, personaInstructions, count = 1, promptFeatureExtractionId, model: requestedModel, mcpServers: mcpServerSlugs, skills: skillSlugs } = req.body; + const worker = req.query.worker as string; + + if (!scenarioObj || typeof scenarioObj !== "object" || !scenarioObj.task || typeof scenarioObj.task !== "string") { + res.status(400).json({ error: "scenario.task is required and must be a string" }); + return; + } + + if (!worker) { + res.status(400).json({ + error: "Worker query parameter is required", + validWorkers: VALID_WORKERS, + example: "/api/v1/requests?worker=worker-1" + }); + return; + } + + if (!VALID_WORKERS.includes(worker as WorkerType)) { + res.status(400).json({ + error: `Invalid worker: ${worker}`, + validWorkers: VALID_WORKERS + }); + return; + } + + // Validate scenario.criteria if provided + if (scenarioObj.criteria !== undefined) { + if (!Array.isArray(scenarioObj.criteria) || !scenarioObj.criteria.every((c: unknown) => typeof c === "string")) { + res.status(400).json({ error: "scenario.criteria must be an array of strings" }); + return; + } + } + + // Validate maxIterations if provided + if (maxIterations !== undefined) { + if (typeof maxIterations !== "number" || maxIterations < 1 || maxIterations > 50) { + res.status(400).json({ error: "maxIterations must be a number between 1 and 50" }); + return; + } + } + + // Validate count if provided + if (typeof count !== "number" || count < 1 || count > 10) { + res.status(400).json({ error: "count must be a number between 1 and 10" }); + return; + } + + const workerType = worker as WorkerType; + + // Resolve model: validate against agent's supportedModels if available + let model: string | undefined = requestedModel; + const agentDoc = await agentCollection.findOne({ _id: workerType, deletedAt: { $exists: false } }); + if (agentDoc && agentDoc.supportedModels.length > 0) { + if (model && !agentDoc.supportedModels.includes(model)) { + res.status(400).json({ + error: `Invalid model "${model}" for agent "${workerType}"`, + supportedModels: agentDoc.supportedModels, + }); + return; + } + if (!model && agentDoc.defaultModel) { + model = agentDoc.defaultModel; + } + } + + // Validate MCP server slugs if provided + let validatedMcpServers: string[] | undefined; + if (mcpServerSlugs !== undefined) { + if (!Array.isArray(mcpServerSlugs) || !mcpServerSlugs.every((s: unknown) => typeof s === "string")) { + res.status(400).json({ error: "mcpServers must be an array of strings (MCP server slugs)" }); + return; + } + if (mcpServerSlugs.length > 0) { + const existingServers = await mcpServerCollection + .find({ _id: { $in: mcpServerSlugs }, deletedAt: { $exists: false } }) + .toArray(); + const existingSlugs = new Set(existingServers.map((s: McpServerDocument) => s._id)); + const missingSlugs = mcpServerSlugs.filter((slug: string) => !existingSlugs.has(slug)); + if (missingSlugs.length > 0) { + res.status(400).json({ error: `MCP server(s) not found: ${missingSlugs.join(", ")}` }); + return; + } + validatedMcpServers = mcpServerSlugs; + } + } + + // Validate and resolve skill slugs if provided + let resolvedSkillRevisions: string[] | undefined; + if (skillSlugs !== undefined) { + if (!Array.isArray(skillSlugs) || !skillSlugs.every((s: unknown) => typeof s === "string")) { + res.status(400).json({ error: "skills must be an array of strings (skill slugs)" }); + return; + } + if (skillSlugs.length > 0) { + // Validate skill slugs exist in our DB + const existingSkills = await skillCollection + .find({ _id: { $in: skillSlugs }, deletedAt: { $exists: false } }) + .toArray(); + const existingSkillSlugs = new Set(existingSkills.map((s: SkillDocument) => s._id)); + const missingSkillSlugs = skillSlugs.filter((slug: string) => !existingSkillSlugs.has(slug)); + if (missingSkillSlugs.length > 0) { + res.status(400).json({ error: `Skill(s) not found: ${missingSkillSlugs.join(", ")}` }); + return; + } + + // Resolve each skill to a SkillRevisionDocument + const uploadArchive = async (archiveName: string, data: Buffer): Promise => { + if (!storageConnectionString && !storageAccountName) { + throw new Error("Blob storage not configured ā€” cannot store skill archives"); + } + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + const credential = new DefaultAzureCredential(); + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + credential + ); + } + const containerClient = blobServiceClient.getContainerClient("skill-archives"); + await containerClient.createIfNotExists(); + const blockBlobClient = containerClient.getBlockBlobClient(archiveName); + await blockBlobClient.upload(data, data.length, { + blobHTTPHeaders: { blobContentType: "application/gzip" }, + }); + return blockBlobClient.url; + }; + + const revisionRefs: string[] = []; + for (const skill of existingSkills) { + try { + const revision = await skillResolver.resolve( + skill.source, + skill.skillName, + skillRevisionStore, + uploadArchive + ); + revisionRefs.push(revision.ref); + } catch (resolveError) { + console.error(`Failed to resolve skill "${skill._id}":`, resolveError); + res.status(422).json({ + error: `Failed to resolve skill "${skill._id}": ${resolveError instanceof Error ? resolveError.message : String(resolveError)}`, + }); + return; + } + } + resolvedSkillRevisions = revisionRefs; + } + } + + // Normalize scenario: ensure criteria is always an array, preserve version + const scenario: RequestDocument['scenario'] = { + task: scenarioObj.task as string, + criteria: Array.isArray(scenarioObj.criteria) ? scenarioObj.criteria as string[] : [], + ...(scenarioObj.version === 'v1' || scenarioObj.version === 'v2' ? { version: scenarioObj.version } : {}), + }; + + const mode = scenario.criteria.length > 0 ? "multi-turn" : "one-shot"; + const queueClient = queueClients.get(workerType)!; + + // Ensure a TaskPrompt entity exists for this task text (idempotent) + const taskPrompt = await taskPromptStore.findOrCreate(scenario.task); + const taskPromptId = taskPrompt._id; + + // Handle multiple runs (count > 1) + if (count > 1) { + const newIds: string[] = []; + const newDocs: RequestDocument[] = []; + const queueMessages: string[] = []; + + for (let i = 0; i < count; i++) { + const requestId = uuidv4(); + newIds.push(requestId); + + const requestDoc: RequestDocument = { + _id: requestId, + scenario, + workerType, + taskPromptId, + status: "pending", + createdAt: new Date(), + ...(model ? { model } : {}), + ...(maxIterations ? { maxIterations } : {}), + ...(personaInstructions ? { personaInstructions } : {}), + ...(personaObj ? { persona: personaObj } : {}), + ...(promptFeatureExtractionId ? { promptFeatureExtractionId } : {}), + ...(validatedMcpServers ? { mcpServers: validatedMcpServers } : {}), + ...(resolvedSkillRevisions ? { skillRevisions: resolvedSkillRevisions } : {}), + }; + newDocs.push(requestDoc); + + const queueMessage: QueueMessage = { requestId }; + const messageContent = Buffer.from(JSON.stringify(queueMessage)).toString("base64"); + queueMessages.push(messageContent); + } + + // Bulk insert all documents + await collection.insertMany(newDocs); + + // Queue all messages + for (const message of queueMessages) { + await queueClient.sendMessage(message); + } + + console.log(`Created ${count} ${mode} requests for ${workerType} and queued for processing`); + + res.status(201).json({ + ids: newIds, + count, + workerType, + taskPromptId, + ...(model ? { model } : {}), + status: "pending", + mode, + message: `${count} requests submitted successfully`, + scenario, + ...(maxIterations ? { maxIterations } : {}), + }); + return; + } + + // Single run (count === 1) - original behavior + const requestId = uuidv4(); + + // Create request document + const requestDoc: RequestDocument = { + _id: requestId, + scenario, + workerType, + taskPromptId, + status: "pending", + createdAt: new Date(), + ...(model ? { model } : {}), + ...(maxIterations ? { maxIterations } : {}), + ...(personaInstructions ? { personaInstructions } : {}), + ...(personaObj ? { persona: personaObj } : {}), + ...(promptFeatureExtractionId ? { promptFeatureExtractionId } : {}), + ...(validatedMcpServers ? { mcpServers: validatedMcpServers } : {}), + ...(resolvedSkillRevisions ? { skillRevisions: resolvedSkillRevisions } : {}), + }; + + // Store in MongoDB + await collection.insertOne(requestDoc); + + // Queue the request for the appropriate worker + const queueMessage: QueueMessage = { requestId }; + const messageContent = Buffer.from(JSON.stringify(queueMessage)).toString("base64"); + await queueClient.sendMessage(messageContent); + + console.log(`Created ${mode} request ${requestId} for ${workerType} and queued for processing`); + + res.status(201).json({ + id: requestId, + workerType, + ...(model ? { model } : {}), + status: requestDoc.status, + mode, + message: "Request submitted successfully", + scenario, + ...(maxIterations ? { maxIterations } : {}), + }); + } catch (error) { + next(error); + } +}); + +// Get request status +app.get("/api/v1/requests/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const resource = await collection.findOne({ _id: id }); + + if (!resource) { + res.status(404).json({ error: "Request not found" }); + return; + } + + // Map _id back to id for API response + res.json({ ...resource, id: resource._id }); + } catch (error) { + next(error); + } +}); + +// Stream logs for a request via SSE (with connection pooling) +app.get("/api/v1/requests/:id/logs", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const fromStart = req.query.fromStart === "true"; + + // Verify request exists + const resource = await collection.findOne({ _id: id }); + + if (!resource) { + res.status(404).json({ error: "Request not found" }); + return; + } + + // Set SSE headers + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + + // If fromStart=true, send existing logs from MongoDB first + if (fromStart && resource.logs && resource.logs.length > 0) { + for (const log of resource.logs) { + res.write(`data: ${JSON.stringify(log)}\n\n`); + } + } + + // If request already completed/failed/exhausted, send final event and close + if (resource.status === "completed" || resource.status === "failed" || resource.status === "exhausted") { + // For completed multi-turn requests, send turns summary + if (resource.turns && resource.turns.length > 0) { + res.write(`data: ${JSON.stringify({ type: "turns_summary", turns: resource.turns.length, passed: resource.status === "completed" })}\n\n`); + } + res.write(`event: done\ndata: {"status":"${resource.status}"}\n\n`); + res.end(); + return; + } + + // Use connection pooling - single shared subscriber + let cleaned = false; + let changeStream: ReturnType | null = null; + let redisSubscribed = false; + + // Inactivity timeout ā€” resets every time a log message is forwarded + const INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes of silence + let inactivityTimer: ReturnType; + + const resetInactivityTimer = () => { + clearTimeout(inactivityTimer); + inactivityTimer = setTimeout(() => { + res.write(`event: timeout\ndata: {"message":"Stream timeout after 5 minutes of inactivity"}\n\n`); + client.cleanup(); + }, INACTIVITY_TIMEOUT_MS); + }; + + // SSE heartbeat every 30s to prevent proxy/LB disconnects + const heartbeat = setInterval(() => { + if (!cleaned) { + res.write(`:\n\n`); // SSE comment ā€” ignored by EventSource clients + } + }, 30_000); + + const client: SSEClient = { + res, + onActivity: resetInactivityTimer, + cleanup: () => { + if (!cleaned) { + cleaned = true; + clearTimeout(inactivityTimer); + clearInterval(heartbeat); + if (changeStream) { + changeStream.close().catch(err => console.error("Error closing change stream:", err)); + } + if (redisSubscribed) { + unsubscribeClient(id, client); + } + res.end(); + } + }, + }; + + // Start the inactivity timer + resetInactivityTimer(); + + // Try Redis subscription if configured (non-blocking - fallback to Change Streams if unavailable) + if (redisHost) { + try { + await subscribeClient(id, client); + redisSubscribed = true; + } catch (err) { + console.error(`Redis subscription failed for ${id}, using Change Streams only:`, err); + } + } + + // Use MongoDB Change Streams as fallback (or primary if Redis unavailable) + try { + changeStream = collection.watch( + [{ $match: { "documentKey._id": id, operationType: "update" } }], + { fullDocument: "updateLookup" } + ); + + changeStream.on("change", (change) => { + if (change.operationType === "update" && change.fullDocument) { + const doc = change.fullDocument; + if (doc.status === "completed" || doc.status === "failed" || doc.status === "exhausted") { + res.write(`event: done\ndata: {"status":"${doc.status}"}\n\n`); + client.cleanup(); + } + } + }); + + changeStream.on("error", (err) => { + console.error(`Change stream error for ${id}:`, err); + // Change stream failed, don't retry - the 5 minute timeout will handle it + }); + } catch (err) { + console.error(`Failed to create change stream for ${id}:`, err); + // Change streams not supported (e.g., some CosmosDB configurations) + } + + // Cleanup on client disconnect + req.on("close", () => client.cleanup()); + + } catch (error) { + next(error); + } +}); + +// List all requests (excludes soft-deleted by default) +app.get("/api/v1/requests", async (req: Request, res: Response, next: NextFunction) => { + try { + const workerFilter = req.query.worker as string; + const taskPromptIdFilter = req.query.taskPromptId as string; + const criteriaFilter = req.query.criteria as string; + const includeDeleted = req.query.includeDeleted === "true"; + + const filter: Record = {}; + if (workerFilter && VALID_WORKERS.includes(workerFilter as WorkerType)) { + filter.workerType = workerFilter; + } + if (taskPromptIdFilter) { + filter.taskPromptId = taskPromptIdFilter; + } + if (!includeDeleted) { + filter.deletedAt = { $exists: false }; + } + + // Filter by MDP criteria state vector (e.g. "has_azure:0|has_cloud:1") + // Matches runs whose LAST turn contains criteria results matching every + // criterion in the state vector. + if (criteriaFilter) { + const criteriaStates = parseStateKey(criteriaFilter); + if (criteriaStates.length > 0) { + filter.$and = criteriaStates.map((cs) => ({ + "turns": { + $elemMatch: { + "criteriaResults": { + $elemMatch: { + criterionId: cs.id, + passed: cs.passed, + }, + }, + }, + }, + })); + } + } + + const resources = await collection + .find(filter) + .sort({ createdAt: -1 }) + .toArray(); + + res.json(resources.map(r => ({ ...r, id: r._id }))); + } catch (error) { + next(error); + } +}); + +// Analysis endpoint - compute pass@k, success@T, and iteration stats +app.get("/api/v1/analysis", async (req: Request, res: Response, next: NextFunction) => { + try { + // Parse k values from query string (default: 1,2,5) + const kParam = (req.query.k as string) || "1,2,5"; + const kValues = kParam.split(",").map(v => parseInt(v.trim(), 10)).filter(v => !isNaN(v) && v > 0); + + // Parse criteria filter from query string (comma-separated criterion IDs) + const criteriaParam = req.query.criteria as string | undefined; + const selectedCriteria = criteriaParam + ? criteriaParam.split(",").map(c => c.trim()).filter(Boolean) + : undefined; + + // Fetch all completed/failed runs (exclude pending/processing, exclude deleted) + const runs = await collection + .find({ + status: { $in: ["completed", "failed", "exhausted"] }, + deletedAt: { $exists: false }, + }) + .project({ + _id: 1, + scenario: 1, + workerType: 1, + status: 1, + turns: 1, + }) + .toArray(); + + // Transform to AnalyzableRun format + const analyzableRuns: AnalyzableRun[] = runs.map(r => ({ + scenario: r.scenario, + workerType: r.workerType, + status: r.status, + turns: r.turns, + })); + + const analysis: AnalysisResponse = computeAnalysis(analyzableRuns, kValues, selectedCriteria); + res.json(analysis); + } catch (error) { + next(error); + } +}); + +// Bulk re-submit requests (create new runs from existing ones) +app.post("/api/v1/requests/bulk-resubmit", async (req: Request, res: Response, next: NextFunction) => { + try { + const { ids, count = 1, overrides } = req.body as { + ids?: string[]; + count?: number; + overrides?: { + workerType?: string; + model?: string | null; + maxIterations?: number | null; + mcpServers?: string[] | null; + skillRevisions?: string[] | null; + }; + }; + + if (!ids || !Array.isArray(ids) || ids.length === 0) { + res.status(400).json({ error: "Request body must include 'ids' array" }); + return; + } + + if (typeof count !== "number" || count < 1 || count > 10) { + res.status(400).json({ error: "count must be a number between 1 and 10" }); + return; + } + + // Validate overrides if provided + if (overrides?.workerType && !VALID_WORKERS.includes(overrides.workerType as WorkerType)) { + res.status(400).json({ error: `Invalid workerType override: ${overrides.workerType}` }); + return; + } + + // Fetch original runs + const originalRuns = await collection.find( + { _id: { $in: ids }, deletedAt: { $exists: false } } + ).toArray(); + + const foundIds = new Set(originalRuns.map(r => r._id)); + const notFound = ids.filter(id => !foundIds.has(id)); + + const newIds: string[] = []; + const newDocs: RequestDocument[] = []; + const queueMessages: Array<{ workerType: WorkerType; message: string }> = []; + + for (const original of originalRuns) { + for (let i = 0; i < count; i++) { + const requestId = uuidv4(); + newIds.push(requestId); + + // Resolve effective values: override > original > omit + const effectiveWorkerType = (overrides?.workerType ?? original.workerType) as WorkerType; + const effectiveModel = overrides?.model !== undefined ? overrides.model : original.model; + const effectiveMaxIterations = overrides?.maxIterations !== undefined ? overrides.maxIterations : original.maxIterations; + const effectiveMcpServers = overrides?.mcpServers !== undefined ? overrides.mcpServers : original.mcpServers; + const effectiveSkillRevisions = overrides?.skillRevisions !== undefined ? overrides.skillRevisions : original.skillRevisions; + + const newDoc: RequestDocument = { + _id: requestId, + scenario: original.scenario, + workerType: effectiveWorkerType, + status: "pending", + createdAt: new Date(), + ...(effectiveMaxIterations ? { maxIterations: effectiveMaxIterations } : {}), + ...(original.personaInstructions ? { personaInstructions: original.personaInstructions } : {}), + ...(original.persona ? { persona: original.persona } : {}), + ...(effectiveModel ? { model: effectiveModel } : {}), + ...(effectiveMcpServers && effectiveMcpServers.length > 0 ? { mcpServers: effectiveMcpServers } : {}), + ...(effectiveSkillRevisions && effectiveSkillRevisions.length > 0 ? { skillRevisions: effectiveSkillRevisions } : {}), + }; + + newDocs.push(newDoc); + + const queueMessage: QueueMessage = { requestId }; + const messageContent = Buffer.from(JSON.stringify(queueMessage)).toString("base64"); + queueMessages.push({ workerType: effectiveWorkerType as WorkerType, message: messageContent }); + } + } + + // Insert all new documents + if (newDocs.length > 0) { + await collection.insertMany(newDocs); + } + + // Queue all messages + for (const { workerType, message } of queueMessages) { + const queueClient = queueClients.get(workerType); + if (queueClient) { + await queueClient.sendMessage(message); + } + } + + console.log(`Bulk re-submitted ${newIds.length} runs from ${originalRuns.length} originals (count=${count})`); + + res.status(201).json({ + submitted: newIds.length, + failed: notFound, + newIds, + }); + } catch (error) { + next(error); + } +}); + +// Bulk soft-delete requests +app.delete("/api/v1/requests/bulk", async (req: Request, res: Response, next: NextFunction) => { + try { + const { ids } = req.body as { ids?: string[] }; + + if (!ids || !Array.isArray(ids) || ids.length === 0) { + res.status(400).json({ error: "Request body must include 'ids' array" }); + return; + } + + // Find which IDs exist and are not already deleted + const existingDocs = await collection.find( + { _id: { $in: ids }, deletedAt: { $exists: false } }, + { projection: { _id: 1 } } + ).toArray(); + const existingIds = new Set(existingDocs.map(d => d._id)); + + // Soft-delete all matching documents + const result = await collection.updateMany( + { _id: { $in: ids }, deletedAt: { $exists: false } }, + { $set: { deletedAt: new Date() } } + ); + + // Determine which IDs were not found or already deleted + const notFound = ids.filter(id => !existingIds.has(id)); + + res.json({ + deleted: result.modifiedCount, + notFound, + }); + } catch (error) { + next(error); + } +}); + +// Soft-delete a request +app.delete("/api/v1/requests/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const result = await collection.updateOne( + { _id: id, deletedAt: { $exists: false } }, + { $set: { deletedAt: new Date() } } + ); + + if (result.matchedCount === 0) { + const exists = await collection.findOne({ _id: id }); + if (!exists) { + res.status(404).json({ error: "Request not found" }); + } else { + res.status(410).json({ error: "Request already deleted" }); + } + return; + } + + res.json({ id, deleted: true }); + } catch (error) { + next(error); + } +}); + +// Download a snapshot for a specific iteration +app.get("/api/v1/requests/:id/snapshots/:iteration", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id, iteration } = req.params; + const iterNum = parseInt(iteration, 10); + if (isNaN(iterNum) || iterNum < 1) { + res.status(400).json({ error: "Invalid iteration number" }); + return; + } + + const resource = await collection.findOne({ _id: id }); + if (!resource) { + res.status(404).json({ error: "Request not found" }); + return; + } + + const turn = resource.turns?.find(t => t.iteration === iterNum); + if (!turn?.snapshotUrl) { + res.status(404).json({ error: `No snapshot for iteration ${iterNum}` }); + return; + } + + // Connect to blob storage + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + new DefaultAzureCredential() + ); + } + + // Parse blob name from snapshot URL + const snapshotUrl = new URL(turn.snapshotUrl); + const containerPrefix = "/snapshots/"; + const containerIndex = snapshotUrl.pathname.indexOf(containerPrefix); + if (containerIndex === -1) { + res.status(500).json({ error: "Invalid snapshot URL format" }); + return; + } + const blobName = snapshotUrl.pathname.substring(containerIndex + containerPrefix.length); + const containerClient = blobServiceClient.getContainerClient("snapshots"); + const blobClient = containerClient.getBlockBlobClient(blobName); + + const downloadResponse = await blobClient.download(); + if (!downloadResponse.readableStreamBody) { + res.status(500).json({ error: "Failed to download snapshot" }); + return; + } + + res.setHeader("Content-Type", "application/gzip"); + res.setHeader("Content-Disposition", `attachment; filename="${id}-iteration-${iterNum}.tar.gz"`); + if (downloadResponse.contentLength) { + res.setHeader("Content-Length", downloadResponse.contentLength); + } + + downloadResponse.readableStreamBody.pipe(res); + } catch (error) { + if (error instanceof RestError && (error.statusCode === 404 || error.code === "ContainerNotFound" || error.code === "BlobNotFound")) { + res.status(404).json({ error: "Snapshot not found ā€” the blob may have been deleted or is no longer available" }); + return; + } + next(error); + } +}); + +// Download a full run archive (run.yaml + iteration snapshots as .tar.gz entries) +app.get("/api/v1/requests/:id/archive", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const resource = await collection.findOne({ _id: id }); + if (!resource) { + res.status(404).json({ error: "Request not found" }); + return; + } + + if (!resource.turns || resource.turns.length === 0) { + res.status(404).json({ error: "No iterations found for this run" }); + return; + } + + // Connect to blob storage + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + new DefaultAzureCredential() + ); + } + const containerClient = blobServiceClient.getContainerClient("snapshots"); + + // Set response headers before streaming + res.setHeader("Content-Type", "application/gzip"); + res.setHeader("Content-Disposition", `attachment; filename="${id}.tar.gz"`); + + // Create streaming tar+gzip pipeline ā†’ response + const pack = tarPack(); + const gzip = createGzip(); + pack.pipe(gzip).pipe(res); + + // Entry 1: run.yaml ā€” the full run document + const yamlContent = yamlStringify(resource, { lineWidth: 120 }); + const yamlBuf = Buffer.from(yamlContent, "utf-8"); + pack.entry({ name: `${id}/run.yaml`, size: yamlBuf.length }, yamlBuf); + + // Entries 2..N: iteration snapshots as-is (.tar.gz blobs) + for (const turn of resource.turns) { + if (!turn.snapshotUrl) continue; + + try { + const snapshotUrl = new URL(turn.snapshotUrl); + const containerPrefix = "/snapshots/"; + const containerIndex = snapshotUrl.pathname.indexOf(containerPrefix); + if (containerIndex === -1) continue; + + const blobName = snapshotUrl.pathname.substring(containerIndex + containerPrefix.length); + const blobClient = containerClient.getBlockBlobClient(blobName); + const downloadResponse = await blobClient.download(); + + if (!downloadResponse.readableStreamBody || !downloadResponse.contentLength) continue; + + // Add the snapshot blob as a tar entry, streaming directly from blob storage + const entry = pack.entry({ + name: `${id}/iteration-${turn.iteration}.tar.gz`, + size: downloadResponse.contentLength, + }); + await pipeline(downloadResponse.readableStreamBody, entry); + } catch (blobError) { + // Skip snapshots that fail to download (e.g. deleted blobs) + if (blobError instanceof RestError && (blobError.statusCode === 404 || blobError.code === "ContainerNotFound" || blobError.code === "BlobNotFound")) { + continue; + } + throw blobError; + } + } + + // Finalize the tar archive + pack.finalize(); + } catch (error) { + if (!res.headersSent) { + if (error instanceof RestError && (error.statusCode === 404 || error.code === "ContainerNotFound" || error.code === "BlobNotFound")) { + res.status(404).json({ error: "Snapshot not found ā€” the blob may have been deleted or is no longer available" }); + return; + } + next(error); + } else { + // Headers already sent ā€” destroy the response to signal an error to the client + res.destroy(); + } + } +}); + +// --- Runs upload (import downloaded archives) --- + +// Download a HAR (HTTP Archive) file for a specific request or turn +// For one-shot runs: GET /api/v1/requests/:id/har +// For multi-turn runs: GET /api/v1/requests/:id/har?iteration=N +app.get("/api/v1/requests/:id/har", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const iterationParam = req.query.iteration as string | undefined; + + const resource = await collection.findOne({ _id: id }); + if (!resource) { + res.status(404).json({ error: "Request not found" }); + return; + } + + // Determine the harUrl ā€” from a specific turn or from the top-level document + let harUrl: string | undefined; + let label: string; + + if (iterationParam) { + const iterNum = parseInt(iterationParam, 10); + if (isNaN(iterNum) || iterNum < 1) { + res.status(400).json({ error: "Invalid iteration number" }); + return; + } + const turn = resource.turns?.find((t: { iteration: number }) => t.iteration === iterNum); + harUrl = turn?.harUrl; + label = `${id}-iteration-${iterNum}`; + } else { + // One-shot: harUrl on document root; multi-turn fallback: last turn + harUrl = resource.harUrl || resource.turns?.[resource.turns.length - 1]?.harUrl; + label = id; + } + + if (!harUrl) { + res.status(404).json({ error: "No HAR capture available" }); + return; + } + + // Connect to blob storage and proxy the HAR file + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + new DefaultAzureCredential() + ); + } + + const parsedUrl = new URL(harUrl); + const containerPrefix = "/snapshots/"; + const containerIndex = parsedUrl.pathname.indexOf(containerPrefix); + if (containerIndex === -1) { + res.status(500).json({ error: "Invalid HAR URL format" }); + return; + } + const blobName = parsedUrl.pathname.substring(containerIndex + containerPrefix.length); + const containerClient = blobServiceClient.getContainerClient("snapshots"); + const blobClient = containerClient.getBlockBlobClient(blobName); + + const downloadResponse = await blobClient.download(); + if (!downloadResponse.readableStreamBody) { + res.status(500).json({ error: "Failed to download HAR file" }); + return; + } + + res.setHeader("Content-Type", "application/json"); + res.setHeader("Content-Disposition", `attachment; filename="${label}.har"`); + if (downloadResponse.contentLength) { + res.setHeader("Content-Length", downloadResponse.contentLength); + } + + downloadResponse.readableStreamBody.pipe(res); + } catch (error) { + if (error instanceof RestError && (error.statusCode === 404 || error.code === "ContainerNotFound" || error.code === "BlobNotFound")) { + res.status(404).json({ error: "HAR file not found ā€” the blob may have been deleted or is no longer available" }); + return; + } + next(error); + } +}); + +// Download a session recording video for a specific request or turn +// For one-shot runs: GET /api/v1/requests/:id/video?index=0 +// For multi-turn runs: GET /api/v1/requests/:id/video?iteration=N&index=0 +app.get("/api/v1/requests/:id/video", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const iterationParam = req.query.iteration as string | undefined; + const indexParam = req.query.index as string | undefined; + const videoIndex = indexParam ? parseInt(indexParam, 10) : 0; + + if (isNaN(videoIndex) || videoIndex < 0) { + res.status(400).json({ error: "Invalid video index" }); + return; + } + + const resource = await collection.findOne({ _id: id }); + if (!resource) { + res.status(404).json({ error: "Request not found" }); + return; + } + + // Determine the videoUrls array ā€” from a specific turn or from the top-level document + let videoUrls: string[] | undefined; + let label: string; + + if (iterationParam) { + const iterNum = parseInt(iterationParam, 10); + if (isNaN(iterNum) || iterNum < 1) { + res.status(400).json({ error: "Invalid iteration number" }); + return; + } + const turn = resource.turns?.find(t => t.iteration === iterNum); + videoUrls = turn?.videoUrls; + label = `${id}-iteration-${iterNum}-video-${videoIndex}`; + } else { + // One-shot: videoUrls on document root; multi-turn fallback: last turn + videoUrls = resource.videoUrls ?? resource.turns?.[resource.turns.length - 1]?.videoUrls; + label = `${id}-video-${videoIndex}`; + } + + if (!videoUrls || videoUrls.length === 0) { + res.status(404).json({ error: "No video recordings available" }); + return; + } + + if (videoIndex >= videoUrls.length) { + res.status(404).json({ error: `Video index ${videoIndex} not found (${videoUrls.length} available)` }); + return; + } + + const videoUrl = videoUrls[videoIndex]; + + // Connect to blob storage and proxy the video file + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + new DefaultAzureCredential() + ); + } + + const parsedUrl = new URL(videoUrl); + const containerPrefix = "/snapshots/"; + const containerIndex = parsedUrl.pathname.indexOf(containerPrefix); + if (containerIndex === -1) { + res.status(500).json({ error: "Invalid video URL format" }); + return; + } + const blobName = parsedUrl.pathname.substring(containerIndex + containerPrefix.length); + const containerClient = blobServiceClient.getContainerClient("snapshots"); + const blobClient = containerClient.getBlockBlobClient(blobName); + + // Get blob properties for content length + const properties = await blobClient.getProperties(); + const totalSize = properties.contentLength ?? 0; + + // Support HTTP Range requests for video seeking + const rangeHeader = req.headers.range; + if (rangeHeader && totalSize > 0) { + const match = rangeHeader.match(/bytes=(\d+)-(\d*)/); + if (match) { + const start = parseInt(match[1], 10); + const end = match[2] ? parseInt(match[2], 10) : totalSize - 1; + const chunkSize = end - start + 1; + + const downloadResponse = await blobClient.download(start, chunkSize); + if (!downloadResponse.readableStreamBody) { + res.status(500).json({ error: "Failed to download video file" }); + return; + } + + res.status(206); + res.setHeader("Content-Type", "video/webm"); + res.setHeader("Content-Range", `bytes ${start}-${end}/${totalSize}`); + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Length", chunkSize); + downloadResponse.readableStreamBody.pipe(res); + return; + } + } + + const downloadResponse = await blobClient.download(); + if (!downloadResponse.readableStreamBody) { + res.status(500).json({ error: "Failed to download video file" }); + return; + } + + res.setHeader("Content-Type", "video/webm"); + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Disposition", `inline; filename="${label}.webm"`); + if (totalSize > 0) { + res.setHeader("Content-Length", totalSize); + } + + downloadResponse.readableStreamBody.pipe(res); + } catch (error) { + if (error instanceof RestError && (error.statusCode === 404 || error.code === "ContainerNotFound" || error.code === "BlobNotFound")) { + res.status(404).json({ error: "Video file not found ā€” the blob may have been deleted or is no longer available" }); + return; + } + next(error); + } +}); + +// POST /api/v1/runs/upload ā€” Upload a run archive (tar.gz) to import a previously downloaded run +app.post("/api/v1/runs/upload", upload.single("archive"), async (req: Request, res: Response, next: NextFunction) => { + const tempDir = mkdtempSync(join(tmpdir(), "run-upload-")); + let uploadedFilePath: string | undefined; + + try { + // Validate file was uploaded + if (!req.file) { + res.status(400).json({ error: "No archive file uploaded. Use 'archive' field for the tar.gz file." }); + return; + } + uploadedFilePath = req.file.path; + + // Extract archive to temp directory + const extractDir = join(tempDir, "extracted"); + execSync(`mkdir -p "${extractDir}" && tar xzf "${uploadedFilePath}" -C "${extractDir}"`, { stdio: "pipe" }); + + // Find the run directory (archive contains / folder with run.yaml) + const entries = readdirSync(extractDir); + if (entries.length === 0) { + res.status(400).json({ error: "Archive is empty" }); + return; + } + + // Determine run directory - could be at root or in a subdirectory + let runDir = extractDir; + let runYamlPath = join(extractDir, "run.yaml"); + + if (!existsSync(runYamlPath)) { + // run.yaml might be inside a subdirectory (e.g., /run.yaml) + // Check each top-level entry for run.yaml + for (const entry of entries) { + const subDir = join(extractDir, entry); + const subRunYaml = join(subDir, "run.yaml"); + try { + const stat = statSync(subDir); + if (stat.isDirectory() && existsSync(subRunYaml)) { + runDir = subDir; + runYamlPath = subRunYaml; + break; + } + } catch { + // Entry might not be a directory, skip + } + } + } + + if (!existsSync(runYamlPath)) { + res.status(400).json({ error: "Invalid archive: run.yaml not found" }); + return; + } + + // Parse run.yaml + const runYamlContent = await readFile(runYamlPath, "utf-8"); + let runDoc: RequestDocument; + try { + runDoc = yamlParse(runYamlContent) as RequestDocument; + } catch (parseErr) { + res.status(400).json({ error: `Failed to parse run.yaml: ${parseErr}` }); + return; + } + + // Validate required fields + if (!runDoc._id) { + res.status(400).json({ error: "Invalid run.yaml: missing _id field" }); + return; + } + if (!runDoc.scenario) { + res.status(400).json({ error: "Invalid run.yaml: missing scenario field" }); + return; + } + if (!runDoc.workerType) { + res.status(400).json({ error: "Invalid run.yaml: missing workerType field" }); + return; + } + if (!runDoc.status) { + res.status(400).json({ error: "Invalid run.yaml: missing status field" }); + return; + } + + // Validate status is terminal (cannot import in-flight runs) + const terminalStatuses = ["completed", "failed", "exhausted"]; + if (!terminalStatuses.includes(runDoc.status)) { + res.status(400).json({ + error: `Cannot upload in-flight run (status: ${runDoc.status}). Only terminal runs can be uploaded.`, + }); + return; + } + + // Check if run already exists + const existingRun = await collection.findOne({ _id: runDoc._id }); + if (existingRun) { + res.status(409).json({ + error: `Run with ID '${runDoc._id}' already exists`, + existingStatus: existingRun.status, + }); + return; + } + + // Upload iteration snapshots to blob storage and update snapshotUrls + const turns = runDoc.turns || []; + const iterationDirs = readdirSync(runDir).filter(name => name.startsWith("iteration-")); + + // Connect to blob storage + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + new DefaultAzureCredential() + ); + } + const containerClient = blobServiceClient.getContainerClient("snapshots"); + await containerClient.createIfNotExists(); + + for (const iterDir of iterationDirs) { + const iterMatch = iterDir.match(/^iteration-(\d+)$/); + if (!iterMatch) continue; + + const iterNum = parseInt(iterMatch[1], 10); + const iterPath = join(runDir, iterDir); + + // Create tar.gz from iteration directory + const iterArchive = join(tempDir, `iter-${iterNum}.tar.gz`); + execSync(`tar czf "${iterArchive}" -C "${iterPath}" .`, { stdio: "pipe" }); + + // Upload to blob storage + const blobName = `${runDoc._id}/iteration-${iterNum}/workspace.tar.gz`; + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + await blockBlobClient.uploadFile(iterArchive, { + blobHTTPHeaders: { blobContentType: "application/gzip" }, + tags: { requestId: runDoc._id, iteration: String(iterNum) }, + }); + + // Update turn's snapshotUrl + const turn = turns.find(t => t.iteration === iterNum); + if (turn) { + turn.snapshotUrl = blockBlobClient.url; + } + } + + // Prepare document for insertion + const docToInsert: RequestDocument = { + _id: runDoc._id, + scenario: runDoc.scenario, + workerType: runDoc.workerType as WorkerType, + status: runDoc.status, + createdAt: runDoc.createdAt ? new Date(runDoc.createdAt) : new Date(), + updatedAt: runDoc.updatedAt ? new Date(runDoc.updatedAt) : undefined, + turns: turns.map(t => ({ + ...t, + timestamp: t.timestamp ? new Date(t.timestamp) : new Date(), + })), + ...(runDoc.result ? { result: runDoc.result } : {}), + ...(runDoc.error ? { error: runDoc.error } : {}), + ...(runDoc.maxIterations ? { maxIterations: runDoc.maxIterations } : {}), + ...(runDoc.personaInstructions ? { personaInstructions: runDoc.personaInstructions } : {}), + ...(runDoc.persona ? { persona: runDoc.persona } : {}), + ...(runDoc.logs && Array.isArray(runDoc.logs) ? { logs: runDoc.logs } : {}), + }; + + // Insert into MongoDB + await collection.insertOne(docToInsert); + + console.log(`Uploaded run ${runDoc._id} with ${iterationDirs.length} iterations`); + + res.status(201).json({ + id: runDoc._id, + status: runDoc.status, + iterations: iterationDirs.length, + message: "Run uploaded successfully", + }); + + } catch (error) { + next(error); + } finally { + // Cleanup temp files + rmSync(tempDir, { recursive: true, force: true }); + if (uploadedFilePath && existsSync(uploadedFilePath)) { + rmSync(uploadedFilePath, { force: true }); + } + } +}); + +// --- Criteria seed & CRUD --- + +// POST /api/v1/criteria/generate-prompt ā€” AI-generate a criteria prompt from a behavior description +app.post("/api/v1/criteria/generate-prompt", async (req: Request, res: Response, next: NextFunction) => { + try { + const { behavior, currentId } = req.body; + if (!behavior || typeof behavior !== "string" || !behavior.trim()) { + return res.status(400).json({ error: "Body must contain a non-empty 'behavior' string" }); + } + + if (!isLlmAvailable()) { + return res.status(503).json({ error: "LLM not configured: register a github-models token or set GITHUB_MODELS_API_KEY" }); + } + + // Fetch existing criteria to give the LLM context for parent/children suggestions + const allCriteria = await criteriaCollection + .find({ deletedAt: { $exists: false } }) + .project({ id: 1, prompt: 1, dependsOn: 1, _id: 0 }) + .toArray(); + + // Exclude the current criterion when editing (to avoid self-reference) + const existingCriteria = currentId + ? allCriteria.filter((c: any) => c.id !== currentId) + : allCriteria; + + const result = await generateCriteriaPrompt( + behavior.trim(), + existingCriteria as { id: string; prompt: string; dependsOn?: string[] }[], + ); + console.log("[generate-prompt] LLM result:", JSON.stringify(result)); + res.json(result); + } catch (err) { + if (err instanceof Error && err.message.includes("not configured")) { + return res.status(503).json({ error: err.message }); + } + next(err); + } +}); + +// POST /api/v1/criteria/seed ā€” bulk seed criteria from a JSON array +// Body: { criteria: [{ id, prompt, dependsOn? }] } +app.post("/api/v1/criteria/seed", async (req: Request, res: Response, next: NextFunction) => { + try { + const { criteria } = req.body; + if (!Array.isArray(criteria)) { + return res.status(400).json({ error: "Body must contain a 'criteria' array" }); + } + + let seeded = 0; + const errors: string[] = []; + + for (const config of criteria) { + if (!config.id || !config.prompt) { + errors.push(`Skipping entry without id or prompt`); + continue; + } + try { + await criteriaCollection.updateOne( + { id: config.id.trim() }, + { + $setOnInsert: { + id: config.id.trim(), + prompt: config.prompt.trim(), + dependsOn: Array.isArray(config.dependsOn) ? config.dependsOn.map((d: any) => String(d).trim()) : [], + createdAt: new Date(), + }, + }, + { upsert: true } + ); + seeded++; + } catch (err) { + errors.push(`Failed to seed ${config.id}: ${err}`); + } + } + + res.json({ seeded, errors }); + } catch (err) { + next(err); + } +}); + +// List all criteria (with optional search) +app.get("/api/v1/criteria", async (_req: Request, res: Response, next: NextFunction) => { + try { + const q = _req.query.q as string | undefined; + const filter: Record = { deletedAt: { $exists: false } }; + if (q) { + filter.$or = [ + { id: { $regex: q, $options: "i" } }, + { prompt: { $regex: q, $options: "i" } }, + ]; + } + const criteria = await criteriaCollection.find(filter).toArray(); + // Sort in JS (CosmosDB doesn't support sort on non-_id fields without explicit indexing policy) + criteria.sort((a, b) => a.id.localeCompare(b.id)); + res.json(criteria); + } catch (error) { + next(error); + } +}); + +// MDP state-transition graph across all runs +app.get("/api/v1/criteria/mdp", async (req: Request, res: Response, next: NextFunction) => { + try { + // Parse optional criteria projection + const criteriaParam = req.query.criteria as string | undefined; + const selectedCriteria = criteriaParam + ? criteriaParam.split(",").map(c => c.trim()).filter(Boolean) + : undefined; + + // Parse optional prompt-feature filter + const featuresParam = req.query.features as string | undefined; + const selectedFeatures = featuresParam + ? featuresParam.split(",").map(f => f.trim()).filter(Boolean) + : undefined; + + // Parse optional ?since= for incremental polling + const sinceParam = req.query.since as string | undefined; + const sinceDate = sinceParam ? new Date(sinceParam) : undefined; + + // Build MongoDB filter + const mdpFilter: Record = { + status: { $in: ["completed", "failed", "exhausted"] }, + deletedAt: { $exists: false }, + }; + + // Optional worker/taskPromptId filters + if (req.query.worker) { + mdpFilter.workerType = req.query.worker; + } + if (req.query.taskPromptId) { + mdpFilter.taskPromptId = req.query.taskPromptId; + } + + // Incremental: only runs updated after ?since= + if (sinceDate && !isNaN(sinceDate.getTime())) { + mdpFilter.updatedAt = { $gt: sinceDate }; + } + + const runs = await collection + .find(mdpFilter) + .project({ + _id: 1, + scenario: 1, + status: 1, + turns: 1, + updatedAt: 1, + taskPromptId: 1, + }) + .toArray(); + + // Batch-lookup task prompts for their features + const taskPromptIds = [...new Set(runs.map(r => r.taskPromptId).filter(Boolean))] as string[]; + const taskPromptFeatures = new Map>(); + if (taskPromptIds.length > 0) { + const taskPrompts = await taskPromptCollection + .find({ _id: { $in: taskPromptIds } }) + .project({ _id: 1, features: 1 }) + .toArray(); + for (const tp of taskPrompts) { + if (tp.features && tp.features.length > 0) { + taskPromptFeatures.set(tp._id, tp.features); + } + } + } + + const mdpRuns: MdpAnalyzableRun[] = runs.map(r => ({ + scenario: r.scenario, + status: r.status, + turns: r.turns, + updatedAt: r.updatedAt, + promptFeatures: r.taskPromptId ? taskPromptFeatures.get(r.taskPromptId) : undefined, + })); + + const mdpResult = computeMdp(mdpRuns, selectedCriteria, selectedFeatures); + res.json(mdpResult); + } catch (error) { + next(error); + } +}); + +// Get criteria DAG graph (nodes + edges) +app.get("/api/v1/criteria/graph", async (_req: Request, res: Response, next: NextFunction) => { + try { + const all = await criteriaCollection.find({ deletedAt: { $exists: false } }).toArray(); + all.sort((a, b) => a.id.localeCompare(b.id)); + const nodes = all.map(c => ({ id: c.id, prompt: c.prompt, dependsOn: c.dependsOn || [] })); + const edges: { source: string; target: string }[] = []; + for (const c of all) { + if (c.dependsOn) { + for (const parentId of c.dependsOn) { + edges.push({ source: parentId, target: c.id }); + } + } + } + res.json({ nodes, edges }); + } catch (error) { + next(error); + } +}); + +// Get single criterion by ID +app.get("/api/v1/criteria/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const criterion = await criteriaCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!criterion) { + res.status(404).json({ error: `Criteria '${id}' not found` }); + return; + } + + // Find dependents (who depends on this criterion) + const dependents = await criteriaCollection.find({ + dependsOn: id, + deletedAt: { $exists: false }, + }).toArray(); + + res.json({ + ...criterion, + dependents: dependents.map(d => d.id), + }); + } catch (error) { + next(error); + } +}); + +// Create a new criterion +app.post("/api/v1/criteria", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id, prompt, dependsOn = [] } = req.body; + + if (!id || typeof id !== "string") { + res.status(400).json({ error: "id is required and must be a string" }); + return; + } + if (!/^[a-z][a-z0-9_]*$/.test(id)) { + res.status(400).json({ error: "id must start with a lowercase letter and contain only [a-z0-9_]" }); + return; + } + if (!prompt || typeof prompt !== "string") { + res.status(400).json({ error: "prompt is required and must be a string" }); + return; + } + if (!Array.isArray(dependsOn) || !dependsOn.every((d: unknown) => typeof d === "string")) { + res.status(400).json({ error: "dependsOn must be an array of strings" }); + return; + } + + // Check for duplicates + const existing = await criteriaCollection.findOne({ id, deletedAt: { $exists: false } }); + if (existing) { + res.status(409).json({ error: `Criteria '${id}' already exists` }); + return; + } + + // Validate dependency references + for (const depId of dependsOn) { + const dep = await criteriaCollection.findOne({ id: depId, deletedAt: { $exists: false } }); + if (!dep) { + res.status(400).json({ error: `Dependency '${depId}' does not exist` }); + return; + } + } + + const doc: CriteriaDocument = { + id, + prompt: prompt.trim(), + dependsOn, + createdAt: new Date(), + }; + + await criteriaCollection.insertOne(doc as any); + res.status(201).json(doc); + } catch (error) { + next(error); + } +}); + +// Update a criterion +app.put("/api/v1/criteria/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { prompt, dependsOn } = req.body; + + const existing = await criteriaCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: `Criteria '${id}' not found` }); + return; + } + + const update: Record = { updatedAt: new Date() }; + if (prompt !== undefined) { + if (typeof prompt !== "string") { + res.status(400).json({ error: "prompt must be a string" }); + return; + } + update.prompt = prompt.trim(); + } + if (dependsOn !== undefined) { + if (!Array.isArray(dependsOn) || !dependsOn.every((d: unknown) => typeof d === "string")) { + res.status(400).json({ error: "dependsOn must be an array of strings" }); + return; + } + // Validate dependency references + for (const depId of dependsOn) { + const dep = await criteriaCollection.findOne({ id: depId, deletedAt: { $exists: false } }); + if (!dep) { + res.status(400).json({ error: `Dependency '${depId}' does not exist` }); + return; + } + } + // Self-reference check + if (dependsOn.includes(id)) { + res.status(400).json({ error: "A criterion cannot depend on itself" }); + return; + } + update.dependsOn = dependsOn; + } + + await criteriaCollection.updateOne( + { id, deletedAt: { $exists: false } }, + { $set: update } + ); + + const updated = await criteriaCollection.findOne({ id, deletedAt: { $exists: false } }); + res.json(updated); + } catch (error) { + next(error); + } +}); + +// Delete a criterion (soft-delete, rejects if has dependents) +app.delete("/api/v1/criteria/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const existing = await criteriaCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: `Criteria '${id}' not found` }); + return; + } + + // Check for dependents + const dependents = await criteriaCollection.find({ + dependsOn: id, + deletedAt: { $exists: false }, + }).toArray(); + + if (dependents.length > 0) { + const depIds = dependents.map(d => d.id).join(", "); + res.status(409).json({ + error: `Cannot delete '${id}': other criteria depend on it`, + dependents: dependents.map(d => d.id), + }); + return; + } + + await criteriaCollection.updateOne( + { id, deletedAt: { $exists: false } }, + { $set: { deletedAt: new Date() } } + ); + + res.json({ id, deleted: true }); + } catch (error) { + next(error); + } +}); + +// --- Prompt Feature CRUD & extraction --- + +// POST /api/v1/prompt-features/generate-prompt ā€” AI-generate a prompt feature prompt from a behavior description +app.post("/api/v1/prompt-features/generate-prompt", async (req: Request, res: Response, next: NextFunction) => { + try { + const { behavior, currentId } = req.body; + if (!behavior || typeof behavior !== "string" || !behavior.trim()) { + return res.status(400).json({ error: "Body must contain a non-empty 'behavior' string" }); + } + + if (!isPromptFeatureLlmAvailable()) { + return res.status(503).json({ error: "LLM not configured: register a github-models token or set GITHUB_MODELS_API_KEY" }); + } + + const allFeatures = await promptFeatureCollection + .find({ deletedAt: { $exists: false } }) + .project({ id: 1, prompt: 1, dependsOn: 1, _id: 0 }) + .toArray(); + + const existingFeatures = currentId + ? allFeatures.filter((f: any) => f.id !== currentId) + : allFeatures; + + const result = await generatePromptFeaturePrompt( + behavior.trim(), + existingFeatures as { id: string; prompt: string; dependsOn?: string[] }[], + ); + console.log("[prompt-features/generate-prompt] LLM result:", JSON.stringify(result)); + res.json(result); + } catch (err) { + if (err instanceof Error && err.message.includes("not configured")) { + return res.status(503).json({ error: err.message }); + } + next(err); + } +}); + +// POST /api/v1/prompt-features/seed ā€” bulk seed prompt features from a JSON array +app.post("/api/v1/prompt-features/seed", async (req: Request, res: Response, next: NextFunction) => { + try { + const { features } = req.body; + if (!Array.isArray(features)) { + return res.status(400).json({ error: "Body must contain a 'features' array" }); + } + + let seeded = 0; + const errors: string[] = []; + + for (const config of features) { + if (!config.id || !config.prompt) { + errors.push("Skipping entry without id or prompt"); + continue; + } + const trimmedId = String(config.id).trim(); + if (!/^[a-z][a-z0-9_]*$/.test(trimmedId)) { + errors.push(`Skipping '${trimmedId}': id must start with a lowercase letter and contain only [a-z0-9_]`); + continue; + } + // Validate dependsOn references exist + const deps = Array.isArray(config.dependsOn) ? config.dependsOn.map((d: any) => String(d).trim()) : []; + const missingDeps: string[] = []; + for (const depId of deps) { + const depExists = await promptFeatureCollection.findOne({ id: depId, deletedAt: { $exists: false } }); + // Also check if the dep is being seeded in the same batch (by id) + const inBatch = features.some((f: any) => f.id && String(f.id).trim() === depId); + if (!depExists && !inBatch) { + missingDeps.push(depId); + } + } + if (missingDeps.length > 0) { + errors.push(`Skipping '${trimmedId}': dependencies not found: ${missingDeps.join(", ")}`); + continue; + } + try { + await promptFeatureCollection.updateOne( + { id: trimmedId }, + { + $setOnInsert: { + id: trimmedId, + prompt: config.prompt.trim(), + dependsOn: deps, + createdAt: new Date(), + }, + }, + { upsert: true } + ); + seeded++; + } catch (err) { + errors.push(`Failed to seed ${config.id}: ${err}`); + } + } + + res.json({ seeded, errors }); + } catch (err) { + next(err); + } +}); + +// ========================================== +// Task Prompt endpoints +// ========================================== + +// GET /api/v1/task-prompts ā€” list all task prompts (paginated, optional search) +app.get("/api/v1/task-prompts", async (req: Request, res: Response, next: NextFunction) => { + try { + const limit = parseInt(req.query.limit as string) || 50; + const offset = parseInt(req.query.offset as string) || 0; + const search = req.query.search as string | undefined; + + const { items, total } = await taskPromptStore.getAll({ limit, offset, search }); + res.json({ items, total, limit, offset }); + } catch (error) { + next(error); + } +}); + +// GET /api/v1/task-prompts/:id ā€” get a single task prompt by ID +app.get("/api/v1/task-prompts/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const taskPrompt = await taskPromptStore.get(id); + if (!taskPrompt) { + return res.status(404).json({ error: "Task prompt not found" }); + } + res.json(taskPrompt); + } catch (error) { + next(error); + } +}); + +// POST /api/v1/task-prompts ā€” create (or find existing) task prompt. Idempotent. +app.post("/api/v1/task-prompts", async (req: Request, res: Response, next: NextFunction) => { + try { + const { text } = req.body; + if (!text || typeof text !== "string" || !text.trim()) { + return res.status(400).json({ error: "Body must contain a non-empty 'text' string" }); + } + + const taskPrompt = await taskPromptStore.findOrCreate(text); + res.status(201).json(taskPrompt); + } catch (error) { + next(error); + } +}); + +// DELETE /api/v1/task-prompts/:id ā€” soft-delete a task prompt +app.delete("/api/v1/task-prompts/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + await taskPromptStore.delete(id); + res.json({ deleted: true }); + } catch (error) { + if (error instanceof Error && error.message.includes("not found")) { + return res.status(404).json({ error: error.message }); + } + next(error); + } +}); + +// POST /api/v1/task-prompts/:id/extract-features ā€” extract prompt features for a task prompt +app.post("/api/v1/task-prompts/:id/extract-features", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { model } = req.body; + const force = req.query.force === "true"; + + const taskPrompt = await taskPromptStore.get(id); + if (!taskPrompt) { + return res.status(404).json({ error: "Task prompt not found" }); + } + + // Return cached features if available (unless force re-extraction) + if (!force && taskPrompt.features && taskPrompt.features.length > 0) { + return res.json({ + taskPromptId: taskPrompt._id, + features: taskPrompt.features, + featuresExtractedAt: taskPrompt.featuresExtractedAt, + cached: true, + }); + } + + if (!isPromptFeatureLlmAvailable()) { + return res.status(503).json({ error: "LLM not configured: register a github-models token or set GITHUB_MODELS_API_KEY" }); + } + + const allFeatures = await promptFeatureCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + + const featureConfigs = allFeatures.map(f => ({ id: f.id, prompt: f.prompt, dependsOn: f.dependsOn })); + const { results, suggestedFeatures } = await extractPromptFeatures(taskPrompt.text, featureConfigs, model); + + // Store features on the task prompt entity + const updated = await taskPromptStore.attachFeatures(id, results); + + res.json({ + taskPromptId: updated._id, + features: updated.features, + featuresExtractedAt: updated.featuresExtractedAt, + suggestedFeatures: suggestedFeatures.length > 0 ? suggestedFeatures : undefined, + cached: false, + }); + } catch (err) { + if (err instanceof Error && err.message.includes("not configured")) { + return res.status(503).json({ error: err.message }); + } + if (err instanceof Error && err.message.includes("not found")) { + return res.status(404).json({ error: err.message }); + } + next(err); + } +}); + +// POST /api/v1/prompt-features/extract-from-text ā€” extract features from raw text without persisting +app.post("/api/v1/prompt-features/extract-from-text", async (req: Request, res: Response, next: NextFunction) => { + try { + const { text, model } = req.body; + if (!text || typeof text !== "string" || !text.trim()) { + return res.status(400).json({ error: "Body must contain a non-empty 'text' string" }); + } + + if (!isPromptFeatureLlmAvailable()) { + return res.status(503).json({ error: "LLM not configured: register a github-models token or set GITHUB_MODELS_API_KEY" }); + } + + const allFeatures = await promptFeatureCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + + const featureConfigs = allFeatures.map(f => ({ id: f.id, prompt: f.prompt, dependsOn: f.dependsOn })); + const { results, suggestedFeatures } = await extractPromptFeatures(text.trim(), featureConfigs, model); + + res.json({ + features: results, + suggestedFeatures: suggestedFeatures.length > 0 ? suggestedFeatures : undefined, + cached: false, + }); + } catch (err) { + if (err instanceof Error && err.message.includes("not configured")) { + return res.status(503).json({ error: err.message }); + } + next(err); + } +}); + +// PATCH /api/v1/task-prompts/:id/features/:featureId ā€” toggle a feature's detected flag +app.patch("/api/v1/task-prompts/:id/features/:featureId", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id, featureId } = req.params; + const { detected } = req.body; + + if (typeof detected !== "boolean") { + return res.status(400).json({ error: "'detected' must be a boolean" }); + } + + const updated = await taskPromptStore.toggleFeature(id, featureId, detected); + res.json(updated); + } catch (err) { + if (err instanceof Error && err.message.includes("not found")) { + return res.status(404).json({ error: err.message }); + } + next(err); + } +}); + +// List all prompt features (with optional search) +app.get("/api/v1/prompt-features", async (_req: Request, res: Response, next: NextFunction) => { + try { + const q = _req.query.q as string | undefined; + const filter: Record = { deletedAt: { $exists: false } }; + if (q) { + filter.$or = [ + { id: { $regex: q, $options: "i" } }, + { prompt: { $regex: q, $options: "i" } }, + ]; + } + const features = await promptFeatureCollection.find(filter).toArray(); + features.sort((a, b) => a.id.localeCompare(b.id)); + res.json(features); + } catch (error) { + next(error); + } +}); + +// Get prompt feature DAG graph (nodes + edges) +app.get("/api/v1/prompt-features/graph", async (_req: Request, res: Response, next: NextFunction) => { + try { + const all = await promptFeatureCollection.find({ deletedAt: { $exists: false } }).toArray(); + all.sort((a, b) => a.id.localeCompare(b.id)); + const nodes = all.map(f => ({ id: f.id, prompt: f.prompt, dependsOn: f.dependsOn || [] })); + const edges: { source: string; target: string }[] = []; + for (const f of all) { + if (f.dependsOn) { + for (const parentId of f.dependsOn) { + edges.push({ source: parentId, target: f.id }); + } + } + } + res.json({ nodes, edges }); + } catch (error) { + next(error); + } +}); + +// Get single prompt feature by ID +app.get("/api/v1/prompt-features/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const feature = await promptFeatureCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!feature) { + res.status(404).json({ error: `Prompt feature '${id}' not found` }); + return; + } + + const dependents = await promptFeatureCollection.find({ + dependsOn: id, + deletedAt: { $exists: false }, + }).toArray(); + + res.json({ + ...feature, + dependents: dependents.map(d => d.id), + }); + } catch (error) { + next(error); + } +}); + +// Create a new prompt feature +app.post("/api/v1/prompt-features", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id, prompt, dependsOn = [] } = req.body; + + if (!id || typeof id !== "string") { + res.status(400).json({ error: "id is required and must be a string" }); + return; + } + if (!/^[a-z][a-z0-9_]*$/.test(id)) { + res.status(400).json({ error: "id must start with a lowercase letter and contain only [a-z0-9_]" }); + return; + } + if (!prompt || typeof prompt !== "string") { + res.status(400).json({ error: "prompt is required and must be a string" }); + return; + } + if (!Array.isArray(dependsOn) || !dependsOn.every((d: unknown) => typeof d === "string")) { + res.status(400).json({ error: "dependsOn must be an array of strings" }); + return; + } + + const existing = await promptFeatureCollection.findOne({ id, deletedAt: { $exists: false } }); + if (existing) { + res.status(409).json({ error: `Prompt feature '${id}' already exists` }); + return; + } + + for (const depId of dependsOn) { + const dep = await promptFeatureCollection.findOne({ id: depId, deletedAt: { $exists: false } }); + if (!dep) { + res.status(400).json({ error: `Dependency '${depId}' does not exist` }); + return; + } + } + + const doc: PromptFeatureDocument = { + id, + prompt: prompt.trim(), + dependsOn, + createdAt: new Date(), + }; + + await promptFeatureCollection.insertOne(doc as any); + res.status(201).json(doc); + } catch (error) { + next(error); + } +}); + +// Update a prompt feature +app.put("/api/v1/prompt-features/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { prompt, dependsOn } = req.body; + + const existing = await promptFeatureCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: `Prompt feature '${id}' not found` }); + return; + } + + const update: Record = { updatedAt: new Date() }; + if (prompt !== undefined) { + if (typeof prompt !== "string") { + res.status(400).json({ error: "prompt must be a string" }); + return; + } + update.prompt = prompt.trim(); + } + if (dependsOn !== undefined) { + if (!Array.isArray(dependsOn) || !dependsOn.every((d: unknown) => typeof d === "string")) { + res.status(400).json({ error: "dependsOn must be an array of strings" }); + return; + } + for (const depId of dependsOn) { + const dep = await promptFeatureCollection.findOne({ id: depId, deletedAt: { $exists: false } }); + if (!dep) { + res.status(400).json({ error: `Dependency '${depId}' does not exist` }); + return; + } + } + if (dependsOn.includes(id)) { + res.status(400).json({ error: "A prompt feature cannot depend on itself" }); + return; + } + update.dependsOn = dependsOn; + } + + await promptFeatureCollection.updateOne( + { id, deletedAt: { $exists: false } }, + { $set: update } + ); + + const updated = await promptFeatureCollection.findOne({ id, deletedAt: { $exists: false } }); + res.json(updated); + } catch (error) { + next(error); + } +}); + +// Delete a prompt feature (soft-delete, rejects if has dependents) +app.delete("/api/v1/prompt-features/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const existing = await promptFeatureCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: `Prompt feature '${id}' not found` }); + return; + } + + const dependents = await promptFeatureCollection.find({ + dependsOn: id, + deletedAt: { $exists: false }, + }).toArray(); + + if (dependents.length > 0) { + res.status(409).json({ + error: `Cannot delete '${id}': other prompt features depend on it`, + dependents: dependents.map(d => d.id), + }); + return; + } + + await promptFeatureCollection.updateOne( + { id, deletedAt: { $exists: false } }, + { $set: { deletedAt: new Date() } } + ); + + res.json({ id, deleted: true }); + } catch (error) { + next(error); + } +}); + +// ==================== Report Endpoints ==================== + +// Create a report for a run (POST /api/v1/reports) +// Accepts optional templateId to associate the report with a report template. +app.post("/api/v1/reports", async (req: Request, res: Response, next: NextFunction) => { + try { + const { requestId, templateId } = req.body; + + if (!requestId || typeof requestId !== "string") { + res.status(400).json({ error: "requestId is required and must be a string" }); + return; + } + + // Verify the run exists + const run = await collection.findOne({ _id: requestId }); + if (!run) { + res.status(404).json({ error: `Run ${requestId} not found` }); + return; + } + + // Verify the template exists (if specified) + if (templateId) { + const template = await reportTemplateCollection.findOne({ id: templateId, deletedAt: { $exists: false } }); + if (!template) { + res.status(404).json({ error: `Report template '${templateId}' not found` }); + return; + } + } + + const reportId = uuidv4(); + + const reportDoc: ReportDocument = { + _id: reportId, + requestId, + ...(templateId ? { templateId } : {}), + status: "pending", + logs: [], + createdAt: new Date(), + }; + + await reportCollection.insertOne(reportDoc); + + // Queue the report for processing + const messageContent = Buffer.from(JSON.stringify({ reportId })).toString("base64"); + await reportQueueClient.sendMessage(messageContent); + + console.log(`Created report ${reportId} for run ${requestId}${templateId ? ` (template: ${templateId})` : ""} and queued for processing`); + + res.status(201).json({ + id: reportId, + requestId, + ...(templateId ? { templateId } : {}), + status: "pending", + message: "Report generation queued", + }); + } catch (error) { + next(error); + } +}); + +// List all reports (GET /api/v1/reports) +app.get("/api/v1/reports", async (req: Request, res: Response, next: NextFunction) => { + try { + const requestIdFilter = req.query.requestId as string; + const filter: Record = {}; + if (requestIdFilter) { + filter.requestId = requestIdFilter; + } + + const reports = await reportCollection + .find(filter) + .sort({ createdAt: -1 }) + .toArray(); + + // Enrich reports with the task from their associated run + const requestIds = [...new Set(reports.map(r => r.requestId))]; + const runs = requestIds.length > 0 + ? await collection.find({ _id: { $in: requestIds } as any }, { projection: { _id: 1, "scenario.task": 1 } }).toArray() + : []; + const taskByRequestId = new Map(runs.map(r => [r._id, r.scenario?.task])); + + res.json(reports.map(r => ({ ...r, id: r._id, task: taskByRequestId.get(r.requestId) }))); + } catch (error) { + next(error); + } +}); + +// Bulk create reports for multiple runs (POST /api/v1/reports/bulk-create) +app.post("/api/v1/reports/bulk-create", async (req: Request, res: Response, next: NextFunction) => { + try { + const { requestIds } = req.body as { requestIds?: string[] }; + + if (!requestIds || !Array.isArray(requestIds) || requestIds.length === 0) { + res.status(400).json({ error: "requestIds must be a non-empty array of strings" }); + return; + } + + // Verify all runs exist + const runs = await collection.find({ _id: { $in: requestIds } as any }).toArray(); + const foundIds = new Set(runs.map(r => r._id)); + const notFound = requestIds.filter(id => !foundIds.has(id)); + + // Create reports only for runs that exist + const validIds = requestIds.filter(id => foundIds.has(id)); + const created: { reportId: string; requestId: string }[] = []; + + for (const requestId of validIds) { + const reportId = uuidv4(); + const reportDoc: ReportDocument = { + _id: reportId, + requestId, + status: "pending", + logs: [], + createdAt: new Date(), + }; + await reportCollection.insertOne(reportDoc); + + const messageContent = Buffer.from(JSON.stringify({ reportId })).toString("base64"); + await reportQueueClient.sendMessage(messageContent); + + created.push({ reportId, requestId }); + } + + console.log(`Bulk created ${created.length} reports for ${validIds.length} runs`); + + res.status(201).json({ + created: created.length, + reports: created, + notFound, + }); + } catch (error) { + next(error); + } +}); + +// Bulk report status (POST /api/v1/reports/bulk-status) +app.post("/api/v1/reports/bulk-status", async (req: Request, res: Response, next: NextFunction) => { + try { + const { requestIds } = req.body as { requestIds?: string[] }; + + if (!requestIds || !Array.isArray(requestIds) || requestIds.length === 0) { + res.status(400).json({ error: "requestIds must be a non-empty array of strings" }); + return; + } + + // Find the latest report for each requestId + const reports = await reportCollection + .find({ requestId: { $in: requestIds } }) + .sort({ createdAt: -1 }) + .toArray(); + + // Build a map of requestId ā†’ latest report status + const statusMap: Record = {}; + for (const report of reports) { + if (!statusMap[report.requestId]) { + statusMap[report.requestId] = { + reportId: report._id, + status: report.status, + }; + } + } + + res.json(statusMap); + } catch (error) { + next(error); + } +}); + +// Get a single report (GET /api/v1/reports/:id) +app.get("/api/v1/reports/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const report = await reportCollection.findOne({ _id: id }); + + if (!report) { + res.status(404).json({ error: "Report not found" }); + return; + } + + res.json({ ...report, id: report._id }); + } catch (error) { + next(error); + } +}); + +// Stream report logs via SSE (GET /api/v1/reports/:id/logs) +app.get("/api/v1/reports/:id/logs", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const fromStart = req.query.fromStart === "true"; + + const report = await reportCollection.findOne({ _id: id }); + + if (!report) { + res.status(404).json({ error: "Report not found" }); + return; + } + + // Set SSE headers + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + + // Replay existing logs if requested + if (fromStart && report.logs && report.logs.length > 0) { + for (const log of report.logs) { + res.write(`data: ${JSON.stringify(log)}\n\n`); + } + } + + // If report already completed/failed, send done and close + if (report.status === "completed" || report.status === "failed") { + res.write(`event: done\ndata: {"status":"${report.status}"}\n\n`); + res.end(); + return; + } + + // Live streaming via Redis + Change Streams (same pattern as requests) + let cleaned = false; + let changeStream: ReturnType | null = null; + let redisSubscribed = false; + + const INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000; + let inactivityTimer: ReturnType; + + const resetInactivityTimer = () => { + clearTimeout(inactivityTimer); + inactivityTimer = setTimeout(() => { + res.write(`event: timeout\ndata: {"message":"Stream timeout after 5 minutes of inactivity"}\n\n`); + client.cleanup(); + }, INACTIVITY_TIMEOUT_MS); + }; + + const heartbeat = setInterval(() => { + if (!cleaned) { + res.write(`:\n\n`); + } + }, 30_000); + + const client: SSEClient = { + res, + onActivity: resetInactivityTimer, + cleanup: () => { + if (!cleaned) { + cleaned = true; + clearTimeout(inactivityTimer); + clearInterval(heartbeat); + if (changeStream) { + changeStream.close().catch(err => console.error("Error closing report change stream:", err)); + } + if (redisSubscribed) { + unsubscribeClient(id, client); + } + res.end(); + } + }, + }; + + resetInactivityTimer(); + + if (redisHost) { + try { + await subscribeClient(id, client); + redisSubscribed = true; + } catch (err) { + console.error(`Redis subscription failed for report ${id}, using Change Streams only:`, err); + } + } + + try { + changeStream = reportCollection.watch( + [{ $match: { "documentKey._id": id, operationType: "update" } }], + { fullDocument: "updateLookup" } + ); + + changeStream.on("change", (change) => { + if (change.operationType === "update" && change.fullDocument) { + const doc = change.fullDocument; + if (doc.status === "completed" || doc.status === "failed") { + res.write(`event: done\ndata: {"status":"${doc.status}"}\n\n`); + client.cleanup(); + } + } + }); + + changeStream.on("error", (err) => { + console.error(`Report change stream error for ${id}:`, err); + }); + } catch (err) { + console.error(`Failed to create change stream for report ${id}:`, err); + } + + req.on("close", () => client.cleanup()); + } catch (error) { + next(error); + } +}); + +// Get reports for a specific run (GET /api/v1/requests/:id/reports) +app.get("/api/v1/requests/:id/reports", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + // Verify the run exists + const run = await collection.findOne({ _id: id }); + if (!run) { + res.status(404).json({ error: "Run not found" }); + return; + } + + const reports = await reportCollection + .find({ requestId: id }) + .sort({ createdAt: -1 }) + .toArray(); + + res.json(reports.map(r => ({ ...r, id: r._id }))); + } catch (error) { + next(error); + } +}); + +// POST /api/v1/reports/trigger ā€” evaluate all report templates' triggers for a completed run +// Called by coding agent workers after a run completes. Creates a report per matching template. +app.post("/api/v1/reports/trigger", async (req: Request, res: Response, next: NextFunction) => { + try { + const { requestId } = req.body; + + if (!requestId || typeof requestId !== "string") { + res.status(400).json({ error: "requestId is required and must be a string" }); + return; + } + + // Fetch the completed run + const run = await collection.findOne({ _id: requestId }); + if (!run) { + res.status(404).json({ error: `Run ${requestId} not found` }); + return; + } + + // Fetch task prompt document (needed for promptFeature trigger evaluation) + let taskPrompt: TaskPromptDocument | null = null; + if (run.taskPromptId) { + taskPrompt = await taskPromptCollection.findOne({ _id: run.taskPromptId }); + } + + // Load all active report templates + const templates = await reportTemplateCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + + if (templates.length === 0) { + // No templates defined ā€” fall back to legacy behavior (create a single report without template) + const reportId = uuidv4(); + const reportDoc: ReportDocument = { + _id: reportId, + requestId, + status: "pending", + logs: [], + createdAt: new Date(), + }; + await reportCollection.insertOne(reportDoc); + const messageContent = Buffer.from(JSON.stringify({ reportId })).toString("base64"); + await reportQueueClient.sendMessage(messageContent); + console.log(`No templates defined ā€” created legacy report ${reportId} for run ${requestId}`); + res.status(201).json({ triggered: 1, reports: [{ id: reportId, requestId, status: "pending" }] }); + return; + } + + // Evaluate each template's trigger against the run + const created: Array<{ id: string; requestId: string; templateId: string; status: string }> = []; + + for (const template of templates) { + // Cast the run to the shared RequestDocument shape for evaluateTrigger + const triggerResult = evaluateTrigger( + template.trigger as any, + run as any, + taskPrompt as any + ); + + if (triggerResult) { + const reportId = uuidv4(); + const reportDoc: ReportDocument = { + _id: reportId, + requestId, + templateId: template.id, + status: "pending", + logs: [], + createdAt: new Date(), + }; + await reportCollection.insertOne(reportDoc); + const messageContent = Buffer.from(JSON.stringify({ reportId })).toString("base64"); + await reportQueueClient.sendMessage(messageContent); + + created.push({ id: reportId, requestId, templateId: template.id, status: "pending" }); + console.log(`Trigger matched template '${template.id}' ā€” created report ${reportId} for run ${requestId}`); + } + } + + console.log(`Trigger evaluation for run ${requestId}: ${created.length}/${templates.length} templates matched`); + res.status(201).json({ triggered: created.length, reports: created }); + } catch (error) { + next(error); + } +}); + +// POST /api/v1/reports/bulk-trigger ā€” evaluate report templates for multiple runs at once +app.post("/api/v1/reports/bulk-trigger", async (req: Request, res: Response, next: NextFunction) => { + try { + const { requestIds } = req.body as { requestIds?: string[] }; + + if (!requestIds || !Array.isArray(requestIds) || requestIds.length === 0) { + res.status(400).json({ error: "requestIds must be a non-empty array of strings" }); + return; + } + + // Fetch runs + const runs = await collection.find({ _id: { $in: requestIds } as any }).toArray(); + const foundIds = new Set(runs.map(r => r._id)); + const notFound = requestIds.filter(id => !foundIds.has(id)); + + // Load all active templates + const templates = await reportTemplateCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + + const created: Array<{ reportId: string; requestId: string; templateId?: string }> = []; + + for (const run of runs) { + if (templates.length === 0) { + // No templates ā€” legacy single report per run + const reportId = uuidv4(); + const reportDoc: ReportDocument = { + _id: reportId, + requestId: run._id, + status: "pending", + logs: [], + createdAt: new Date(), + }; + await reportCollection.insertOne(reportDoc); + const messageContent = Buffer.from(JSON.stringify({ reportId })).toString("base64"); + await reportQueueClient.sendMessage(messageContent); + created.push({ reportId, requestId: run._id }); + } else { + // Fetch task prompt for trigger evaluation + let taskPrompt: TaskPromptDocument | null = null; + if (run.taskPromptId) { + taskPrompt = await taskPromptCollection.findOne({ _id: run.taskPromptId }); + } + + for (const template of templates) { + const triggerResult = evaluateTrigger( + template.trigger as any, + run as any, + taskPrompt as any + ); + + if (triggerResult) { + const reportId = uuidv4(); + const reportDoc: ReportDocument = { + _id: reportId, + requestId: run._id, + templateId: template.id, + status: "pending", + logs: [], + createdAt: new Date(), + }; + await reportCollection.insertOne(reportDoc); + const messageContent = Buffer.from(JSON.stringify({ reportId })).toString("base64"); + await reportQueueClient.sendMessage(messageContent); + created.push({ reportId, requestId: run._id, templateId: template.id }); + } + } + } + } + + console.log(`Bulk trigger: created ${created.length} reports for ${runs.length} runs`); + + res.status(201).json({ + created: created.length, + reports: created, + notFound, + }); + } catch (error) { + next(error); + } +}); + +// ============================================================ +// Report Template CRUD routes (/api/v1/report-templates) +// ============================================================ + +// List all report templates +app.get("/api/v1/report-templates", async (req: Request, res: Response, next: NextFunction) => { + try { + const q = req.query.q as string | undefined; + const filter: Record = { deletedAt: { $exists: false } }; + if (q) { + filter.$or = [ + { id: { $regex: q, $options: "i" } }, + { name: { $regex: q, $options: "i" } }, + { description: { $regex: q, $options: "i" } }, + ]; + } + const templates = await reportTemplateCollection.find(filter).toArray(); + templates.sort((a, b) => a.id.localeCompare(b.id)); + res.json(templates); + } catch (error) { + next(error); + } +}); + +// Get single report template by ID +app.get("/api/v1/report-templates/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const template = await reportTemplateCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!template) { + res.status(404).json({ error: `Report template '${id}' not found` }); + return; + } + res.json(template); + } catch (error) { + next(error); + } +}); + +// Create a report template +app.post("/api/v1/report-templates", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id, name, description, userPrompt, systemPrompt, trigger } = req.body; + + if (!id || typeof id !== "string") { + res.status(400).json({ error: "id is required and must be a string" }); + return; + } + if (!/^[a-z][a-z0-9_-]*$/.test(id)) { + res.status(400).json({ error: "id must start with a lowercase letter and contain only lowercase letters, numbers, hyphens, and underscores" }); + return; + } + if (!name || typeof name !== "string") { + res.status(400).json({ error: "name is required and must be a string" }); + return; + } + if (!userPrompt || typeof userPrompt !== "string") { + res.status(400).json({ error: "userPrompt is required and must be a string" }); + return; + } + + // Validate systemPrompt if provided + if (systemPrompt !== undefined) { + if (!systemPrompt || typeof systemPrompt !== "object") { + res.status(400).json({ error: "systemPrompt must be an object with 'mode' and 'content'" }); + return; + } + if (!["append", "override"].includes(systemPrompt.mode)) { + res.status(400).json({ error: "systemPrompt.mode must be 'append' or 'override'" }); + return; + } + if (!systemPrompt.content || typeof systemPrompt.content !== "string") { + res.status(400).json({ error: "systemPrompt.content is required and must be a string" }); + return; + } + } + + // Validate trigger if provided + if (trigger !== undefined) { + const triggerError = validateTrigger(trigger); + if (triggerError) { + res.status(400).json({ error: triggerError }); + return; + } + } + + // Check for duplicate id + const existing = await reportTemplateCollection.findOne({ id }); + if (existing && !existing.deletedAt) { + res.status(409).json({ error: `Report template '${id}' already exists` }); + return; + } + + const now = new Date(); + + if (existing && existing.deletedAt) { + // Un-delete: update the soft-deleted document + await reportTemplateCollection.updateOne( + { id }, + { + $set: { + name, + ...(description !== undefined ? { description } : {}), + userPrompt, + ...(systemPrompt !== undefined ? { systemPrompt } : {}), + ...(trigger !== undefined ? { trigger } : {}), + updatedAt: now, + }, + $unset: { deletedAt: "" }, + } + ); + const updated = await reportTemplateCollection.findOne({ id }); + res.status(201).json(updated); + } else { + const templateDoc: ReportTemplateDocument = { + id, + name, + ...(description ? { description } : {}), + userPrompt, + ...(systemPrompt ? { systemPrompt } : {}), + ...(trigger ? { trigger } : {}), + createdAt: now, + }; + await reportTemplateCollection.insertOne(templateDoc as any); + res.status(201).json(templateDoc); + } + } catch (error) { + next(error); + } +}); + +// Update a report template +app.put("/api/v1/report-templates/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { name, description, userPrompt, systemPrompt, trigger } = req.body; + + const existing = await reportTemplateCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: `Report template '${id}' not found` }); + return; + } + + const updateFields: Record = { updatedAt: new Date() }; + if (name !== undefined) updateFields.name = name; + if (description !== undefined) updateFields.description = description; + if (userPrompt !== undefined) { + if (typeof userPrompt !== "string" || !userPrompt.trim()) { + res.status(400).json({ error: "userPrompt must be a non-empty string" }); + return; + } + updateFields.userPrompt = userPrompt; + } + if (systemPrompt !== undefined) { + if (systemPrompt === null) { + // Allow removing systemPrompt by setting to null + updateFields.systemPrompt = undefined; + } else { + if (!["append", "override"].includes(systemPrompt.mode)) { + res.status(400).json({ error: "systemPrompt.mode must be 'append' or 'override'" }); + return; + } + if (!systemPrompt.content || typeof systemPrompt.content !== "string") { + res.status(400).json({ error: "systemPrompt.content is required and must be a string" }); + return; + } + updateFields.systemPrompt = systemPrompt; + } + } + if (trigger !== undefined) { + if (trigger === null) { + // Allow removing trigger (reverts to "always" behavior) + updateFields.trigger = undefined; + } else { + const triggerError = validateTrigger(trigger); + if (triggerError) { + res.status(400).json({ error: triggerError }); + return; + } + updateFields.trigger = trigger; + } + } + + await reportTemplateCollection.updateOne({ id }, { $set: updateFields }); + const updated = await reportTemplateCollection.findOne({ id }); + res.json(updated); + } catch (error) { + next(error); + } +}); + +// Delete a report template (soft-delete) +app.delete("/api/v1/report-templates/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const existing = await reportTemplateCollection.findOne({ id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: `Report template '${id}' not found` }); + return; + } + + await reportTemplateCollection.updateOne( + { id }, + { $set: { deletedAt: new Date(), updatedAt: new Date() } } + ); + + res.status(204).send(); + } catch (error) { + next(error); + } +}); + +/** + * Validate a ReportTrigger object shape. + * Returns an error message if invalid, or null if valid. + */ +function validateTrigger(trigger: unknown): string | null { + if (!trigger || typeof trigger !== "object") { + return "trigger must be an object"; + } + const t = trigger as Record; + if (!t.type || typeof t.type !== "string") { + return "trigger.type is required and must be a string"; + } + switch (t.type) { + case "always": + return null; + case "criteria": + if (!Array.isArray(t.criteriaIds) || t.criteriaIds.length === 0) { + return "trigger.criteriaIds must be a non-empty array of strings"; + } + if (!t.criteriaIds.every((id: unknown) => typeof id === "string")) { + return "trigger.criteriaIds must only contain strings"; + } + if (t.match !== undefined && t.match !== "any" && t.match !== "all") { + return "trigger.match must be 'any' or 'all'"; + } + return null; + case "taskPrompt": + if (!Array.isArray(t.taskPromptIds) || t.taskPromptIds.length === 0) { + return "trigger.taskPromptIds must be a non-empty array of strings"; + } + if (!t.taskPromptIds.every((id: unknown) => typeof id === "string")) { + return "trigger.taskPromptIds must only contain strings"; + } + return null; + case "promptFeature": + if (!Array.isArray(t.featureIds) || t.featureIds.length === 0) { + return "trigger.featureIds must be a non-empty array of strings"; + } + if (!t.featureIds.every((id: unknown) => typeof id === "string")) { + return "trigger.featureIds must only contain strings"; + } + if (t.match !== undefined && t.match !== "any" && t.match !== "all") { + return "trigger.match must be 'any' or 'all'"; + } + return null; + default: + return `Unknown trigger type: '${t.type}'. Valid types: always, criteria, taskPrompt, promptFeature`; + } +} + +// ============================================================================= +// Token Manager proxy (admin CRUD - excludes /acquire which is worker-only) +// ============================================================================= +const TOKEN_MANAGER_URL = process.env.TOKEN_MANAGER_URL || ""; + +if (TOKEN_MANAGER_URL) { + const proxyToTokenManager = async (req: Request, res: Response, next: NextFunction) => { + try { + const targetUrl = `${TOKEN_MANAGER_URL}${req.originalUrl}`; + const headers: Record = { "content-type": "application/json" }; + const fetchOpts: RequestInit = { + method: req.method, + headers, + }; + if (req.method !== "GET" && req.method !== "HEAD") { + fetchOpts.body = JSON.stringify(req.body); + } + const upstream = await fetch(targetUrl, fetchOpts); + const contentType = upstream.headers.get("content-type") || "application/json"; + const body = await upstream.text(); + res.status(upstream.status).set("content-type", contentType).send(body); + } catch (error) { + next(error); + } + }; + + // CRUD routes proxied to Token Manager (portal uses these) + app.post("/api/v1/tokens/preview", proxyToTokenManager); // must be before :id routes + app.post("/api/v1/tokens", proxyToTokenManager); + app.get("/api/v1/tokens", proxyToTokenManager); + app.get("/api/v1/tokens/:id", proxyToTokenManager); + app.put("/api/v1/tokens/:id", proxyToTokenManager); + app.delete("/api/v1/tokens/:id", proxyToTokenManager); + app.post("/api/v1/tokens/:id/validate", proxyToTokenManager); + // NOTE: POST /api/v1/tokens/acquire is intentionally NOT proxied. + // Workers call token-manager directly (ClusterIP) for /acquire. + + console.log(`[api] Token Manager proxy enabled ā†’ ${TOKEN_MANAGER_URL}`); +} else { + console.log("[api] Token Manager proxy disabled (TOKEN_MANAGER_URL not set)"); +} + +// ============================================= +// Coding Agents CRUD +// ============================================= + +// List all agents +app.get("/api/v1/agents", async (_req: Request, res: Response, next: NextFunction) => { + try { + const agents = await agentCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + // Sort in JS for CosmosDB compatibility + agents.sort((a, b) => a._id.localeCompare(b._id)); + res.json(agents.map((a) => ({ ...a, id: a._id }))); + } catch (error) { + next(error); + } +}); + +// Get a single agent +app.get("/api/v1/agents/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const agent = await agentCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!agent) { + res.status(404).json({ error: "Agent not found" }); + return; + } + res.json({ ...agent, id: agent._id }); + } catch (error) { + next(error); + } +}); + +// Create or upsert an agent (idempotent ā€” used by seed jobs) +app.post("/api/v1/agents", async (req: Request, res: Response, next: NextFunction) => { + try { + const { _id, name, description, supportedModels, defaultModel } = req.body; + + if (!_id || typeof _id !== "string") { + res.status(400).json({ error: "_id is required and must be a string" }); + return; + } + if (!name || typeof name !== "string") { + res.status(400).json({ error: "name is required and must be a string" }); + return; + } + if (!Array.isArray(supportedModels) || !supportedModels.every((m: unknown) => typeof m === "string")) { + res.status(400).json({ error: "supportedModels is required and must be an array of strings" }); + return; + } + if (defaultModel !== undefined && typeof defaultModel !== "string") { + res.status(400).json({ error: "defaultModel must be a string" }); + return; + } + if (defaultModel && supportedModels.length > 0 && !supportedModels.includes(defaultModel)) { + res.status(400).json({ error: "defaultModel must be one of supportedModels" }); + return; + } + + const now = new Date(); + const existing = await agentCollection.findOne({ _id }); + + if (existing) { + // Upsert: update existing (un-delete if soft-deleted) + await agentCollection.updateOne( + { _id }, + { + $set: { + name, + ...(description !== undefined ? { description } : {}), + supportedModels, + ...(defaultModel !== undefined ? { defaultModel } : {}), + updatedAt: now, + }, + $unset: { deletedAt: "" }, + } + ); + const updated = await agentCollection.findOne({ _id }); + res.json({ ...updated, id: updated!._id }); + } else { + // Create new + const agentDoc: CodingAgentDocument = { + _id, + name, + ...(description ? { description } : {}), + supportedModels, + ...(defaultModel ? { defaultModel } : {}), + createdAt: now, + }; + await agentCollection.insertOne(agentDoc); + res.status(201).json({ ...agentDoc, id: agentDoc._id }); + } + } catch (error) { + next(error); + } +}); + +// Update an agent +app.put("/api/v1/agents/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { name, description, supportedModels, defaultModel } = req.body; + + const existing = await agentCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Agent not found" }); + return; + } + + const updateFields: Record = { updatedAt: new Date() }; + if (name !== undefined) updateFields.name = name; + if (description !== undefined) updateFields.description = description; + if (supportedModels !== undefined) { + if (!Array.isArray(supportedModels) || !supportedModels.every((m: unknown) => typeof m === "string")) { + res.status(400).json({ error: "supportedModels must be an array of strings" }); + return; + } + updateFields.supportedModels = supportedModels; + } + if (defaultModel !== undefined) { + const models = (supportedModels as string[] | undefined) || existing.supportedModels; + if (defaultModel && models.length > 0 && !models.includes(defaultModel)) { + res.status(400).json({ error: "defaultModel must be one of supportedModels" }); + return; + } + updateFields.defaultModel = defaultModel; + } + + await agentCollection.updateOne({ _id: id }, { $set: updateFields }); + + const updated = await agentCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Soft-delete an agent +app.delete("/api/v1/agents/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const existing = await agentCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Agent not found" }); + return; + } + + await agentCollection.updateOne( + { _id: id }, + { $set: { deletedAt: new Date(), updatedAt: new Date() } } + ); + + res.status(204).send(); + } catch (error) { + next(error); + } +}); + +// ============================================================ +// Models routes (/api/v1/models) ā€” lifecycle-tracked model scanning +// ============================================================ + +// List models (filterable by agentId and/or provider) +app.get("/api/v1/models", async (req: Request, res: Response, next: NextFunction) => { + try { + const { agentId, provider } = req.query; + const filter: Record = {}; + if (agentId && typeof agentId === "string") filter.agentId = agentId; + if (provider && typeof provider === "string") filter.provider = provider; + + const models = await modelCollection + .find(filter) + .sort({ modelId: 1 }) + .toArray(); + res.json(models); + } catch (error) { + next(error); + } +}); + +// Get a single model by compound ID (agentId:modelId) +app.get("/api/v1/models/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const model = await modelCollection.findOne({ _id: id }); + if (!model) { + res.status(404).json({ error: "Model not found" }); + return; + } + res.json(model); + } catch (error) { + next(error); + } +}); + +// Sync models from a scanner ā€” bulk upsert with lifecycle reconciliation +// POST /api/v1/models/sync +// Body: { agentId, provider, models: [{ id, providerAvailableFrom?, providerEndOfLife?, metadata? }], scannedAt } +app.post("/api/v1/models/sync", async (req: Request, res: Response, next: NextFunction) => { + try { + const { agentId, provider, models, scannedAt } = req.body; + + // Validate required fields + if (!agentId || typeof agentId !== "string") { + res.status(400).json({ error: "agentId is required and must be a string" }); + return; + } + if (!provider || typeof provider !== "string") { + res.status(400).json({ error: "provider is required and must be a string" }); + return; + } + if (!Array.isArray(models)) { + res.status(400).json({ error: "models is required and must be an array" }); + return; + } + if (!scannedAt || typeof scannedAt !== "string") { + res.status(400).json({ error: "scannedAt is required and must be an ISO 8601 string" }); + return; + } + + const now = new Date(scannedAt); + const scannedModelIds = new Set(); + + const added: string[] = []; + const unchanged: string[] = []; + + // Upsert each scanned model + for (const model of models) { + if (!model.id || typeof model.id !== "string") continue; // skip invalid entries + scannedModelIds.add(model.id); + + const compoundId = `${agentId}:${model.id}`; + const existing = await modelCollection.findOne({ _id: compoundId }); + + if (existing) { + // Model still present ā€” update lastSeenAt, clear disappearedAt + const updateFields: Record = { lastSeenAt: now }; + if (existing.disappearedAt) { + // Model reappeared + updateFields.disappearedAt = undefined; + } + if (model.providerAvailableFrom) { + updateFields.providerAvailableFrom = new Date(model.providerAvailableFrom); + } + if (model.providerEndOfLife) { + updateFields.providerEndOfLife = new Date(model.providerEndOfLife); + } + if (model.metadata) { + updateFields.metadata = model.metadata; + } + + const unsetFields: Record = {}; + if (existing.disappearedAt) { + unsetFields.disappearedAt = ""; + } + + await modelCollection.updateOne( + { _id: compoundId }, + { + $set: updateFields, + ...(Object.keys(unsetFields).length > 0 ? { $unset: unsetFields } : {}), + }, + ); + unchanged.push(model.id); + } else { + // New model ā€” insert + const doc: ModelDocument = { + _id: compoundId, + modelId: model.id, + provider, + agentId, + firstSeenAt: now, + lastSeenAt: now, + ...(model.providerAvailableFrom + ? { providerAvailableFrom: new Date(model.providerAvailableFrom) } + : {}), + ...(model.providerEndOfLife + ? { providerEndOfLife: new Date(model.providerEndOfLife) } + : {}), + ...(model.metadata ? { metadata: model.metadata } : {}), + }; + await modelCollection.insertOne(doc); + added.push(model.id); + } + } + + // Mark disappeared models ā€” those in DB for this agent+provider but not in scan + const existingModels = await modelCollection + .find({ agentId, provider, disappearedAt: { $exists: false } }) + .toArray(); + + const removed: string[] = []; + for (const existing of existingModels) { + if (!scannedModelIds.has(existing.modelId)) { + await modelCollection.updateOne( + { _id: existing._id }, + { $set: { disappearedAt: now } }, + ); + removed.push(existing.modelId); + } + } + + // Update agent's supportedModels with active (non-disappeared) models + const activeModels = await modelCollection + .find({ agentId, disappearedAt: { $exists: false } }) + .toArray(); + const activeModelIds = activeModels.map((m) => m.modelId).sort(); + + const agent = await agentCollection.findOne({ _id: agentId }); + if (agent) { + await agentCollection.updateOne( + { _id: agentId }, + { $set: { supportedModels: activeModelIds, updatedAt: new Date() } }, + ); + } + + const report = { added, removed, unchanged }; + console.log( + `Model sync for ${agentId}/${provider}: +${added.length} -${removed.length} =${unchanged.length}`, + ); + res.json(report); + } catch (error) { + next(error); + } +}); + +// ============================================================ +// MCP Server CRUD routes (/api/v1/mcp/servers) +// ============================================================ + +const VALID_MCP_TRANSPORT_TYPES = ["sse", "http"] as const; + +// List MCP servers +app.get("/api/v1/mcp/servers", async (_req: Request, res: Response, next: NextFunction) => { + try { + const servers = await mcpServerCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + servers.sort((a, b) => a._id.localeCompare(b._id)); + res.json(servers.map((s) => ({ ...s, id: s._id }))); + } catch (error) { + next(error); + } +}); + +// Get MCP server by slug +app.get("/api/v1/mcp/servers/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const server = await mcpServerCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!server) { + res.status(404).json({ error: "MCP server not found" }); + return; + } + res.json({ ...server, id: server._id }); + } catch (error) { + next(error); + } +}); + +// Create MCP server +app.post("/api/v1/mcp/servers", async (req: Request, res: Response, next: NextFunction) => { + try { + const { _id, name, type, url, headers, description } = req.body; + + if (!_id || typeof _id !== "string") { + res.status(400).json({ error: "_id (slug) is required and must be a string" }); + return; + } + if (!/^[a-z0-9][a-z0-9-]*[a-z0-9]$/.test(_id) && !/^[a-z0-9]$/.test(_id)) { + res.status(400).json({ error: "_id must be a lowercase slug (letters, numbers, hyphens)" }); + return; + } + if (!name || typeof name !== "string") { + res.status(400).json({ error: "name is required and must be a string" }); + return; + } + if (!type || !VALID_MCP_TRANSPORT_TYPES.includes(type)) { + res.status(400).json({ error: `type is required and must be one of: ${VALID_MCP_TRANSPORT_TYPES.join(", ")}` }); + return; + } + if (!url || typeof url !== "string") { + res.status(400).json({ error: "url is required and must be a string" }); + return; + } + if (headers !== undefined) { + if (!Array.isArray(headers) || !headers.every((h: unknown) => typeof h === "object" && h !== null && typeof (h as Record).name === "string" && typeof (h as Record).value === "string")) { + res.status(400).json({ error: "headers must be an array of { name: string, value: string }" }); + return; + } + } + + const now = new Date(); + const existing = await mcpServerCollection.findOne({ _id }); + + if (existing) { + // Upsert: un-delete if soft-deleted, update fields + await mcpServerCollection.updateOne( + { _id }, + { + $set: { + name, + type, + url, + ...(headers !== undefined ? { headers } : {}), + ...(description !== undefined ? { description } : {}), + updatedAt: now, + }, + $unset: { deletedAt: "" }, + } + ); + const updated = await mcpServerCollection.findOne({ _id }); + res.json({ ...updated, id: updated!._id }); + } else { + const serverDoc: McpServerDocument = { + _id, + name, + type, + url, + ...(headers ? { headers } : {}), + ...(description ? { description } : {}), + createdAt: now, + }; + await mcpServerCollection.insertOne(serverDoc); + res.status(201).json({ ...serverDoc, id: serverDoc._id }); + } + } catch (error) { + next(error); + } +}); + +// Update MCP server +app.put("/api/v1/mcp/servers/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { name, type, url, headers, description } = req.body; + + const existing = await mcpServerCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "MCP server not found" }); + return; + } + + const updateFields: Record = { updatedAt: new Date() }; + if (name !== undefined) updateFields.name = name; + if (type !== undefined) { + if (!VALID_MCP_TRANSPORT_TYPES.includes(type)) { + res.status(400).json({ error: `type must be one of: ${VALID_MCP_TRANSPORT_TYPES.join(", ")}` }); + return; + } + updateFields.type = type; + } + if (url !== undefined) updateFields.url = url; + if (headers !== undefined) { + if (!Array.isArray(headers) || !headers.every((h: unknown) => typeof h === "object" && h !== null && typeof (h as Record).name === "string" && typeof (h as Record).value === "string")) { + res.status(400).json({ error: "headers must be an array of { name: string, value: string }" }); + return; + } + updateFields.headers = headers; + } + if (description !== undefined) updateFields.description = description; + + await mcpServerCollection.updateOne({ _id: id }, { $set: updateFields }); + const updated = await mcpServerCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Delete MCP server (soft-delete) +app.delete("/api/v1/mcp/servers/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + + const existing = await mcpServerCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "MCP server not found" }); + return; + } + + await mcpServerCollection.updateOne( + { _id: id }, + { $set: { deletedAt: new Date(), updatedAt: new Date() } } + ); + + res.status(204).send(); + } catch (error) { + next(error); + } +}); + +// ===================================================================== +// Skills API +// ===================================================================== + +// List all skills (with optional ?q= text search) +app.get("/api/v1/skills", async (_req: Request, res: Response, next: NextFunction) => { + try { + const skills = await skillCollection + .find({ deletedAt: { $exists: false } }) + .toArray(); + skills.sort((a, b) => a._id.localeCompare(b._id)); + res.json(skills.map((s) => ({ ...s, id: s._id }))); + } catch (error) { + next(error); + } +}); + +// Unified skill search ā€” merges internal DB + skills.sh results +// MUST be defined before /:id(*) to avoid being caught by the wildcard +app.get("/api/v1/skills/search", async (req: Request, res: Response, next: NextFunction) => { + try { + const { q, limit: limitStr } = req.query; + + if (!q || typeof q !== "string" || !q.trim()) { + res.status(400).json({ error: "Query parameter 'q' is required" }); + return; + } + + const limit = Math.min(Math.max(parseInt(limitStr as string, 10) || 10, 1), 50); + const query = q.trim(); + + // Search internal DB (case-insensitive regex) + const regex = { $regex: query, $options: "i" }; + const internalSkills = await skillCollection + .find({ + deletedAt: { $exists: false }, + $or: [ + { name: regex }, + { skillName: regex }, + { description: regex }, + ], + }) + .limit(limit) + .toArray(); + + const internalResults: SkillSearchResult[] = internalSkills.map((s) => ({ + id: s._id, + name: s.name, + source: s.source, + description: s.description, + internal: true, + })); + + // Also track internal slugs to deduplicate + const internalSlugs = new Set(internalSkills.map((s) => s._id)); + + // Search skills.sh (external registry) + let externalResults: SkillSearchResult[] = []; + try { + const skillsShUrl = `https://skills.sh/api/search?q=${encodeURIComponent(query)}&limit=${limit}`; + const externalRes = await fetch(skillsShUrl, { + headers: { "User-Agent": "scope-mt-api" }, + signal: AbortSignal.timeout(5000), + }); + if (externalRes.ok) { + const data = await externalRes.json() as { skills?: Array<{ id: string; name: string; installs?: number; source?: string }> }; + if (data.skills && Array.isArray(data.skills)) { + externalResults = data.skills + .filter((s) => !internalSlugs.has(s.id)) + .map((s) => ({ + id: s.id, + name: s.name, + source: s.source ?? s.id.split("/").slice(0, 2).join("/"), + internal: false, + installs: s.installs, + })); + } + } + } catch { + // skills.sh is optional ā€” don't fail the request if it's down + console.warn("skills.sh search failed, returning only internal results"); + } + + // Merge: internal first, then external + const results = [...internalResults, ...externalResults].slice(0, limit); + res.json(results); + } catch (error) { + next(error); + } +}); + +// Search external skills registry only (skills.sh) +app.get("/api/v1/skills/search/external", async (req: Request, res: Response, next: NextFunction) => { + try { + const { q, limit: limitStr } = req.query; + + if (!q || typeof q !== "string" || !q.trim()) { + res.status(400).json({ error: "Query parameter 'q' is required" }); + return; + } + + const limit = Math.min(Math.max(parseInt(limitStr as string, 10) || 10, 1), 50); + const query = q.trim(); + + let externalResults: SkillSearchResult[] = []; + try { + const skillsShUrl = `https://skills.sh/api/search?q=${encodeURIComponent(query)}&limit=${limit}`; + const externalRes = await fetch(skillsShUrl, { + headers: { "User-Agent": "scope-mt-api" }, + signal: AbortSignal.timeout(5000), + }); + if (externalRes.ok) { + const data = await externalRes.json() as { skills?: Array<{ id: string; name: string; installs?: number; source?: string; description?: string }> }; + if (data.skills && Array.isArray(data.skills)) { + // Deduplicate against internal skills + const internalSlugs = new Set( + (await skillCollection.find({ deletedAt: { $exists: false } }, { projection: { _id: 1 } }).toArray()).map((s) => s._id) + ); + externalResults = data.skills + .filter((s) => !internalSlugs.has(s.id)) + .map((s) => ({ + id: s.id, + name: s.name, + source: s.source ?? s.id.split("/").slice(0, 2).join("/"), + description: s.description, + internal: false, + installs: s.installs, + })); + } + } + } catch { + console.warn("skills.sh search failed"); + } + + res.json(externalResults.slice(0, limit)); + } catch (error) { + next(error); + } +}); + +// List skill revisions for a given skill slug (source/skillName) +// NOTE: Must be before the generic GET /:id(*) to avoid the greedy wildcard matching "slug/revisions" as the id. +app.get("/api/v1/skills/:id(*)/revisions", async (req: Request, res: Response, next: NextFunction) => { + try { + const id = req.params.id ?? req.params[0]; + const skill = await skillCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!skill) { + res.status(404).json({ error: "Skill not found" }); + return; + } + + const limitStr = req.query.limit as string | undefined; + const limit = Math.min(Math.max(parseInt(limitStr ?? "20", 10), 1), 100); + + const revisions = await skillRevisionStore.listBySkill(skill.source, skill.skillName, { limit }); + res.json(revisions); + } catch (error) { + next(error); + } +}); + +// Get skill by slug (must be after /search and /revisions to avoid wildcard matching) +app.get("/api/v1/skills/:id(*)", async (req: Request, res: Response, next: NextFunction) => { + try { + const id = req.params.id ?? req.params[0]; + const skill = await skillCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!skill) { + res.status(404).json({ error: "Skill not found" }); + return; + } + res.json({ ...skill, id: skill._id }); + } catch (error) { + next(error); + } +}); + +// Create / import a skill +app.post("/api/v1/skills", async (req: Request, res: Response, next: NextFunction) => { + try { + const { source, skillName, name, description, origin } = req.body; + + if (!source || typeof source !== "string") { + res.status(400).json({ error: "source is required and must be a string (GitHub repo, e.g. 'vercel-labs/agent-skills')" }); + return; + } + if (!skillName || typeof skillName !== "string") { + res.status(400).json({ error: "skillName is required and must be a string" }); + return; + } + if (!name || typeof name !== "string") { + res.status(400).json({ error: "name is required and must be a string" }); + return; + } + if (origin !== undefined && origin !== "skills-sh" && origin !== "manual") { + res.status(400).json({ error: "origin must be 'skills-sh' or 'manual'" }); + return; + } + + const _id = `${source}/${skillName}`; + const now = new Date(); + const existing = await skillCollection.findOne({ _id }); + + if (existing) { + // Upsert: un-delete if soft-deleted, update fields + await skillCollection.updateOne( + { _id }, + { + $set: { + name, + source, + skillName, + ...(description !== undefined ? { description } : {}), + ...(origin ? { origin } : {}), + updatedAt: now, + }, + $unset: { deletedAt: "" }, + } + ); + const updated = await skillCollection.findOne({ _id }); + res.json({ ...updated, id: updated!._id }); + } else { + const skillDoc: SkillDocument = { + _id, + source, + skillName, + name, + ...(description ? { description } : {}), + origin: origin || "manual", + createdAt: now, + }; + await skillCollection.insertOne(skillDoc as any); + res.status(201).json({ ...skillDoc, id: skillDoc._id }); + } + } catch (error) { + next(error); + } +}); + +// Delete skill (soft-delete) +app.delete("/api/v1/skills/:id(*)", async (req: Request, res: Response, next: NextFunction) => { + try { + const id = req.params.id ?? req.params[0]; + + const existing = await skillCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Skill not found" }); + return; + } + + await skillCollection.updateOne( + { _id: id }, + { $set: { deletedAt: new Date(), updatedAt: new Date() } } + ); + + // Also delete all associated skill revisions + await skillRevisionStore.deleteBySkill(existing.source, existing.skillName); + + res.status(204).send(); + } catch (error) { + next(error); + } +}); + +// ===================================================================== +// Skill Revisions API +// ===================================================================== + +// Download skill revision archive (tar.gz) by ref ā€” used by workers to fetch skill files through the API +app.get("/api/v1/skill-revisions/by-ref/:ref(*)/archive", async (req: Request, res: Response, next: NextFunction) => { + try { + const ref = req.params.ref ?? req.params[0]; + // Strip trailing "/archive" that Express includes in the wildcard match + const cleanRef = ref.replace(/\/archive$/, ""); + const revision = await skillRevisionStore.getByRef(cleanRef); + if (!revision) { + res.status(404).json({ error: "Skill revision not found" }); + return; + } + + if (!revision.archiveUrl) { + res.status(404).json({ error: "Skill revision has no archive" }); + return; + } + + // Parse the blob name from the archiveUrl + // archiveUrl format: https://.blob.core.windows.net/skill-archives/ + // or Azurite: http://127.0.0.1:10000/devstoreaccount1/skill-archives/ + const archiveUrlObj = new URL(revision.archiveUrl); + const pathParts = archiveUrlObj.pathname.split("/").filter(Boolean); + // pathParts: ["skill-archives", ""] or ["devstoreaccount1", "skill-archives", ""] + const containerIdx = pathParts.indexOf("skill-archives"); + if (containerIdx === -1 || containerIdx >= pathParts.length - 1) { + res.status(500).json({ error: "Cannot parse archive blob path" }); + return; + } + const blobName = pathParts.slice(containerIdx + 1).join("/"); + + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else if (storageAccountName) { + const credential = new DefaultAzureCredential(); + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + credential + ); + } else { + res.status(500).json({ error: "Blob storage not configured" }); + return; + } + + const containerClient = blobServiceClient.getContainerClient("skill-archives"); + const blobClient = containerClient.getBlobClient(blobName); + + const downloadResponse = await blobClient.download(); + if (!downloadResponse.readableStreamBody) { + res.status(500).json({ error: "Failed to download archive from blob storage" }); + return; + } + + res.setHeader("Content-Type", "application/gzip"); + res.setHeader("Content-Disposition", `attachment; filename="${blobName}"`); + if (downloadResponse.contentLength !== undefined) { + res.setHeader("Content-Length", downloadResponse.contentLength.toString()); + } + + downloadResponse.readableStreamBody.pipe(res); + } catch (error) { + if (error instanceof RestError && error.statusCode === 404) { + res.status(404).json({ error: "Archive blob not found in storage" }); + return; + } + next(error); + } +}); + +// Get skill revision by human-readable ref +app.get("/api/v1/skill-revisions/by-ref/:ref(*)", async (req: Request, res: Response, next: NextFunction) => { + try { + const ref = req.params.ref ?? req.params[0]; + const revision = await skillRevisionStore.getByRef(ref); + if (!revision) { + res.status(404).json({ error: "Skill revision not found" }); + return; + } + res.json(revision); + } catch (error) { + next(error); + } +}); + +// Get skill revision by ID (UUIDv5) +app.get("/api/v1/skill-revisions/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const revision = await skillRevisionStore.get(id); + if (!revision) { + res.status(404).json({ error: "Skill revision not found" }); + return; + } + res.json(revision); + } catch (error) { + next(error); + } +}); + +// Resolve a skill ā€” trigger resolution from GitHub and create a revision +app.post("/api/v1/skills/:id(*)/resolve", async (req: Request, res: Response, next: NextFunction) => { + try { + const id = req.params.id ?? req.params[0]; + const skill = await skillCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!skill) { + res.status(404).json({ error: "Skill not found" }); + return; + } + + // Upload archive to blob storage + const uploadArchive = async (archiveName: string, data: Buffer): Promise => { + if (!storageConnectionString && !storageAccountName) { + throw new Error("Blob storage not configured ā€” cannot store skill archives"); + } + + let blobServiceClient: BlobServiceClient; + if (storageConnectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(storageConnectionString); + } else { + const credential = new DefaultAzureCredential(); + blobServiceClient = new BlobServiceClient( + `https://${storageAccountName}.blob.core.windows.net`, + credential + ); + } + + const containerClient = blobServiceClient.getContainerClient("skill-archives"); + await containerClient.createIfNotExists(); + const blockBlobClient = containerClient.getBlockBlobClient(archiveName); + await blockBlobClient.upload(data, data.length, { + blobHTTPHeaders: { blobContentType: "application/gzip" }, + }); + return blockBlobClient.url; + }; + + const revision = await skillResolver.resolve(skill.source, skill.skillName, skillRevisionStore, uploadArchive); + res.json(revision); + } catch (error) { + next(error); + } +}); + +// ===================================================================== +// Insights API +// ===================================================================== + +// List all insights (with optional ?q= text search, ?blocked= filter) +app.get("/api/v1/insights", async (req: Request, res: Response, next: NextFunction) => { + try { + const { q, blocked } = req.query; + const filter: Record = { deletedAt: { $exists: false } }; + + if (blocked !== undefined) { + filter.blocked = blocked === "true"; + } + + if (q && typeof q === "string" && q.trim()) { + // Case-insensitive regex search across title, description, category, and tags + const regex = { $regex: q.trim(), $options: "i" }; + filter.$or = [ + { title: regex }, + { description: regex }, + { category: regex }, + { tags: regex }, + ]; + } + + const insights = await insightsCollection + .find(filter) + .sort({ createdAt: -1 }) + .toArray(); + + res.json(insights.map((i) => ({ ...i, id: i._id }))); + } catch (error) { + next(error); + } +}); + +// Search insights by keyword (fuzzy regex match) +app.get("/api/v1/insights/search", async (req: Request, res: Response, next: NextFunction) => { + try { + const { q, blocked } = req.query; + + if (!q || typeof q !== "string" || !q.trim()) { + res.status(400).json({ error: "Query parameter 'q' is required" }); + return; + } + + const filter: Record = { deletedAt: { $exists: false } }; + + if (blocked !== undefined) { + filter.blocked = blocked === "true"; + } else { + // Default: exclude blocked insights from search + filter.blocked = { $ne: true }; + } + + // Split query into words and match all of them (AND) across title/description/tags + const words = q.trim().split(/\s+/); + filter.$and = words.map((word) => { + const regex = { $regex: word, $options: "i" }; + return { + $or: [ + { title: regex }, + { description: regex }, + { category: regex }, + { tags: regex }, + ], + }; + }); + + const insights = await insightsCollection + .find(filter) + .sort({ referenceCount: -1, createdAt: -1 }) + .limit(20) + .toArray(); + + res.json(insights.map((i) => ({ ...i, id: i._id }))); + } catch (error) { + next(error); + } +}); + +// Get a single insight +app.get("/api/v1/insights/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const insight = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!insight) { + res.status(404).json({ error: "Insight not found" }); + return; + } + res.json({ ...insight, id: insight._id }); + } catch (error) { + next(error); + } +}); + +// Create a new insight +app.post("/api/v1/insights", async (req: Request, res: Response, next: NextFunction) => { + try { + const { title, description, category, tags, createdBy, sourceReportId } = req.body; + + if (!title || typeof title !== "string" || !title.trim()) { + res.status(400).json({ error: "title is required" }); + return; + } + if (!description || typeof description !== "string" || !description.trim()) { + res.status(400).json({ error: "description is required" }); + return; + } + + const now = new Date(); + const doc: InsightDocument = { + _id: uuidv4(), + title: title.trim(), + description: description.trim(), + category: category?.trim() || undefined, + tags: Array.isArray(tags) ? tags.map((t: string) => t.trim()).filter(Boolean) : undefined, + upvotes: 0, + downvotes: 0, + blocked: false, + referenceCount: 0, + createdBy: createdBy === "agent" ? "agent" : "user", + sourceReportId: sourceReportId || undefined, + createdAt: now, + }; + + await insightsCollection.insertOne(doc); + res.status(201).json({ ...doc, id: doc._id }); + } catch (error) { + next(error); + } +}); + +// Update an insight +app.put("/api/v1/insights/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const existing = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + const { title, description, category, tags } = req.body; + const updateFields: Record = { updatedAt: new Date() }; + + if (title !== undefined) updateFields.title = title.trim(); + if (description !== undefined) updateFields.description = description.trim(); + if (category !== undefined) updateFields.category = category?.trim() || undefined; + if (tags !== undefined) updateFields.tags = Array.isArray(tags) ? tags.map((t: string) => t.trim()).filter(Boolean) : undefined; + + await insightsCollection.updateOne({ _id: id }, { $set: updateFields }); + const updated = await insightsCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Soft-delete an insight +app.delete("/api/v1/insights/:id", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const existing = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + await insightsCollection.updateOne( + { _id: id }, + { $set: { deletedAt: new Date(), updatedAt: new Date() } } + ); + res.status(204).send(); + } catch (error) { + next(error); + } +}); + +// Upvote an insight +app.post("/api/v1/insights/:id/upvote", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const existing = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + await insightsCollection.updateOne({ _id: id }, { $inc: { upvotes: 1 }, $set: { updatedAt: new Date() } }); + const updated = await insightsCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Downvote an insight +app.post("/api/v1/insights/:id/downvote", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const existing = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + await insightsCollection.updateOne({ _id: id }, { $inc: { downvotes: 1 }, $set: { updatedAt: new Date() } }); + const updated = await insightsCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Block an insight +app.post("/api/v1/insights/:id/block", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const existing = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + await insightsCollection.updateOne({ _id: id }, { $set: { blocked: true, updatedAt: new Date() } }); + const updated = await insightsCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Unblock an insight +app.post("/api/v1/insights/:id/unblock", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const existing = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!existing) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + await insightsCollection.updateOne({ _id: id }, { $set: { blocked: false, updatedAt: new Date() } }); + const updated = await insightsCollection.findOne({ _id: id }); + res.json({ ...updated, id: updated!._id }); + } catch (error) { + next(error); + } +}); + +// Get reports that reference a specific insight +app.get("/api/v1/insights/:id/reports", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const insight = await insightsCollection.findOne({ _id: id, deletedAt: { $exists: false } }); + if (!insight) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + const reports = await reportCollection + .find({ "insightReferences.insightId": id }) + .sort({ createdAt: -1 }) + .toArray(); + + res.json(reports.map((r) => ({ ...r, id: r._id }))); + } catch (error) { + next(error); + } +}); + +// Get insights referenced by a specific report +app.get("/api/v1/reports/:id/insights", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const report = await reportCollection.findOne({ _id: id }); + if (!report) { + res.status(404).json({ error: "Report not found" }); + return; + } + + if (!report.insightReferences || report.insightReferences.length === 0) { + res.json([]); + return; + } + + const insightIds = report.insightReferences.map((ref) => ref.insightId); + const insights = await insightsCollection + .find({ _id: { $in: insightIds }, deletedAt: { $exists: false } }) + .toArray(); + + // Enrich with reference metadata + const enriched = insights.map((insight) => { + const ref = report.insightReferences!.find((r) => r.insightId === insight._id); + return { + ...insight, + id: insight._id, + referencedAt: ref?.referencedAt, + isNew: ref?.isNew, + }; + }); + + res.json(enriched); + } catch (error) { + next(error); + } +}); + +// Add an insight reference to a report +app.post("/api/v1/reports/:id/insights", async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const { insightId, isNew } = req.body; + + if (!insightId || typeof insightId !== "string") { + res.status(400).json({ error: "insightId is required" }); + return; + } + + const report = await reportCollection.findOne({ _id: id }); + if (!report) { + res.status(404).json({ error: "Report not found" }); + return; + } + + const insight = await insightsCollection.findOne({ _id: insightId, deletedAt: { $exists: false } }); + if (!insight) { + res.status(404).json({ error: "Insight not found" }); + return; + } + + // Check if already referenced + const alreadyReferenced = report.insightReferences?.some((ref) => ref.insightId === insightId); + if (alreadyReferenced) { + res.status(409).json({ error: "Insight already referenced by this report" }); + return; + } + + const reference: InsightReference = { + insightId, + referencedAt: new Date(), + isNew: isNew === true, + }; + + // Add reference to report + await reportCollection.updateOne( + { _id: id }, + { $push: { insightReferences: reference }, $set: { updatedAt: new Date() } } + ); + + // Increment reference count on insight + await insightsCollection.updateOne( + { _id: insightId }, + { $inc: { referenceCount: 1 }, $set: { updatedAt: new Date() } } + ); + + res.status(201).json(reference); + } catch (error) { + next(error); + } +}); + +// ā”€ā”€ā”€ Feature Flags ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +// GET /api/v1/feature-flags ā€” list all feature flags +app.get("/api/v1/feature-flags", async (_req: Request, res: Response, next: NextFunction) => { + try { + const flags = await featureFlagCollection.find({}).toArray(); + res.json(flags); + } catch (error) { + next(error); + } +}); + +// PUT /api/v1/feature-flags/:key ā€” update a feature flag +app.put("/api/v1/feature-flags/:key", async (req: Request, res: Response, next: NextFunction) => { + try { + const { key } = req.params; + const { enabled } = req.body; + + if (typeof enabled !== "boolean") { + res.status(400).json({ error: "'enabled' must be a boolean" }); + return; + } + + const result = await featureFlagCollection.findOneAndUpdate( + { key }, + { $set: { enabled, updatedAt: new Date() } }, + { returnDocument: "after" } + ); + + if (!result) { + res.status(404).json({ error: `Feature flag '${key}' not found` }); + return; + } + + res.json(result); + } catch (error) { + next(error); + } +}); + +// Error handler +app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => { + console.error("Error:", err); + res.status(500).json({ error: err.message || "Internal server error" }); +}); + +async function main(): Promise { + await initializeClients(); + + app.listen(port, () => { + console.log(`API server listening on port ${port}`); + }); +} + +main().catch((error) => { + console.error("Failed to start server:", error); + process.exit(1); +}); diff --git a/apps/api/src/llm-token.ts b/apps/api/src/llm-token.ts new file mode 100644 index 000000000..73149eeb3 --- /dev/null +++ b/apps/api/src/llm-token.ts @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Shared helper for acquiring a GitHub Models API token. + * + * Priority: + * 1. GITHUB_MODELS_API_KEY env var (explicit override) + * 2. TokenManagerClient acquire("github-models") via TOKEN_MANAGER_URL + * 3. GITHUB_TOKEN env var (generic fallback used by TokenManagerClient) + * + * Both llm.ts and prompt-feature-llm.ts use this module instead of + * reading env vars directly. + */ +import { TokenManagerClient } from "shared"; + +let tokenManagerClient: TokenManagerClient | null = null; + +function getTokenManagerClient(): TokenManagerClient | null { + if (tokenManagerClient) return tokenManagerClient; + const url = process.env.TOKEN_MANAGER_URL; + if (!url) return null; + tokenManagerClient = new TokenManagerClient(url); + return tokenManagerClient; +} + +/** + * Returns true if a GitHub Models token is available from any source: + * - GITHUB_MODELS_API_KEY env var + * - TOKEN_MANAGER_URL (token manager with registered github-models tokens) + * - GITHUB_TOKEN env var (generic fallback) + */ +export function isGitHubModelsTokenAvailable(): boolean { + return !!( + process.env.GITHUB_MODELS_API_KEY || + process.env.TOKEN_MANAGER_URL || + process.env.GITHUB_TOKEN + ); +} + +/** + * Acquire a GitHub Models API token. + * + * @throws Error if no token source is available. + */ +export async function acquireGitHubModelsToken(): Promise { + // 1. Explicit env var override + const explicit = process.env.GITHUB_MODELS_API_KEY; + if (explicit) return explicit; + + // 2. Token Manager (handles its own GITHUB_TOKEN fallback internally) + const client = getTokenManagerClient(); + if (client) { + return client.acquireToken("github-models"); + } + + // 3. Bare GITHUB_TOKEN fallback (no token manager) + const fallback = process.env.GITHUB_TOKEN; + if (fallback) return fallback; + + throw new Error( + "No GitHub Models token available: set GITHUB_MODELS_API_KEY, GITHUB_TOKEN, or configure TOKEN_MANAGER_URL" + ); +} diff --git a/packages/api/src/llm.ts b/apps/api/src/llm.ts similarity index 91% rename from packages/api/src/llm.ts rename to apps/api/src/llm.ts index 3f15867be..1ca4683bb 100644 --- a/packages/api/src/llm.ts +++ b/apps/api/src/llm.ts @@ -3,6 +3,7 @@ import ModelClient, { isUnexpected } from "@azure-rest/ai-inference"; import { AzureKeyCredential } from "@azure/core-auth"; +import { isGitHubModelsTokenAvailable, acquireGitHubModelsToken } from "./llm-token.js"; const GITHUB_MODELS_ENDPOINT = "https://models.inference.ai.azure.com"; @@ -44,18 +45,8 @@ export interface GenerateResult { suggestedChildren: string[]; } -let client: ReturnType | null = null; - -function getClient(): ReturnType | null { - if (client) return client; - const token = process.env.GITHUB_MODELS_API_KEY; - if (!token) return null; - client = ModelClient(GITHUB_MODELS_ENDPOINT, new AzureKeyCredential(token)); - return client; -} - export function isLlmAvailable(): boolean { - return !!process.env.GITHUB_MODELS_API_KEY; + return isGitHubModelsTokenAvailable(); } function buildUserMessage(behavior: string, existingCriteria: ExistingCriterion[]): string { @@ -79,10 +70,8 @@ export async function generateCriteriaPrompt( existingCriteria: ExistingCriterion[] = [], model?: string, ): Promise { - const llm = getClient(); - if (!llm) { - throw new Error("LLM not configured: GITHUB_MODELS_API_KEY is not set"); - } + const token = await acquireGitHubModelsToken(); + const llm = ModelClient(GITHUB_MODELS_ENDPOINT, new AzureKeyCredential(token)); const modelName = model || process.env.LLM_MODEL || "gpt-4.1"; const userMessage = buildUserMessage(behavior, existingCriteria); diff --git a/apps/api/src/prompt-feature-llm.ts b/apps/api/src/prompt-feature-llm.ts new file mode 100644 index 000000000..ae9c70f02 --- /dev/null +++ b/apps/api/src/prompt-feature-llm.ts @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import ModelClient, { isUnexpected } from "@azure-rest/ai-inference"; +import { AzureKeyCredential } from "@azure/core-auth"; +import { PromptFeatureConfig, PromptFeatureResult, SuggestedPromptFeature } from "shared"; +import { isGitHubModelsTokenAvailable, acquireGitHubModelsToken } from "./llm-token.js"; + +const GITHUB_MODELS_ENDPOINT = "https://models.inference.ai.azure.com"; + +// --------------------------------------------------------------------------- +// Generate prompt feature prompt (mirrors criteria generateCriteriaPrompt) +// --------------------------------------------------------------------------- + +const GENERATE_SYSTEM_PROMPT = `You are an expert at writing detection prompts for AI coding agent benchmark task analysis. + +Given a natural-language description of a characteristic to detect in a task prompt (the instructions given to a coding agent), you must: + +1. Write a concise detection prompt (1-3 sentences) that an LLM will use to decide whether a task prompt exhibits that characteristic. The prompt should be specific about what phrases, patterns, or requirements to look for. Keep it factual and objective. + +2. Suggest a short, descriptive snake_case identifier for this prompt feature. The ID must: + - Start with "asks_for_" or a similar verb prefix + - Contain only lowercase letters, digits, and underscores + - Be concise but descriptive (e.g., asks_for_testing, asks_for_api, asks_for_docker) + +3. Suggest **parent dependencies** ā€” existing prompt features that should logically be detected BEFORE this one can be evaluated. For example, if the new feature checks for "Azure Functions", it likely depends on "asks_for_azure" being detected first. Only suggest IDs from the provided existing list. + +4. Suggest **children dependents** ā€” existing prompt features that should logically depend on this new feature. Only suggest IDs from the provided existing list. + +Here are examples of good detection prompts: +- "The task prompt asks the agent to use, deploy to, or configure cloud infrastructure or services (Azure, AWS, GCP)." +- "The task prompt asks the agent to create or modify a REST API, web service, HTTP server, or backend endpoint." +- "The task prompt asks the agent to write tests, add test coverage, or set up a testing framework." + +Respond with ONLY a JSON object in this exact format (no markdown, no code fences): +{"prompt": "your detection prompt here", "suggestedId": "asks_for_something", "suggestedParents": ["existing_id_1"], "suggestedChildren": ["existing_id_2"]} + +If no parents or children are appropriate, use empty arrays.`; + +export interface ExistingPromptFeature { + id: string; + prompt: string; + dependsOn?: string[]; +} + +export interface GeneratePromptFeatureResult { + prompt: string; + suggestedId: string; + suggestedParents: string[]; + suggestedChildren: string[]; +} + +export function isLlmAvailable(): boolean { + return isGitHubModelsTokenAvailable(); +} + +function buildGenerateUserMessage(behavior: string, existing: ExistingPromptFeature[]): string { + const parts: string[] = []; + + if (existing.length > 0) { + parts.push("EXISTING PROMPT FEATURES (use only these IDs for parent/children suggestions):"); + for (const f of existing) { + const deps = f.dependsOn?.length ? ` [parents: ${f.dependsOn.join(", ")}]` : ""; + parts.push(`- ${f.id}: ${f.prompt}${deps}`); + } + parts.push(""); + } + + parts.push(`NEW PROMPT FEATURE TO CREATE:\n${behavior}`); + return parts.join("\n"); +} + +export async function generatePromptFeaturePrompt( + behavior: string, + existingFeatures: ExistingPromptFeature[] = [], + model?: string, +): Promise { + const token = await acquireGitHubModelsToken(); + const llm = ModelClient(GITHUB_MODELS_ENDPOINT, new AzureKeyCredential(token)); + + const modelName = model || process.env.LLM_MODEL || "gpt-4.1"; + const userMessage = buildGenerateUserMessage(behavior, existingFeatures); + + const response = await llm.path("/chat/completions").post({ + body: { + messages: [ + { role: "system", content: GENERATE_SYSTEM_PROMPT }, + { role: "user", content: userMessage }, + ], + model: modelName, + temperature: 0.3, + max_tokens: 512, + }, + }); + + if (isUnexpected(response)) { + const errBody = response.body as any; + throw new Error( + `LLM request failed: ${errBody?.error?.message || response.status}`, + ); + } + + const content = response.body.choices?.[0]?.message?.content; + if (!content) { + throw new Error("LLM returned empty response"); + } + + const cleaned = content.replace(/```json\s*|```\s*/g, "").trim(); + try { + const parsed = JSON.parse(cleaned); + if (!parsed.prompt || !parsed.suggestedId) { + throw new Error("Missing required fields"); + } + + const sanitizedId = parsed.suggestedId + .toLowerCase() + .replace(/[^a-z0-9_]/g, "_") + .replace(/^[^a-z]+/, "") + .replace(/_+/g, "_") + .replace(/_$/, ""); + + const existingIds = new Set(existingFeatures.map((f) => f.id)); + const suggestedParents = Array.isArray(parsed.suggestedParents) + ? parsed.suggestedParents.filter((pid: string) => existingIds.has(pid)) + : []; + const suggestedChildren = Array.isArray(parsed.suggestedChildren) + ? parsed.suggestedChildren.filter((cid: string) => existingIds.has(cid)) + : []; + + return { + prompt: parsed.prompt.trim(), + suggestedId: sanitizedId || "asks_for_something", + suggestedParents, + suggestedChildren, + }; + } catch { + throw new Error(`Failed to parse LLM response as JSON: ${cleaned}`); + } +} + +// --------------------------------------------------------------------------- +// Extract prompt features from a task prompt +// --------------------------------------------------------------------------- + +const EXTRACT_SYSTEM_PROMPT = `You are an expert at analyzing task prompts for AI coding agent benchmarks. + +Given a task prompt (the instructions given to a coding agent) and optionally a list of prompt features to detect, you must: + +1. If existing features are provided, determine which are present in the task prompt. A feature is "detected" if the task prompt explicitly or implicitly asks for, mentions, or requires the characteristic described by that feature. + +2. Suggest NEW features that the task prompt exhibits but that are NOT covered by any of the existing features. Only suggest features that represent clearly distinct, meaningful characteristics. Each suggestion needs: + - suggestedId: a snake_case identifier starting with a verb prefix (e.g., asks_for_X, requires_X, uses_X) + - behavior: a short natural-language description of the characteristic + - prompt: a concise detection prompt (1-3 sentences) for an LLM to detect this characteristic in other task prompts + +Respond with ONLY a JSON object in this exact format (no markdown, no code fences): +{"results": [{"featureId": "feature_id_here", "detected": true}, {"featureId": "other_feature", "detected": false}], "suggestedFeatures": [{"suggestedId": "asks_for_something", "behavior": "short description", "prompt": "detection prompt"}]} + +If no existing features are provided, "results" should be an empty array ā€” focus on suggesting new features instead. Include ALL provided features in "results" if any are given. Be precise ā€” only mark a feature as detected if the task prompt clearly relates to it. If no new features should be suggested, use an empty array for "suggestedFeatures".`; + +function buildExtractUserMessage(taskText: string, features: PromptFeatureConfig[]): string { + const parts: string[] = []; + + if (features.length > 0) { + parts.push("PROMPT FEATURES TO DETECT:"); + for (const f of features) { + parts.push(`- ${f.id}: ${f.prompt}`); + } + } else { + parts.push("PROMPT FEATURES TO DETECT:\n(none defined yet \u2014 suggest new features based on the task prompt)"); + } + parts.push(""); + parts.push(`TASK PROMPT TO ANALYZE:\n${taskText}`); + + return parts.join("\n"); +} + +export interface ExtractionResult { + results: PromptFeatureResult[]; + suggestedFeatures: SuggestedPromptFeature[]; +} + +export async function extractPromptFeatures( + taskText: string, + features: PromptFeatureConfig[], + model?: string, +): Promise { + const token = await acquireGitHubModelsToken(); + const llm = ModelClient(GITHUB_MODELS_ENDPOINT, new AzureKeyCredential(token)); + + const modelName = model || process.env.LLM_MODEL || "gpt-4.1"; + const userMessage = buildExtractUserMessage(taskText, features); + + const response = await llm.path("/chat/completions").post({ + body: { + messages: [ + { role: "system", content: EXTRACT_SYSTEM_PROMPT }, + { role: "user", content: userMessage }, + ], + model: modelName, + temperature: 0.1, // Lower temperature for more deterministic detection + max_tokens: 2048, + }, + }); + + if (isUnexpected(response)) { + const errBody = response.body as any; + throw new Error( + `LLM extraction failed: ${errBody?.error?.message || response.status}`, + ); + } + + const content = response.body.choices?.[0]?.message?.content; + if (!content) { + throw new Error("LLM returned empty response"); + } + + const cleaned = content.replace(/```json\s*|```\s*/g, "").trim(); + try { + const parsed = JSON.parse(cleaned); + + // Support both old format (plain array) and new format ({results, suggestedFeatures}) + const rawResults: Array<{ featureId: string; detected: boolean }> = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed.results) + ? parsed.results + : []; + + // Build a map of LLM results + const llmResults = new Map(); + for (const item of rawResults) { + if (item.featureId && typeof item.detected === "boolean") { + llmResults.set(item.featureId, item.detected); + } + } + + // Build complete results for all features (mark as not evaluated if + // an ancestor was not detected ā€” skip descendant evaluation) + const results: PromptFeatureResult[] = features.map(f => ({ + featureId: f.id, + detected: llmResults.get(f.id) ?? false, + evaluated: llmResults.has(f.id), + })); + + // Parse suggested features (sanitize IDs) + const rawSuggestions: SuggestedPromptFeature[] = []; + if (!Array.isArray(parsed) && Array.isArray(parsed.suggestedFeatures)) { + for (const s of parsed.suggestedFeatures) { + if (s.suggestedId && s.behavior && s.prompt) { + const sanitizedId = String(s.suggestedId) + .toLowerCase() + .replace(/[^a-z0-9_]/g, "_") + .replace(/^[^a-z]+/, "") + .replace(/_+/g, "_") + .replace(/_$/, ""); + rawSuggestions.push({ + suggestedId: sanitizedId || "asks_for_something", + behavior: String(s.behavior).trim(), + prompt: String(s.prompt).trim(), + }); + } + } + } + + return { results, suggestedFeatures: rawSuggestions }; + } catch { + throw new Error(`Failed to parse LLM extraction response as JSON: ${cleaned}`); + } +} diff --git a/packages/api/tsconfig.json b/apps/api/tsconfig.json similarity index 100% rename from packages/api/tsconfig.json rename to apps/api/tsconfig.json diff --git a/packages/cli/package.json b/apps/cli/package.json similarity index 90% rename from packages/cli/package.json rename to apps/cli/package.json index 734b266b5..8767b75c8 100644 --- a/packages/cli/package.json +++ b/apps/cli/package.json @@ -11,7 +11,9 @@ "start": "node dist/index.js" }, "dependencies": { + "cli-table3": "^0.6.5", "commander": "^14.0.0", + "csv-stringify": "^6.6.0", "dotenv": "^16.6.1", "eventsource": "^2.0.2", "ink": "^6.0.0", diff --git a/packages/cli/src/components/DemoApp.tsx b/apps/cli/src/components/DemoApp.tsx similarity index 100% rename from packages/cli/src/components/DemoApp.tsx rename to apps/cli/src/components/DemoApp.tsx diff --git a/packages/cli/src/components/FilterBar.tsx b/apps/cli/src/components/FilterBar.tsx similarity index 100% rename from packages/cli/src/components/FilterBar.tsx rename to apps/cli/src/components/FilterBar.tsx diff --git a/packages/cli/src/components/LogStream.tsx b/apps/cli/src/components/LogStream.tsx similarity index 100% rename from packages/cli/src/components/LogStream.tsx rename to apps/cli/src/components/LogStream.tsx diff --git a/packages/cli/src/components/ProgressHeader.tsx b/apps/cli/src/components/ProgressHeader.tsx similarity index 100% rename from packages/cli/src/components/ProgressHeader.tsx rename to apps/cli/src/components/ProgressHeader.tsx diff --git a/packages/cli/src/config-loader.ts b/apps/cli/src/config-loader.ts similarity index 100% rename from packages/cli/src/config-loader.ts rename to apps/cli/src/config-loader.ts diff --git a/packages/cli/src/hooks/useRequestSubmit.ts b/apps/cli/src/hooks/useRequestSubmit.ts similarity index 100% rename from packages/cli/src/hooks/useRequestSubmit.ts rename to apps/cli/src/hooks/useRequestSubmit.ts diff --git a/packages/cli/src/hooks/useSSELogs.ts b/apps/cli/src/hooks/useSSELogs.ts similarity index 100% rename from packages/cli/src/hooks/useSSELogs.ts rename to apps/cli/src/hooks/useSSELogs.ts diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts new file mode 100644 index 000000000..0e5cd46d5 --- /dev/null +++ b/apps/cli/src/index.ts @@ -0,0 +1,3844 @@ +#!/usr/bin/env node +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import dotenv from "dotenv"; +import { Command } from "commander"; +import EventSource from "eventsource"; +import { execSync } from "child_process"; +import { mkdtempSync, mkdirSync, createWriteStream, rmSync, readFileSync, readdirSync, existsSync, statSync } from "fs"; +import { tmpdir } from "os"; +import { join, resolve, dirname, basename, extname } from "path"; +import { pipeline } from "stream/promises"; +import { Readable } from "stream"; +import { parse as yamlParse, parseAllDocuments, stringify as yamlStringify } from "yaml"; +import React from "react"; +import { render } from "ink"; +import { DemoApp } from "./components/DemoApp.js"; +import { resolveScenarioAndPersona } from "./config-loader.js"; +import { configureHelp, generateOutputFormatsHelp } from "./utils/helpFormatter.js"; +import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon, styleText } from "./utils/style.js"; +import { formatData, isMachineReadable } from "./utils/formatters.js"; +import type { OutputFormat, DisplayField } from "./utils/types.js"; +import { runGetAction } from "./run-get-action.js"; + +dotenv.config(); + +/** Strip trailing slashes from a URL to avoid double-slash issues when appending paths */ +const normalizeUrl = (url: string): string => url.replace(/\/+$/, ''); + +function printFollowUpCommands(id: string): void { + console.log(`\n${label('Run ID:')} ${value(id)}`); + console.log(`\n${label('Next steps:')}`); + console.log(` ${dimTimestamp('Get details:')} pnpm cli run get -i ${id}`); + console.log(` ${dimTimestamp('Check status:')} pnpm cli run status -i ${id}`); + console.log(` ${dimTimestamp('Stream logs:')} pnpm cli run logs -i ${id}`); + console.log(` ${dimTimestamp('Download:')} pnpm cli run download -i ${id}`); + console.log(` ${dimTimestamp('List all runs:')} pnpm cli run list`); +} + +const program = new Command(); + +const DEFAULT_WORKERS = [ + "coder-acp-claude-code", + "coder-acp-copilot", + "coder-vscode-web", +]; + +// Output format definitions with descriptions and categories +const OUTPUT_FORMATS = { + table: { section: 'Human-readable formats', description: 'Formatted table with borders (default for lists)' }, + tsv: { section: 'Machine-readable formats', description: 'Tab-separated values for Unix tools (cut, awk, grep, xargs)' }, + json: { section: 'Machine-readable formats', description: 'JSON format for programmatic access and AI agents' }, + yaml: { section: 'Machine-readable formats', description: 'YAML format for human-friendly structured data' }, +} as const; + +/** + * Add the standard `-o, --output ` option to a command. + * @param cmd - The Commander command to add the option to. + * @param extra - Additional format names beyond the defaults (table, tsv, json, yaml). + * @returns The command (for chaining). + */ +function withOutputOption(cmd: Command, extra?: string[]): Command { + const formats = ['table', 'tsv', 'json', 'yaml', ...(extra ?? [])]; + return cmd.option("-o, --output ", `Output format: ${formats.join(', ')}`, "table"); +} + +program + .name("scope-mt") + .description("Scope MT ā€” AI coding agent benchmarking CLI") + .version("1.0.0") + .action(() => { + program.help(); + }) + .addHelpText('after', generateOutputFormatsHelp(OUTPUT_FORMATS)); + +configureHelp(program); + +const run = program + .command("run") + .description("Submit, monitor, and manage benchmark runs") + .action(() => { + run.help(); + }); + +configureHelp(run); + +run + .command("submit") + .description("Submit a request to a worker and stream logs") + .option("-s, --scenario ", "Path to scenario YAML file (provides task + criteria)") + .option("-p, --persona ", "Path to persona YAML file (provides judge personality)") + .option("-t, --traits ", "Path to traits.yaml (default: config/traits.yaml next to persona)") + .option("-m, --message ", "Message/task to process (overrides scenario task)") + .option("-w, --worker ", "Worker to use (coder-acp-claude-code, coder-acp-copilot, coder-vscode-web)", "coder-acp-copilot") + .option("-c, --criteria ", "Evaluation criteria (overrides scenario criteria)") + .option("--max-iterations ", "Max judge iterations for multi-turn mode", parseInt) + .option("--model ", "Model to use for the coding agent") + .option("--mcp-servers ", "MCP server slugs to use for this run") + .option("--skills ", "Skill slugs to use for this run (e.g. vercel-labs/agent-skills/my-skill)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("--no-stream", "Don't stream logs, just submit") + .action(async (options) => { + const { scenario, persona, traits, worker, url, stream, maxIterations, model, mcpServers: mcpServerSlugs, skills: skillSlugs } = options; + + try { + // Resolve scenario + persona YAML if provided + let message = options.message; + let criteria = options.criteria; + let scenarioVersion: 'v1' | 'v2' | undefined; + let personaInstructions: string | undefined; + let personaObj: Record | undefined; + + if (scenario) { + const resolved = resolveScenarioAndPersona(scenario, persona, traits); + // Scenario provides task and criteria (CLI flags override) + if (!message) message = resolved.task; + if (!criteria || criteria.length === 0) criteria = resolved.criteria; + scenarioVersion = resolved.version; + personaInstructions = resolved.personaInstructions; + personaObj = resolved.persona; + + console.log(`${label('Scenario:')} ${value(scenario)} (version: ${value(scenarioVersion || 'v1')})`); + if (persona) console.log(`${label('Persona:')} ${value(persona)}`); + console.log(`${label('Task:')} ${message.substring(0, 100)}${message.length > 100 ? '...' : ''}`); + console.log(`${label('Criteria:')} ${value(String(criteria.length))} items`); + console.log(); + } + + if (!message) { + console.error(errorText("Error: --message or --scenario is required")); + process.exit(1); + } + + // Build request body ā€” scenario is the source of truth + const body: Record = { + scenario: { + ...(scenarioVersion ? { version: scenarioVersion } : {}), + task: message, + criteria: criteria || [], + }, + }; + if (maxIterations) { + body.maxIterations = maxIterations; + } + if (model) { + body.model = model; + } + if (personaInstructions) { + body.personaInstructions = personaInstructions; + } + if (personaObj) { + body.persona = personaObj; + } + if (mcpServerSlugs && mcpServerSlugs.length > 0) { + body.mcpServers = mcpServerSlugs; + } + if (skillSlugs && skillSlugs.length > 0) { + body.skills = skillSlugs; + } + + const response = await fetch(`${normalizeUrl(url)}/api/v1/requests?worker=${worker}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error); + process.exit(1); + } + + const result = await response.json(); + console.log(`${successText('Request submitted:')} ${value(result.id)}`); + console.log(`${label('Worker:')} ${value(result.workerType)}`); + if (result.model) console.log(`${label('Model:')} ${value(result.model)}`); + console.log(`${label('Mode:')} ${value(result.mode || 'one-shot')}`); + console.log(`${label('Status:')} ${value(result.status)}`); + + if (!stream) { + printFollowUpCommands(result.id); + return; + } + + // Stream logs + console.log(`\n${banner('--- Streaming logs ---')}\n`); + + const eventSource = new EventSource(`${normalizeUrl(url)}/api/v1/requests/${result.id}/logs`); + + eventSource.onmessage = (event) => { + try { + const log = JSON.parse(event.data); + const timestamp = new Date(log.timestamp).toLocaleTimeString(); + const src = log.source ? `[${log.source}] ` : ''; + const iter = log.data?.iteration != null ? `[iter ${log.data.iteration}] ` : ''; + + // Detect special criterion/DAG log events and render them with status icons + if (log.data?.type === "criterion_result") { + const d = log.data; + const icon = criterionIcon(d.evaluated, d.passed); + console.log(`[${dimTimestamp(timestamp)}] [${colorLevel(log.level)}] ${src}${iter} ${icon} ${d.criterionId}`); + if (d.feedback && !d.passed && d.evaluated) { + console.log(` ${styleText('gray', d.feedback.substring(0, 120))}`); + } + return; + } + + if (log.data?.type === "criteria_dag_status") { + console.log(`[${dimTimestamp(timestamp)}] [${colorLevel(log.level)}] ${src}${iter}${log.message}`); + const results = log.data.results as Array<{ criterionId: string; passed: boolean; evaluated: boolean; feedback: string }>; + if (results) { + for (const r of results) { + const icon = criterionIcon(r.evaluated, r.passed); + console.log(` ${icon} ${r.criterionId}`); + } + } + return; + } + + console.log(`[${dimTimestamp(timestamp)}] [${colorLevel(log.level)}] ${src}${iter}${log.message}`); + if (log.data) { + // Filter out keys already rendered in the log line prefix + const { iteration: _iter, type: _type, ...rest } = log.data; + if (Object.keys(rest).length > 0) { + const dataStr = JSON.stringify(rest, null, 2) + .split("\n") + .map((line) => ` ${line}`) + .join("\n"); + console.log(dataStr); + } + } + } catch { + console.log(event.data); + } + }; + + eventSource.addEventListener("done", (event) => { + try { + const data = JSON.parse((event as MessageEvent).data); + console.log(`\n${successText(`--- Processing ${data.status} ---`)}`); + } catch { + console.log(`\n${successText('--- Done ---')}`); + } + printFollowUpCommands(result.id); + eventSource.close(); + process.exit(0); + }); + + eventSource.addEventListener("error", () => { + console.error(`\n${errorText('--- Connection error ---')}`); + eventSource.close(); + process.exit(1); + }); + + eventSource.addEventListener("timeout", () => { + console.log(`\n${warnBanner('--- Stream timeout ---')}`); + eventSource.close(); + process.exit(0); + }); + + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +run + .command("status") + .description("Get status of a request") + .requiredOption("-i, --id ", "Request ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + const { id } = options; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/requests/${id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error); + process.exit(1); + } + + const request = await response.json(); + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'workerType', label: 'Worker' }, + { key: 'status', label: 'Status' }, + { key: 'mode', label: 'Mode' }, + { key: 'createdAt', label: 'Created' }, + { key: 'completedAt', label: 'Completed' }, + ]; + console.log(formatData([request], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(request.id)}`); + console.log(`${label('Worker:')} ${value(request.workerType)}`); + console.log(`${label('Status:')} ${value(request.status)}`); + if (request.mode) console.log(`${label('Mode:')} ${value(request.mode)}`); + if (request.createdAt) console.log(`${label('Created:')} ${value(request.createdAt)}`); + if (request.completedAt) console.log(`${label('Completed:')} ${value(request.completedAt)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +run + .command("get") + .description("Get full details of a run") + .requiredOption("-i, --id ", "Run ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + await runGetAction({ id: options.id, url: options.url, output: options.output }); + }); + +run + .command("logs") + .description("Stream logs for a request") + .requiredOption("-i, --id ", "Request ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("--from-start", "Include historical logs from start") + .action(async (options) => { + const { id } = options; + const url = options.fromStart + ? `${normalizeUrl(options.url)}/api/v1/requests/${id}/logs?fromStart=true` + : `${normalizeUrl(options.url)}/api/v1/requests/${id}/logs`; + + const eventSource = new EventSource(url); + + eventSource.onmessage = (event) => { + try { + const log = JSON.parse(event.data); + const timestamp = new Date(log.timestamp).toLocaleTimeString(); + const src = log.source ? `[${log.source}] ` : ''; + const iter = log.data?.iteration != null ? `[iter ${log.data.iteration}] ` : ''; + + if (log.data?.type === "criterion_result") { + const d = log.data; + const icon = criterionIcon(d.evaluated, d.passed); + console.log(`[${dimTimestamp(timestamp)}] [${colorLevel(log.level)}] ${src}${iter} ${icon} ${d.criterionId}`); + return; + } + + if (log.data?.type === "criteria_dag_status") { + console.log(`[${dimTimestamp(timestamp)}] [${colorLevel(log.level)}] ${src}${iter}${log.message}`); + const results = log.data.results as Array<{ criterionId: string; passed: boolean; evaluated: boolean }>; + if (results) { + for (const r of results) { + const icon = criterionIcon(r.evaluated, r.passed); + console.log(` ${icon} ${r.criterionId}`); + } + } + return; + } + + console.log(`[${dimTimestamp(timestamp)}] [${colorLevel(log.level)}] ${src}${iter}${log.message}`); + } catch { + console.log(event.data); + } + }; + + eventSource.addEventListener("done", () => { + console.log(`\n${successText('--- Done ---')}`); + eventSource.close(); + process.exit(0); + }); + + eventSource.addEventListener("error", () => { + console.error(`\n${errorText('--- Connection error ---')}`); + eventSource.close(); + process.exit(1); + }); + }); + +withOutputOption( +run + .command("list") + .description("List all requests") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("-w, --worker ", "Filter by worker") + .option("--include-deleted", "Include soft-deleted runs") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + let url = `${normalizeUrl(options.url)}/api/v1/requests`; + const params = new URLSearchParams(); + if (options.worker) { + params.set("worker", options.worker); + } + if (options.includeDeleted) { + params.set("includeDeleted", "true"); + } + const qs = params.toString(); + if (qs) url += `?${qs}`; + + const response = await fetch(url); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error); + process.exit(1); + } + + const requests = await response.json(); + if (Array.isArray(requests) && requests.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner('No requests found.')); + return; + } + if (Array.isArray(requests)) { + if (!isMachineReadable(format)) { + console.log(label(`Found ${requests.length} request(s):\n`)); + } + + const displayFields: DisplayField[] = [ + { key: 'id', label: 'ID', + formatter: (req: any) => req.id ?? '(no id)', + tableFormatter: (req: any) => value(req.id ?? '(no id)'), + }, + { key: 'workerType', label: 'Worker', + formatter: (req: any) => req.workerType ?? 'unknown', + }, + { key: 'status', label: 'Status', + formatter: (req: any) => req.status ?? 'unknown', + tableFormatter: (req: any) => { + const s = req.status ?? 'unknown'; + return s === 'completed' ? successText(s) : s === 'failed' ? errorText(s) : value(s); + }, + }, + ]; + + console.log(formatData(requests, displayFields, format)); + } else { + console.log(JSON.stringify(requests, null, 2)); + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +run + .command("demo") + .description("Run concurrent requests to all coders with a live TUI dashboard") + .requiredOption("-m, --message ", "Message/prompt to send to all coders") + .option("-c, --count ", "Number of requests to send to each coder", "1") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("-w, --workers ", "Comma-separated list of workers", DEFAULT_WORKERS.join(",")) + .action((options) => { + const { message, count, url, workers: workersStr } = options; + const workersList = workersStr.split(",").map((w: string) => w.trim()); + const countNum = parseInt(count, 10); + + if (isNaN(countNum) || countNum < 1) { + console.error(errorText("Error: count must be a positive integer")); + process.exit(1); + } + + console.clear(); + const { waitUntilExit } = render( + React.createElement(DemoApp, { + apiUrl: url, + message, + count: countNum, + workers: workersList, + }) + ); + waitUntilExit().catch(() => {}); + }); + +run + .command("delete") + .description("Soft-delete a run (can still be listed with --include-deleted)") + .requiredOption("-i, --id ", "Request ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + const { id, url } = options; + try { + const response = await fetch(`${normalizeUrl(url)}/api/v1/requests/${id}`, { method: "DELETE" }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + console.log(`${successText('Deleted run')} ${value(id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +run + .command("download") + .description("Download all artifacts of a run (workspaces + run document)") + .requiredOption("-i, --id ", "Request ID") + .option("-o, --output ", "Output file path (default: .tar.gz)") + .option("-e, --extract", "Extract the archive after downloading") + .option("-d, --dir ", "Extraction directory (implies --extract)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + const { id, url } = options; + const shouldExtract = options.extract || !!options.dir; + const downloadDir = process.env.SCOPE_MT_DOWNLOAD_OUTPUT_DIR; + const defaultFile = downloadDir ? join(downloadDir, `${id}.tar.gz`) : `${id}.tar.gz`; + const outputFile = options.output || defaultFile; + + try { + // Step 1: Fetch request document (lightweight ā€” for metadata display) + console.log(`${label('Fetching run')} ${value(id)}...`); + const response = await fetch(`${normalizeUrl(url)}/api/v1/requests/${id}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error); + process.exit(1); + } + const request = await response.json(); + + if (!request.turns || request.turns.length === 0) { + console.error(errorText("Error: No iterations found for this run")); + process.exit(1); + } + + console.log(`${label('Status:')} ${value(request.status)}`); + console.log(`${label('Worker:')} ${value(request.workerType)}`); + console.log(`${label('Iterations:')} ${value(String(request.turns.length))}`); + console.log(); + + // Step 2: Download the full archive from the server + console.log(`${label('Downloading archive')}...`); + const archiveResp = await fetch(`${normalizeUrl(url)}/api/v1/requests/${id}/archive`); + if (!archiveResp.ok || !archiveResp.body) { + const error = await archiveResp.json().catch(() => ({ error: archiveResp.statusText })); + console.error(errorText("Error downloading archive:"), error); + process.exit(1); + } + + const outputPath = resolve(outputFile); + mkdirSync(dirname(outputPath), { recursive: true }); + const fileStream = createWriteStream(outputPath); + await pipeline(Readable.fromWeb(archiveResp.body as any), fileStream); + console.log(`${successText('Archive:')} ${value(outputPath)}`); + + // Step 3: Optionally extract + if (shouldExtract) { + const extractDir = options.dir || downloadDir || "."; + mkdirSync(extractDir, { recursive: true }); + execSync(`tar xzf "${outputPath}" -C "${extractDir}"`, { stdio: "pipe" }); + + // Extract nested iteration-*.tar.gz files into iteration-N/ directories + const runExtractDir = join(extractDir, id); + if (existsSync(runExtractDir)) { + const nestedArchives = readdirSync(runExtractDir).filter(f => f.startsWith("iteration-") && f.endsWith(".tar.gz")); + for (const archive of nestedArchives) { + const iterName = archive.replace(".tar.gz", ""); + const iterDir = join(runExtractDir, iterName); + mkdirSync(iterDir, { recursive: true }); + execSync(`tar xzf "${join(runExtractDir, archive)}" -C "${iterDir}"`, { stdio: "pipe" }); + rmSync(join(runExtractDir, archive), { force: true }); + } + } + + rmSync(outputPath, { force: true }); + console.log(`${successText('Extracted to:')} ${value(resolve(extractDir, id))}`); + } + + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +run + .command("upload") + .description("Upload a run archive to the API (previously downloaded via 'run download')") + .argument("", "Path to .tar.gz archive or extracted directory") + .option("--dry-run", "Preview what would be uploaded without sending") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (inputPath: string, options) => { + const { url, dryRun } = options; + + try { + const resolvedPath = resolve(inputPath); + + // Check if path exists + if (!existsSync(resolvedPath)) { + console.error(errorText(`Path not found: ${resolvedPath}`)); + process.exit(1); + } + + const stats = statSync(resolvedPath); + let archivePath: string; + let tempDir: string | undefined; + + if (stats.isDirectory()) { + // Directory: validate run.yaml exists, then create tar.gz + const runYamlPath = join(resolvedPath, "run.yaml"); + if (!existsSync(runYamlPath)) { + console.error(errorText(`Invalid directory: run.yaml not found at ${runYamlPath}`)); + process.exit(1); + } + + // Create temporary tar.gz archive + tempDir = mkdtempSync(join(tmpdir(), "scope-mt-upload-")); + archivePath = join(tempDir, `${basename(resolvedPath)}.tar.gz`); + + console.log(`${label('Creating archive from')} ${value(resolvedPath)}...`); + execSync(`tar czf "${archivePath}" -C "${dirname(resolvedPath)}" "${basename(resolvedPath)}"`, { stdio: "pipe" }); + } else if (resolvedPath.endsWith(".tar.gz")) { + // Archive file: use as-is + archivePath = resolvedPath; + } else { + console.error(errorText("Input must be a .tar.gz archive or a directory")); + process.exit(1); + } + + const archiveStats = statSync(archivePath); + const archiveSizeKB = Math.round(archiveStats.size / 1024); + + console.log(`${label('Archive:')} ${value(archivePath)}`); + console.log(`${label('Size:')} ${value(`${archiveSizeKB} KB`)}`); + console.log(); + + if (dryRun) { + console.log(warnBanner("Dry run - no data will be uploaded")); + console.log(`Would upload to: ${normalizeUrl(url)}/api/v1/runs/upload`); + + // Cleanup temp dir if created + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }); + } + return; + } + + // Upload archive + console.log(`${label('Uploading to')} ${value(normalizeUrl(url))}...`); + + const formData = new FormData(); + const archiveBuffer = readFileSync(archivePath); + const blob = new Blob([archiveBuffer], { type: "application/gzip" }); + formData.append("archive", blob, basename(archivePath)); + + const response = await fetch(`${normalizeUrl(url)}/api/v1/runs/upload`, { + method: "POST", + body: formData, + }); + + // Cleanup temp dir if created + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }); + } + + if (!response.ok) { + const errorData = await response.json().catch(() => ({ error: response.statusText })); + + if (response.status === 409) { + console.error(errorText(`Error: ${errorData.error || "Run already exists"}`)); + } else if (response.status === 400) { + console.error(errorText(`Error: ${errorData.error || "Invalid archive"}`)); + } else { + console.error(errorText(`Error: ${errorData.error || response.statusText}`)); + } + process.exit(1); + } + + const result = await response.json(); + console.log(); + console.log(successText("Run uploaded successfully!")); + console.log(`${label('Run ID:')} ${value(result.id)}`); + console.log(`${label('Status:')} ${value(result.status)}`); + console.log(`${label('Iterations:')} ${value(String(result.iterations))}`); + + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Criteria management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const criteria = program + .command("criteria") + .description("Manage evaluation criteria (CRUD, import, graph)") + .action(() => { + criteria.help(); + }); + +configureHelp(criteria); + +withOutputOption( +criteria + .command("list") + .description("List all criteria") + .option("-q, --query ", "Filter by ID or prompt text") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = new URLSearchParams(); + if (options.query) params.set("q", options.query); + const qs = params.toString(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria${qs ? `?${qs}` : ""}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const items = await response.json() as Array<{ id: string; prompt: string; dependsOn?: string[] }>; + if (items.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No criteria found.")); + return; + } + + if (!isMachineReadable(format)) { + console.log(label(`Found ${items.length} criteria:\n`)); + } + + const displayFields: DisplayField[] = [ + { key: 'id', label: 'ID', + tableFormatter: (c: any) => value(c.id), + }, + { key: 'dependsOn', label: 'Deps', formatter: (c: any) => String((c.dependsOn ?? []).length) }, + { key: 'prompt', label: 'Prompt', formatter: (c: any) => { + const prompt = c.prompt.replace(/\n/g, ' '); + return prompt.length > 60 ? prompt.substring(0, 60) + 'ā€¦' : prompt; + }, tableFormatter: (c: any) => { + const prompt = c.prompt.replace(/\n/g, ' '); + const truncated = prompt.length > 60 ? prompt.substring(0, 60) + 'ā€¦' : prompt; + return dimTimestamp(truncated); + }}, + ]; + + console.log(formatData(items, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +criteria + .command("get") + .description("Get details of a single criterion") + .requiredOption("-i, --id ", "Criterion ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria/${options.id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const c = await response.json() as { + id: string; prompt: string; dependsOn?: string[]; + dependents: string[]; createdAt: string; updatedAt?: string; + }; + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'prompt', label: 'Prompt' }, + { key: 'dependsOn', label: 'Depends On', formatter: (item: any) => (item.dependsOn ?? []).join(', ') || '(none)' }, + { key: 'dependents', label: 'Dependents', formatter: (item: any) => (item.dependents ?? []).join(', ') || '(none)' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([c], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(c.id)}`); + console.log(`${label('Prompt:')}`); + for (const line of c.prompt.trim().split('\n')) { + console.log(` ${line}`); + } + if ((c.dependsOn ?? []).length > 0) { + console.log(`${label('Depends on:')} ${c.dependsOn!.map(d => value(d)).join(', ')}`); + } else { + console.log(`${label('Depends on:')} ${dimTimestamp('(none ā€” root criterion)')}`); + } + if (c.dependents.length > 0) { + console.log(`${label('Dependents:')} ${c.dependents.map(d => value(d)).join(', ')}`); + } + console.log(`${label('Created:')} ${value(c.createdAt)}`); + if (c.updatedAt) console.log(`${label('Updated:')} ${value(c.updatedAt)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +criteria + .command("create") + .description("Create a new criterion") + .requiredOption("--id ", "Criterion ID (lowercase snake_case)") + .requiredOption("--prompt ", "Evaluation prompt for the judge") + .option("-d, --depends-on ", "IDs of parent criteria") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = { + id: options.id, + prompt: options.prompt, + }; + if (options.dependsOn && options.dependsOn.length > 0) { + body.dependsOn = options.dependsOn; + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const created = await response.json(); + console.log(`${successText('Created criterion')} ${value(created.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +criteria + .command("update") + .description("Update an existing criterion") + .requiredOption("-i, --id ", "Criterion ID") + .option("--prompt ", "New evaluation prompt") + .option("-d, --depends-on ", "New parent criteria IDs (replaces all)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = {}; + if (options.prompt !== undefined) body.prompt = options.prompt; + if (options.dependsOn !== undefined) body.dependsOn = options.dependsOn; + + if (Object.keys(body).length === 0) { + console.error(errorText("Error: provide --prompt and/or --depends-on")); + process.exit(1); + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria/${options.id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + console.log(`${successText('Updated criterion')} ${value(options.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +criteria + .command("delete") + .description("Delete a criterion (soft-delete; fails if other criteria depend on it)") + .requiredOption("-i, --id ", "Criterion ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria/${options.id}`, { + method: "DELETE", + }); + + if (!response.ok) { + const error = await response.json(); + if (error.dependents) { + console.error(errorText(`Cannot delete '${options.id}': depended on by ${error.dependents.join(', ')}`)); + } else { + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + } + process.exit(1); + } + + console.log(`${successText('Deleted criterion')} ${value(options.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +criteria + .command("graph") + .description("Display the criteria dependency graph as ASCII") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria/graph`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const graph = await response.json() as { + nodes: Array<{ id: string; prompt: string; dependsOn: string[] }>; + edges: Array<{ source: string; target: string }>; + }; + + if (isMachineReadable(format)) { + console.log(format === 'json' ? JSON.stringify(graph, null, 2) : format === 'yaml' ? yamlStringify(graph).trimEnd() : JSON.stringify(graph)); + return; + } + + if (graph.nodes.length === 0) { + console.log(warnBanner("No criteria in the graph.")); + return; + } + + console.log(label(`Criteria DAG ā€” ${graph.nodes.length} nodes, ${graph.edges.length} edges\n`)); + + // Topological layering (Kahn's algorithm) + const inDegree = new Map(); + const children = new Map(); + for (const n of graph.nodes) { + inDegree.set(n.id, 0); + children.set(n.id, []); + } + for (const e of graph.edges) { + inDegree.set(e.target, (inDegree.get(e.target) ?? 0) + 1); + children.get(e.source)?.push(e.target); + } + + const layers: string[][] = []; + let queue = graph.nodes.filter(n => (inDegree.get(n.id) ?? 0) === 0).map(n => n.id); + while (queue.length > 0) { + layers.push([...queue]); + const next: string[] = []; + for (const id of queue) { + for (const child of children.get(id) ?? []) { + const deg = (inDegree.get(child) ?? 1) - 1; + inDegree.set(child, deg); + if (deg === 0) next.push(child); + } + } + queue = next; + } + + // Render layers + for (let i = 0; i < layers.length; i++) { + const layerNodes = layers[i]; + const row = layerNodes.map(id => value(id)).join(' '); + console.log(` ${dimTimestamp(`Layer ${i}:`)} ${row}`); + } + + // Show edges + if (graph.edges.length > 0) { + console.log(`\n ${label('Edges:')}`); + for (const e of graph.edges) { + console.log(` ${value(e.source)} ${styleText('gray', 'ā†’')} ${value(e.target)}`); + } + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +criteria + .command("import") + .description("Import criteria from YAML file(s) into the database (upsert ā€” won't overwrite existing)") + .argument("", "Path to a .yaml file or a directory of .yaml files") + .option("--dry-run", "Preview what would be imported without sending to API") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (inputPath: string, options) => { + try { + const absPath = resolve(inputPath); + if (!existsSync(absPath)) { + console.error(errorText(`Path not found: ${absPath}`)); + process.exit(1); + } + + // Collect YAML files + let yamlFiles: string[]; + if (statSync(absPath).isDirectory()) { + yamlFiles = readdirSync(absPath) + .filter(f => extname(f) === '.yaml' || extname(f) === '.yml') + .sort() + .map(f => join(absPath, f)); + if (yamlFiles.length === 0) { + console.error(errorText(`No .yaml files found in ${absPath}`)); + process.exit(1); + } + console.log(`${label('Directory:')} ${value(absPath)} (${yamlFiles.length} files)`); + } else { + yamlFiles = [absPath]; + console.log(`${label('File:')} ${value(absPath)}`); + } + + // Parse all criteria from files (supports multi-document YAML) + const allCriteria: Array<{ id: string; prompt: string; dependsOn?: string[] }> = []; + const parseErrors: string[] = []; + + for (const file of yamlFiles) { + const content = readFileSync(file, 'utf-8'); + const fname = basename(file); + + try { + // Try multi-document parse first (handles --- separators) + const docs = parseAllDocuments(content); + for (let docIdx = 0; docIdx < docs.length; docIdx++) { + const doc = docs[docIdx].toJSON(); + if (!doc || typeof doc !== 'object') continue; + + const criterion = mapYamlCriterion(doc, fname, docIdx); + if (criterion) { + allCriteria.push(criterion); + } else { + parseErrors.push(`${fname}${docs.length > 1 ? ` (doc ${docIdx + 1})` : ''}: missing id or prompt`); + } + } + } catch (e) { + parseErrors.push(`${fname}: ${e instanceof Error ? e.message : String(e)}`); + } + } + + if (parseErrors.length > 0) { + console.log(`\n${warnBanner('Parse warnings:')}`); + for (const err of parseErrors) { + console.log(` ${errorText('āš ')} ${err}`); + } + } + + if (allCriteria.length === 0) { + console.error(errorText('No valid criteria found to import.')); + process.exit(1); + } + + console.log(`\n${label('Parsed:')} ${value(String(allCriteria.length))} criteria`); + + // Show preview + for (const c of allCriteria) { + const deps = (c.dependsOn ?? []).length; + const depsStr = deps > 0 ? ` ${dimTimestamp(`(${deps} dep${deps > 1 ? 's' : ''})`)}` : ''; + console.log(` ${value(c.id)}${depsStr}`); + } + + if (options.dryRun) { + console.log(`\n${warnBanner('Dry run ā€” no changes made.')}`); + return; + } + + // Seed via API + console.log(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/criteria/seed`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ criteria: allCriteria }), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const result = await response.json() as { seeded: number; errors: string[] }; + console.log(`${successText('Seeded:')} ${value(String(result.seeded))} criteria`); + if (result.seeded < allCriteria.length) { + console.log(`${dimTimestamp(`(${allCriteria.length - result.seeded} already existed ā€” skipped)`)}`); + } + if (result.errors.length > 0) { + console.log(`\n${warnBanner('Seed errors:')}`); + for (const err of result.errors) { + console.log(` ${errorText('āš ')} ${err}`); + } + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +/** + * Map a parsed YAML object to a criteria API payload. + * Handles snake_case `depends_on` ā†’ camelCase `dependsOn` conversion. + */ +function mapYamlCriterion( + doc: Record, + filename: string, + _docIndex: number +): { id: string; prompt: string; dependsOn?: string[] } | null { + const id = doc.id as string | undefined; + const prompt = doc.prompt as string | undefined; + if (!id || !prompt) return null; + + // Support both snake_case (YAML convention) and camelCase + const depsRaw = (doc.depends_on ?? doc.dependsOn) as string[] | undefined; + const dependsOn = Array.isArray(depsRaw) ? depsRaw.map(d => String(d).trim()) : undefined; + + return { + id: id.trim(), + prompt: prompt.trim(), + ...(dependsOn && dependsOn.length > 0 ? { dependsOn } : {}), + }; +} + +// ā”€ā”€ā”€ Prompt Feature management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const promptFeature = program + .command("prompt-feature") + .description("Manage prompt features (CRUD, import, extract, graph)") + .action(() => { + promptFeature.help(); + }); + +configureHelp(promptFeature); + +withOutputOption( +promptFeature + .command("list") + .description("List all prompt features") + .option("-q, --query ", "Filter by ID or prompt text") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = new URLSearchParams(); + if (options.query) params.set("q", options.query); + const qs = params.toString(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features${qs ? `?${qs}` : ""}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const items = await response.json() as Array<{ id: string; prompt: string; dependsOn?: string[] }>; + if (items.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No prompt features found.")); + return; + } + + if (!isMachineReadable(format)) { + console.log(label(`Found ${items.length} prompt features:\n`)); + } + + const displayFields: DisplayField[] = [ + { key: 'id', label: 'ID', + tableFormatter: (f: any) => value(f.id), + }, + { key: 'dependsOn', label: 'Deps', formatter: (f: any) => String((f.dependsOn ?? []).length) }, + { key: 'prompt', label: 'Prompt', formatter: (f: any) => { + const prompt = f.prompt.replace(/\n/g, ' '); + return prompt.length > 60 ? prompt.substring(0, 60) + 'ā€¦' : prompt; + }, tableFormatter: (f: any) => { + const prompt = f.prompt.replace(/\n/g, ' '); + const truncated = prompt.length > 60 ? prompt.substring(0, 60) + 'ā€¦' : prompt; + return dimTimestamp(truncated); + }}, + ]; + + console.log(formatData(items, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +promptFeature + .command("get") + .description("Get details of a single prompt feature") + .requiredOption("-i, --id ", "Prompt feature ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features/${options.id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const f = await response.json() as { + id: string; prompt: string; dependsOn?: string[]; + dependents: string[]; createdAt: string; updatedAt?: string; + }; + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'prompt', label: 'Prompt' }, + { key: 'dependsOn', label: 'Depends On', formatter: (item: any) => (item.dependsOn ?? []).join(', ') || '(none)' }, + { key: 'dependents', label: 'Dependents', formatter: (item: any) => (item.dependents ?? []).join(', ') || '(none)' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([f], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(f.id)}`); + console.log(`${label('Prompt:')}`); + for (const line of f.prompt.trim().split('\n')) { + console.log(` ${line}`); + } + if ((f.dependsOn ?? []).length > 0) { + console.log(`${label('Depends on:')} ${f.dependsOn!.map(d => value(d)).join(', ')}`); + } else { + console.log(`${label('Depends on:')} ${dimTimestamp('(none ā€” root feature)')}`); + } + if (f.dependents.length > 0) { + console.log(`${label('Dependents:')} ${f.dependents.map(d => value(d)).join(', ')}`); + } + console.log(`${label('Created:')} ${value(f.createdAt)}`); + if (f.updatedAt) console.log(`${label('Updated:')} ${value(f.updatedAt)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +promptFeature + .command("create") + .description("Create a new prompt feature") + .requiredOption("--id ", "Prompt feature ID (lowercase snake_case)") + .requiredOption("--prompt ", "Detection prompt for the feature") + .option("-d, --depends-on ", "IDs of parent features") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = { + id: options.id, + prompt: options.prompt, + }; + if (options.dependsOn && options.dependsOn.length > 0) { + body.dependsOn = options.dependsOn; + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const created = await response.json(); + console.log(`${successText('Created prompt feature')} ${value(created.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +promptFeature + .command("update") + .description("Update an existing prompt feature") + .requiredOption("-i, --id ", "Prompt feature ID") + .option("--prompt ", "New detection prompt") + .option("-d, --depends-on ", "New parent feature IDs (replaces all)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = {}; + if (options.prompt !== undefined) body.prompt = options.prompt; + if (options.dependsOn !== undefined) body.dependsOn = options.dependsOn; + + if (Object.keys(body).length === 0) { + console.error(errorText("Error: provide --prompt and/or --depends-on")); + process.exit(1); + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features/${options.id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + console.log(`${successText('Updated prompt feature')} ${value(options.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +promptFeature + .command("delete") + .description("Delete a prompt feature (soft-delete; fails if other features depend on it)") + .requiredOption("-i, --id ", "Prompt feature ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features/${options.id}`, { + method: "DELETE", + }); + + if (!response.ok) { + const error = await response.json(); + if (error.dependents) { + console.error(errorText(`Cannot delete '${options.id}': depended on by ${error.dependents.join(', ')}`)); + } else { + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + } + process.exit(1); + } + + console.log(`${successText('Deleted prompt feature')} ${value(options.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +promptFeature + .command("graph") + .description("Display the prompt feature dependency graph as ASCII") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features/graph`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const graph = await response.json() as { + nodes: Array<{ id: string; prompt: string; dependsOn: string[] }>; + edges: Array<{ source: string; target: string }>; + }; + + if (isMachineReadable(format)) { + console.log(format === 'json' ? JSON.stringify(graph, null, 2) : format === 'yaml' ? yamlStringify(graph).trimEnd() : JSON.stringify(graph)); + return; + } + + if (graph.nodes.length === 0) { + console.log(warnBanner("No prompt features in the graph.")); + return; + } + + console.log(label(`Prompt Feature DAG ā€” ${graph.nodes.length} nodes, ${graph.edges.length} edges\n`)); + + const inDegree = new Map(); + const children = new Map(); + for (const n of graph.nodes) { + inDegree.set(n.id, 0); + children.set(n.id, []); + } + for (const e of graph.edges) { + inDegree.set(e.target, (inDegree.get(e.target) ?? 0) + 1); + children.get(e.source)?.push(e.target); + } + + const layers: string[][] = []; + let queue = graph.nodes.filter(n => (inDegree.get(n.id) ?? 0) === 0).map(n => n.id); + while (queue.length > 0) { + layers.push([...queue]); + const next: string[] = []; + for (const id of queue) { + for (const child of children.get(id) ?? []) { + const deg = (inDegree.get(child) ?? 1) - 1; + inDegree.set(child, deg); + if (deg === 0) next.push(child); + } + } + queue = next; + } + + for (let i = 0; i < layers.length; i++) { + const layerNodes = layers[i]; + const row = layerNodes.map(id => value(id)).join(' '); + console.log(` ${dimTimestamp(`Layer ${i}:`)} ${row}`); + } + + if (graph.edges.length > 0) { + console.log(`\n ${label('Edges:')}`); + for (const e of graph.edges) { + console.log(` ${value(e.source)} ${styleText('gray', 'ā†’')} ${value(e.target)}`); + } + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +promptFeature + .command("import") + .description("Import prompt features from YAML file(s) into the database (upsert)") + .argument("", "Path to a .yaml file or a directory of .yaml files") + .option("--dry-run", "Preview what would be imported without sending to API") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (inputPath: string, options) => { + try { + const absPath = resolve(inputPath); + if (!existsSync(absPath)) { + console.error(errorText(`Path not found: ${absPath}`)); + process.exit(1); + } + + let yamlFiles: string[]; + if (statSync(absPath).isDirectory()) { + yamlFiles = readdirSync(absPath) + .filter(f => extname(f) === '.yaml' || extname(f) === '.yml') + .sort() + .map(f => join(absPath, f)); + if (yamlFiles.length === 0) { + console.error(errorText(`No .yaml files found in ${absPath}`)); + process.exit(1); + } + console.log(`${label('Directory:')} ${value(absPath)} (${yamlFiles.length} files)`); + } else { + yamlFiles = [absPath]; + console.log(`${label('File:')} ${value(absPath)}`); + } + + const allFeatures: Array<{ id: string; prompt: string; dependsOn?: string[] }> = []; + const parseErrors: string[] = []; + + for (const file of yamlFiles) { + const content = readFileSync(file, 'utf-8'); + const fname = basename(file); + + try { + const docs = parseAllDocuments(content); + for (let docIdx = 0; docIdx < docs.length; docIdx++) { + const doc = docs[docIdx].toJSON(); + if (!doc || typeof doc !== 'object') continue; + + // Reuse the same YAML mapping logic + const feature = mapYamlCriterion(doc, fname, docIdx); + if (feature) { + allFeatures.push(feature); + } else { + parseErrors.push(`${fname}${docs.length > 1 ? ` (doc ${docIdx + 1})` : ''}: missing id or prompt`); + } + } + } catch (e) { + parseErrors.push(`${fname}: ${e instanceof Error ? e.message : String(e)}`); + } + } + + if (parseErrors.length > 0) { + console.log(`\n${warnBanner('Parse warnings:')}`); + for (const err of parseErrors) { + console.log(` ${errorText('āš ')} ${err}`); + } + } + + if (allFeatures.length === 0) { + console.error(errorText('No valid prompt features found to import.')); + process.exit(1); + } + + console.log(`\n${label('Parsed:')} ${value(String(allFeatures.length))} prompt features`); + + for (const f of allFeatures) { + const deps = (f.dependsOn ?? []).length; + const depsStr = deps > 0 ? ` ${dimTimestamp(`(${deps} dep${deps > 1 ? 's' : ''})`)}` : ''; + console.log(` ${value(f.id)}${depsStr}`); + } + + if (options.dryRun) { + console.log(`\n${warnBanner('Dry run ā€” no changes made.')}`); + return; + } + + console.log(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/prompt-features/seed`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ features: allFeatures }), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const result = await response.json() as { seeded: number; errors: string[] }; + console.log(`${successText('Seeded:')} ${value(String(result.seeded))} prompt features`); + if (result.seeded < allFeatures.length) { + console.log(`${dimTimestamp(`(${allFeatures.length - result.seeded} already existed ā€” skipped)`)}`); + } + if (result.errors.length > 0) { + console.log(`\n${warnBanner('Seed errors:')}`); + for (const err of result.errors) { + console.log(` ${errorText('āš ')} ${err}`); + } + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +promptFeature + .command("extract") + .description("Extract prompt features from a task text or scenario file (uses task prompt pipeline)") + .option("-t, --task ", "Task text to analyze") + .option("-s, --scenario ", "Path to scenario YAML file (uses its task text)") + .option("--model ", "LLM model to use for extraction") + .option("--force", "Force re-extraction even if already extracted") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + let taskText = options.task; + + if (!taskText && options.scenario) { + const absPath = resolve(options.scenario); + if (!existsSync(absPath)) { + console.error(errorText(`Scenario file not found: ${absPath}`)); + process.exit(1); + } + const content = readFileSync(absPath, 'utf-8'); + const parsed = yamlParse(content); + taskText = parsed?.task || parsed?.scenario?.task; + if (!taskText) { + console.error(errorText("Could not find 'task' field in scenario file")); + process.exit(1); + } + if (!isMachineReadable(format)) console.log(`${label('Scenario:')} ${value(basename(absPath))}`); + } + + if (!taskText) { + console.error(errorText("Error: provide --task or --scenario")); + process.exit(1); + } + + // Step 1: Register task prompt (idempotent) + if (!isMachineReadable(format)) console.log(`${label('Registering task prompt...')}`); + const createResponse = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: taskText }), + }); + + if (!createResponse.ok) { + const error = await createResponse.json(); + console.error(errorText("Error creating task prompt:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const taskPromptDoc = await createResponse.json() as { _id: string }; + if (!isMachineReadable(format)) console.log(`${label('Task prompt ID:')} ${value(taskPromptDoc._id)}`); + + // Step 2: Extract features on the task prompt entity + if (!isMachineReadable(format)) console.log(`${label('Extracting prompt features...')}`); + + const qs = options.force ? "?force=true" : ""; + const body: Record = {}; + if (options.model) body.model = options.model; + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts/${encodeURIComponent(taskPromptDoc._id)}/extract-features${qs}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const extraction = await response.json() as { + features: Array<{ featureId: string; detected: boolean; evaluated: boolean }>; + cached: boolean; + }; + + if (isMachineReadable(format)) { + console.log(format === 'json' ? JSON.stringify(extraction, null, 2) : format === 'yaml' ? yamlStringify(extraction).trimEnd() : JSON.stringify(extraction)); + return; + } + + if (extraction.cached) { + console.log(dimTimestamp('(cached ā€” use --force to re-extract)')); + } + + const detected = extraction.features.filter(r => r.detected); + const notDetected = extraction.features.filter(r => !r.detected && r.evaluated); + const skipped = extraction.features.filter(r => !r.evaluated); + + console.log(`\n${label('Results:')}`); + if (detected.length > 0) { + console.log(` ${successText('Detected:')}`); + for (const r of detected) { + console.log(` ${criterionIcon(true, true)} ${value(r.featureId)}`); + } + } + if (notDetected.length > 0) { + console.log(` ${dimTimestamp('Not detected:')}`); + for (const r of notDetected) { + console.log(` ${criterionIcon(true, false)} ${dimTimestamp(r.featureId)}`); + } + } + if (skipped.length > 0) { + console.log(` ${warnBanner('Skipped (not evaluated):')}`); + for (const r of skipped) { + console.log(` ā—‹ ${dimTimestamp(r.featureId)}`); + } + } + + console.log(`\n${label('Summary:')} ${value(String(detected.length))} detected, ${dimTimestamp(String(notDetected.length))} not detected, ${dimTimestamp(String(skipped.length))} skipped`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Report management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const report = program + .command("report") + .description("Generate, view, and monitor run reports") + .action(() => { + report.help(); + }); + +configureHelp(report); + +report + .command("generate") + .description("Generate a report for a benchmark run") + .requiredOption("-i, --id ", "Run ID to generate a report for") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("--stream", "Stream report generation logs in real time", true) + .option("--no-stream", "Do not stream logs after submission") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/reports`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ requestId: options.id }), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const result = await response.json() as { id: string; requestId: string; status: string }; + console.log(`${successText("Report queued")} ${dimTimestamp(`(${result.id})`)}`); + console.log(`${label('Report ID:')} ${value(result.id)}`); + console.log(`${label('Run ID:')} ${value(result.requestId)}`); + + if (options.stream) { + console.log(`\n${label('Streaming logs...')}\n`); + const eventSource = new EventSource( + `${normalizeUrl(options.url)}/api/v1/reports/${result.id}/logs?fromStart=true` + ); + + eventSource.onmessage = (event: MessageEvent) => { + try { + const log = JSON.parse(event.data) as { + timestamp: string; + level: string; + source?: string; + message: string; + }; + const ts = dimTimestamp(new Date(log.timestamp).toLocaleTimeString()); + const lvl = colorLevel(log.level); + const src = log.source ? ` ${dimTimestamp(`[${log.source}]`)}` : ""; + console.log(`${ts} ${lvl}${src} ${log.message}`); + } catch { + console.log(event.data); + } + }; + + eventSource.addEventListener("done", () => { + console.log(`\n${successText("Report generation complete")}`); + console.log(`\n${label('Next steps:')}`); + console.log(` ${dimTimestamp('View report:')} pnpm cli report get -i ${result.id}`); + eventSource.close(); + process.exit(0); + }); + + eventSource.addEventListener("timeout", () => { + console.log(`\n${warnBanner("Stream timed out")}`); + eventSource.close(); + process.exit(0); + }); + + eventSource.onerror = () => { + eventSource.close(); + process.exit(1); + }; + } else { + console.log(`\n${label('Next steps:')}`); + console.log(` ${dimTimestamp('Stream logs:')} pnpm cli report logs -i ${result.id}`); + console.log(` ${dimTimestamp('View report:')} pnpm cli report get -i ${result.id}`); + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +report + .command("get") + .description("Get a report by ID") + .requiredOption("-i, --id ", "Report ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +, ['markdown']) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/reports/${options.id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const report = await response.json() as { + id: string; + requestId: string; + status: string; + reporter?: { id: string; name: string; model: string; agentId: string; agentVersion: string }; + content?: string; + createdAt: string; + updatedAt?: string; + error?: string; + }; + + if (format === 'markdown') { + console.log(report.content ?? ''); + return; + } + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'requestId', label: 'Run' }, + { key: 'status', label: 'Status' }, + { key: 'reporter', label: 'Reporter', formatter: (r: any) => r.reporter ? `${r.reporter.name} (${r.reporter.agentId}@${r.reporter.agentVersion})` : '' }, + { key: 'model', label: 'Model', formatter: (r: any) => r.reporter?.model || '' }, + { key: 'content', label: 'Content', formatter: (r: any) => r.content || '' }, + { key: 'error', label: 'Error', formatter: (r: any) => r.error || '' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([report], fields, format)); + return; + } + + console.log(`${label('Report:')} ${value(report.id)}`); + console.log(`${label('Run:')} ${value(report.requestId)}`); + console.log(`${label('Status:')} ${colorLevel(report.status === "completed" ? "info" : report.status === "failed" ? "error" : "warn")} ${report.status}`); + console.log(`${label('Created:')} ${dimTimestamp(new Date(report.createdAt).toLocaleString())}`); + if (report.updatedAt) { + console.log(`${label('Updated:')} ${dimTimestamp(new Date(report.updatedAt).toLocaleString())}`); + } + if (report.reporter) { + console.log(`${label('Reporter:')} ${value(report.reporter.name)} (${report.reporter.agentId}@${report.reporter.agentVersion})`); + console.log(`${label('Model:')} ${value(report.reporter.model)}`); + } + if (report.error) { + console.log(`${label('Error:')} ${errorText(report.error)}`); + } + if (report.content) { + console.log(`\n${banner('ā”€ā”€ā”€ Report Content ā”€ā”€ā”€')}\n`); + console.log(report.content); + } else if (report.status === "pending" || report.status === "generating") { + console.log(`\n${dimTimestamp('Report is still being generated. Stream logs with:')}`); + console.log(` pnpm cli report logs -i ${report.id}`); + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +report + .command("list") + .description("List all reports (optionally filter by run)") + .option("-r, --run ", "Filter by run ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = new URLSearchParams(); + if (options.run) params.set("requestId", options.run); + const qs = params.toString(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/reports${qs ? `?${qs}` : ""}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const reports = await response.json() as Array<{ + id: string; + requestId: string; + status: string; + reporter?: { model: string }; + createdAt: string; + }>; + + if (reports.length === 0) { + if (!isMachineReadable(format)) console.log(dimTimestamp("No reports found")); + return; + } + + if (!isMachineReadable(format)) { + console.log(`${label(`Reports (${reports.length}):`)}\n`); + } + + const displayFields: DisplayField[] = [ + { key: 'id', label: 'ID', + tableFormatter: (r: any) => value(r.id), + }, + { key: 'requestId', label: 'Run ID', + tableFormatter: (r: any) => dimTimestamp(r.requestId), + }, + { key: 'status', label: 'Status', + tableFormatter: (r: any) => { + return r.status === 'completed' ? successText('āœ“ ' + r.status) + : r.status === 'failed' ? errorText('āœ— ' + r.status) + : dimTimestamp('ā€¦ ' + r.status); + }, + }, + { key: 'model', label: 'Model', formatter: (r: any) => r.reporter?.model ?? 'N/A', + tableFormatter: (r: any) => r.reporter?.model ? dimTimestamp(r.reporter.model) : 'N/A', + }, + { key: 'createdAt', label: 'Created', formatter: (r: any) => new Date(r.createdAt).toLocaleString(), + tableFormatter: (r: any) => dimTimestamp(new Date(r.createdAt).toLocaleString()), + }, + ]; + + console.log(formatData(reports, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +report + .command("logs") + .description("Stream report generation logs") + .requiredOption("-i, --id ", "Report ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .option("--from-start", "Include historical logs from the beginning", false) + .action(async (options) => { + try { + // Verify report exists first + const checkResponse = await fetch(`${normalizeUrl(options.url)}/api/v1/reports/${options.id}`); + if (!checkResponse.ok) { + const error = await checkResponse.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const fromStartParam = options.fromStart ? "&fromStart=true" : ""; + const eventSource = new EventSource( + `${normalizeUrl(options.url)}/api/v1/reports/${options.id}/logs?${fromStartParam}` + ); + + eventSource.onmessage = (event: MessageEvent) => { + try { + const log = JSON.parse(event.data) as { + timestamp: string; + level: string; + source?: string; + message: string; + }; + const ts = dimTimestamp(new Date(log.timestamp).toLocaleTimeString()); + const lvl = colorLevel(log.level); + const src = log.source ? ` ${dimTimestamp(`[${log.source}]`)}` : ""; + console.log(`${ts} ${lvl}${src} ${log.message}`); + } catch { + console.log(event.data); + } + }; + + eventSource.addEventListener("done", () => { + console.log(`\n${successText("Report generation complete")}`); + eventSource.close(); + process.exit(0); + }); + + eventSource.addEventListener("timeout", () => { + console.log(`\n${warnBanner("Stream timed out")}`); + eventSource.close(); + process.exit(0); + }); + + eventSource.onerror = () => { + eventSource.close(); + process.exit(1); + }; + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Report Template management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const reportTemplate = program + .command("report-template") + .description("Manage report templates (CRUD, import)") + .action(() => { + reportTemplate.help(); + }); + +configureHelp(reportTemplate); + +withOutputOption( +reportTemplate + .command("list") + .description("List all report templates") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const items = await response.json() as Array<{ + id: string; name: string; description?: string; + userPrompt: string; systemPrompt?: { mode: string; content: string }; + trigger?: { type: string; [k: string]: unknown }; + createdAt: string; + }>; + + if (items.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No report templates found.")); + return; + } + + if (!isMachineReadable(format)) { + console.log(label(`Found ${items.length} report templates:\n`)); + } + + const displayFields: DisplayField[] = [ + { key: 'id', label: 'ID', tableFormatter: (t: any) => value(t.id) }, + { key: 'name', label: 'Name' }, + { key: 'trigger', label: 'Trigger', formatter: (t: any) => t.trigger?.type ?? 'always' }, + { key: 'systemPrompt', label: 'SysPrompt', formatter: (t: any) => t.systemPrompt ? t.systemPrompt.mode : 'ā€”' }, + { key: 'userPrompt', label: 'UserPrompt', formatter: (t: any) => { + const p = t.userPrompt.replace(/\n/g, ' '); + return p.length > 50 ? p.substring(0, 50) + 'ā€¦' : p; + }, tableFormatter: (t: any) => { + const p = t.userPrompt.replace(/\n/g, ' '); + const truncated = p.length > 50 ? p.substring(0, 50) + 'ā€¦' : p; + return dimTimestamp(truncated); + }}, + ]; + + console.log(formatData(items, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +reportTemplate + .command("get") + .description("Get details of a single report template") + .requiredOption("-i, --id ", "Report template ID (slug)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates/${options.id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const t = await response.json() as { + id: string; name: string; description?: string; + userPrompt: string; systemPrompt?: { mode: string; content: string }; + trigger?: { type: string; [k: string]: unknown }; + createdAt: string; updatedAt?: string; + }; + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'name', label: 'Name' }, + { key: 'description', label: 'Description', formatter: (item: any) => item.description || '' }, + { key: 'trigger', label: 'Trigger', formatter: (item: any) => item.trigger?.type ?? 'always' }, + { key: 'userPrompt', label: 'User Prompt' }, + { key: 'systemPrompt', label: 'System Prompt', formatter: (item: any) => item.systemPrompt ? `${item.systemPrompt.mode}: ${item.systemPrompt.content}` : '' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([t], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(t.id)}`); + console.log(`${label('Name:')} ${t.name}`); + if (t.description) console.log(`${label('Description:')} ${t.description}`); + console.log(`${label('Trigger:')} ${value(t.trigger?.type ?? 'always')}`); + if (t.trigger && t.trigger.type !== 'always') { + console.log(`${label('Trigger cfg:')} ${dimTimestamp(JSON.stringify(t.trigger))}`); + } + console.log(`${label('User prompt:')}`); + for (const line of t.userPrompt.trim().split('\n')) { + console.log(` ${line}`); + } + if (t.systemPrompt) { + console.log(`${label('Sys prompt:')} ${value(t.systemPrompt.mode)}`); + for (const line of t.systemPrompt.content.trim().split('\n')) { + console.log(` ${line}`); + } + } + console.log(`${label('Created:')} ${value(t.createdAt)}`); + if (t.updatedAt) console.log(`${label('Updated:')} ${value(t.updatedAt)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +reportTemplate + .command("create") + .description("Create a new report template") + .requiredOption("--id ", "Template ID slug (lowercase, hyphens)") + .requiredOption("--name ", "Display name for the template") + .requiredOption("--user-prompt ", "User prompt (the instruction for the report agent)") + .option("--description ", "Optional description") + .option("--system-prompt-mode ", "System prompt mode: append or override") + .option("--system-prompt-content ", "System prompt content") + .option("--trigger-type ", "Trigger type: always, criteria, taskPrompt, promptFeature") + .option("--trigger-ids ", "Trigger IDs (criteria IDs, task prompt IDs, or feature IDs)") + .option("--trigger-match ", "Trigger match mode: any or all (default: all)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = { + id: options.id, + name: options.name, + userPrompt: options.userPrompt, + }; + if (options.description) body.description = options.description; + if (options.systemPromptMode && options.systemPromptContent) { + body.systemPrompt = { + mode: options.systemPromptMode, + content: options.systemPromptContent, + }; + } + if (options.triggerType) { + const trigger: Record = { type: options.triggerType }; + if (options.triggerType === 'criteria' && options.triggerIds) { + trigger.criteriaIds = options.triggerIds; + if (options.triggerMatch) trigger.match = options.triggerMatch; + } else if (options.triggerType === 'taskPrompt' && options.triggerIds) { + trigger.taskPromptIds = options.triggerIds; + } else if (options.triggerType === 'promptFeature' && options.triggerIds) { + trigger.featureIds = options.triggerIds; + if (options.triggerMatch) trigger.match = options.triggerMatch; + } + body.trigger = trigger; + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const created = await response.json(); + console.log(`${successText('Created report template')} ${value(created.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +reportTemplate + .command("update") + .description("Update an existing report template") + .requiredOption("-i, --id ", "Report template ID (slug)") + .option("--name ", "New display name") + .option("--description ", "New description") + .option("--user-prompt ", "New user prompt") + .option("--system-prompt-mode ", "System prompt mode: append or override") + .option("--system-prompt-content ", "System prompt content") + .option("--trigger-type ", "New trigger type: always, criteria, taskPrompt, promptFeature") + .option("--trigger-ids ", "Trigger IDs") + .option("--trigger-match ", "Trigger match mode: any or all") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = {}; + if (options.name !== undefined) body.name = options.name; + if (options.description !== undefined) body.description = options.description; + if (options.userPrompt !== undefined) body.userPrompt = options.userPrompt; + if (options.systemPromptMode && options.systemPromptContent) { + body.systemPrompt = { + mode: options.systemPromptMode, + content: options.systemPromptContent, + }; + } + if (options.triggerType) { + const trigger: Record = { type: options.triggerType }; + if (options.triggerType === 'criteria' && options.triggerIds) { + trigger.criteriaIds = options.triggerIds; + if (options.triggerMatch) trigger.match = options.triggerMatch; + } else if (options.triggerType === 'taskPrompt' && options.triggerIds) { + trigger.taskPromptIds = options.triggerIds; + } else if (options.triggerType === 'promptFeature' && options.triggerIds) { + trigger.featureIds = options.triggerIds; + if (options.triggerMatch) trigger.match = options.triggerMatch; + } + body.trigger = trigger; + } + + if (Object.keys(body).length === 0) { + console.error(errorText("Error: provide at least one field to update")); + process.exit(1); + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates/${options.id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + console.log(`${successText('Updated report template')} ${value(options.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +reportTemplate + .command("delete") + .description("Delete a report template (soft-delete)") + .requiredOption("-i, --id ", "Report template ID (slug)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates/${options.id}`, { + method: "DELETE", + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + console.log(`${successText('Deleted report template')} ${value(options.id)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +reportTemplate + .command("import") + .description("Import report templates from a YAML file (upsert via create/update)") + .argument("", "Path to a .yaml file or directory of .yaml files") + .option("--dry-run", "Preview what would be imported without sending to API") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (inputPath: string, options) => { + try { + const absPath = resolve(inputPath); + if (!existsSync(absPath)) { + console.error(errorText(`Path not found: ${absPath}`)); + process.exit(1); + } + + let yamlFiles: string[]; + if (statSync(absPath).isDirectory()) { + yamlFiles = readdirSync(absPath) + .filter(f => extname(f) === '.yaml' || extname(f) === '.yml') + .sort() + .map(f => join(absPath, f)); + if (yamlFiles.length === 0) { + console.error(errorText(`No .yaml files found in ${absPath}`)); + process.exit(1); + } + console.log(`${label('Directory:')} ${value(absPath)} (${yamlFiles.length} files)`); + } else { + yamlFiles = [absPath]; + console.log(`${label('File:')} ${value(absPath)}`); + } + + const allTemplates: Array> = []; + const parseErrors: string[] = []; + + for (const file of yamlFiles) { + const content = readFileSync(file, 'utf-8'); + const fname = basename(file); + + try { + const docs = parseAllDocuments(content); + for (let docIdx = 0; docIdx < docs.length; docIdx++) { + const doc = docs[docIdx].toJSON(); + if (!doc || typeof doc !== 'object') continue; + + const template = mapYamlReportTemplate(doc, fname, docIdx); + if (template) { + allTemplates.push(template); + } else { + parseErrors.push(`${fname}${docs.length > 1 ? ` (doc ${docIdx + 1})` : ''}: missing id, name, or user_prompt`); + } + } + } catch (e) { + parseErrors.push(`${fname}: ${e instanceof Error ? e.message : String(e)}`); + } + } + + if (parseErrors.length > 0) { + console.log(`\n${warnBanner('Parse warnings:')}`); + for (const err of parseErrors) { + console.log(` ${errorText('āš ')} ${err}`); + } + } + + if (allTemplates.length === 0) { + console.error(errorText('No valid report templates found to import.')); + process.exit(1); + } + + console.log(`\n${label('Parsed:')} ${value(String(allTemplates.length))} report templates`); + for (const t of allTemplates) { + const triggerStr = t.trigger ? ` ${dimTimestamp(`(trigger: ${(t.trigger as any).type})`)}` : ''; + console.log(` ${value(t.id as string)} ā€” ${t.name}${triggerStr}`); + } + + if (options.dryRun) { + console.log(`\n${warnBanner('Dry run ā€” no changes made.')}`); + return; + } + + console.log(); + let created = 0; + let updated = 0; + const errors: string[] = []; + + for (const t of allTemplates) { + const id = t.id as string; + // Try to GET existing + const getResp = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates/${id}`); + if (getResp.ok) { + // Update + const { id: _id, ...updateBody } = t; + const resp = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates/${id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(updateBody), + }); + if (resp.ok) { + updated++; + console.log(` ${dimTimestamp('updated')} ${value(id)}`); + } else { + const err = await resp.json(); + errors.push(`${id}: ${err.error || JSON.stringify(err)}`); + } + } else if (getResp.status === 404) { + // Create + const resp = await fetch(`${normalizeUrl(options.url)}/api/v1/report-templates`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(t), + }); + if (resp.ok) { + created++; + console.log(` ${successText('created')} ${value(id)}`); + } else { + const err = await resp.json(); + errors.push(`${id}: ${err.error || JSON.stringify(err)}`); + } + } else { + errors.push(`${id}: failed to check existence (status ${getResp.status})`); + } + } + + console.log(`\n${successText('Import complete:')} ${value(String(created))} created, ${value(String(updated))} updated`); + if (errors.length > 0) { + console.log(`\n${warnBanner('Errors:')}`); + for (const err of errors) { + console.log(` ${errorText('āš ')} ${err}`); + } + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +/** + * Map a parsed YAML object to a report template API payload. + * Handles snake_case ā†’ camelCase conversion for YAML convention. + */ +function mapYamlReportTemplate( + doc: Record, + _filename: string, + _docIndex: number +): Record | null { + const id = doc.id as string | undefined; + const name = doc.name as string | undefined; + const userPrompt = (doc.user_prompt ?? doc.userPrompt) as string | undefined; + if (!id || !name || !userPrompt) return null; + + const result: Record = { + id: id.trim(), + name: name.trim(), + userPrompt: userPrompt.trim(), + }; + + const description = doc.description as string | undefined; + if (description) result.description = description.trim(); + + // System prompt: support snake_case YAML + const sysCfg = (doc.system_prompt ?? doc.systemPrompt) as Record | undefined; + if (sysCfg && sysCfg.mode && sysCfg.content) { + result.systemPrompt = { + mode: String(sysCfg.mode).trim(), + content: String(sysCfg.content).trim(), + }; + } + + // Trigger + const triggerCfg = doc.trigger as Record | undefined; + if (triggerCfg && triggerCfg.type) { + const trigger: Record = { type: String(triggerCfg.type).trim() }; + const triggerType = trigger.type as string; + + if (triggerType === 'criteria') { + const ids = (triggerCfg.criteria_ids ?? triggerCfg.criteriaIds) as string[] | undefined; + if (Array.isArray(ids)) trigger.criteriaIds = ids.map(s => String(s).trim()); + if (triggerCfg.match) trigger.match = String(triggerCfg.match).trim(); + } else if (triggerType === 'taskPrompt') { + const ids = (triggerCfg.task_prompt_ids ?? triggerCfg.taskPromptIds) as string[] | undefined; + if (Array.isArray(ids)) trigger.taskPromptIds = ids.map(s => String(s).trim()); + } else if (triggerType === 'promptFeature') { + const ids = (triggerCfg.feature_ids ?? triggerCfg.featureIds) as string[] | undefined; + if (Array.isArray(ids)) trigger.featureIds = ids.map(s => String(s).trim()); + if (triggerCfg.match) trigger.match = String(triggerCfg.match).trim(); + } + + result.trigger = trigger; + } + + return result; +} + +// ā”€ā”€ā”€ Agent management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const agent = program + .command("agent") + .description("Manage coding agent definitions and their supported models") + .action(() => { + agent.help(); + }); + +configureHelp(agent); + +withOutputOption( +agent + .command("list") + .description("List all coding agents") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/agents`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const agents = await response.json() as Array<{ _id: string; name: string; supportedModels: string[]; defaultModel?: string }>; + if (agents.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No agents found.")); + return; + } + if (!isMachineReadable(format)) { + console.log(label(`Found ${agents.length} agents:\n`)); + } + const displayFields: DisplayField[] = [ + { key: '_id', label: 'ID', tableFormatter: (a: any) => value(a._id) }, + { key: 'name', label: 'Name' }, + { key: 'supportedModels', label: 'Models', formatter: (a: any) => (a.supportedModels || []).join(', ') || 'ā€”' }, + { key: 'defaultModel', label: 'Default', formatter: (a: any) => a.defaultModel || 'ā€”' }, + ]; + console.log(formatData(agents, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +agent + .command("get") + .description("Get details of a coding agent") + .requiredOption("-i, --id ", "Agent ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const agentDoc = await response.json(); + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: '_id', label: 'ID' }, + { key: 'name', label: 'Name' }, + { key: 'description', label: 'Description', formatter: (a: any) => a.description || '' }, + { key: 'supportedModels', label: 'Supported Models', formatter: (a: any) => (a.supportedModels || []).join(', ') }, + { key: 'defaultModel', label: 'Default Model', formatter: (a: any) => a.defaultModel || '' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([agentDoc], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(agentDoc._id)}`); + console.log(`${label('Name:')} ${value(agentDoc.name)}`); + if (agentDoc.description) console.log(`${label('Description:')} ${agentDoc.description}`); + console.log(`${label('Supported Models:')} ${(agentDoc.supportedModels || []).join(', ') || '(none)'}`); + console.log(`${label('Default Model:')} ${agentDoc.defaultModel || '(none)'}`); + console.log(`${label('Created:')} ${new Date(agentDoc.createdAt).toLocaleString()}`); + if (agentDoc.updatedAt) console.log(`${label('Updated:')} ${new Date(agentDoc.updatedAt).toLocaleString()}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +agent + .command("update") + .description("Update a coding agent") + .requiredOption("-i, --id ", "Agent ID") + .option("--name ", "Display name") + .option("--description ", "Description") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = {}; + if (options.name) body.name = options.name; + if (options.description) body.description = options.description; + if (Object.keys(body).length === 0) { + console.error(errorText("Error: provide at least one field to update (--name, --description)")); + process.exit(1); + } + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const updated = await response.json(); + console.log(successText(`Agent ${updated._id} updated.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +agent + .command("delete") + .description("Delete a coding agent (soft-delete)") + .requiredOption("-i, --id ", "Agent ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`, { + method: "DELETE", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Agent ${options.id} deleted.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Agent model sub-commands ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const agentModel = agent + .command("model") + .description("Manage supported models for a coding agent") + .action(() => { + agentModel.help(); + }); + +configureHelp(agentModel); + +withOutputOption( +agentModel + .command("list") + .description("List supported models for a coding agent") + .requiredOption("-i, --id ", "Agent ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const agentDoc = await response.json(); + const models = (agentDoc.supportedModels || []) as string[]; + if (models.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner(`Agent ${agentDoc._id} has no supported models.`)); + return; + } + if (!isMachineReadable(format)) { + console.log(label(`Models for ${agentDoc._id}:\n`)); + } + const items = models.map((m: string) => ({ model: m, default: m === agentDoc.defaultModel ? 'āœ“' : '' })); + const displayFields: DisplayField[] = [ + { key: 'model', label: 'Model', tableFormatter: (r: any) => value(r.model) }, + { key: 'default', label: 'Default' }, + ]; + console.log(formatData(items, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +agentModel + .command("add") + .description("Add a supported model to a coding agent") + .requiredOption("-i, --id ", "Agent ID") + .requiredOption("--model ", "Model name to add") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + // Fetch current agent + const getResp = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`); + if (!getResp.ok) { + const error = await getResp.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const agentDoc = await getResp.json(); + const models: string[] = agentDoc.supportedModels || []; + if (models.includes(options.model)) { + console.log(warnBanner(`Model ${options.model} is already supported by ${agentDoc._id}.`)); + return; + } + models.push(options.model); + const putResp = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ supportedModels: models }), + }); + if (!putResp.ok) { + const error = await putResp.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Model ${options.model} added to ${agentDoc._id}.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +agentModel + .command("remove") + .description("Remove a supported model from a coding agent") + .requiredOption("-i, --id ", "Agent ID") + .requiredOption("--model ", "Model name to remove") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const getResp = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`); + if (!getResp.ok) { + const error = await getResp.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const agentDoc = await getResp.json(); + const models: string[] = agentDoc.supportedModels || []; + const idx = models.indexOf(options.model); + if (idx === -1) { + console.log(warnBanner(`Model ${options.model} is not supported by ${agentDoc._id}.`)); + return; + } + models.splice(idx, 1); + const body: Record = { supportedModels: models }; + // If removed model was the default, clear defaultModel + if (agentDoc.defaultModel === options.model) { + body.defaultModel = null; + } + const putResp = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!putResp.ok) { + const error = await putResp.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Model ${options.model} removed from ${agentDoc._id}.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +agentModel + .command("set-default") + .description("Set the default model for a coding agent") + .requiredOption("-i, --id ", "Agent ID") + .requiredOption("--model ", "Model name to set as default") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + // Verify the model is supported + const getResp = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`); + if (!getResp.ok) { + const error = await getResp.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const agentDoc = await getResp.json(); + const models: string[] = agentDoc.supportedModels || []; + if (!models.includes(options.model)) { + console.error(errorText(`Error: model ${options.model} is not in the supported models for ${agentDoc._id}. Add it first with: agent model add -i ${agentDoc._id} --model ${options.model}`)); + process.exit(1); + } + const putResp = await fetch(`${normalizeUrl(options.url)}/api/v1/agents/${encodeURIComponent(options.id)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ defaultModel: options.model }), + }); + if (!putResp.ok) { + const error = await putResp.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Default model for ${agentDoc._id} set to ${options.model}.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ MCP server management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const mcp = program + .command("mcp") + .description("Manage MCP (Model Context Protocol) resources") + .action(() => { + mcp.help(); + }); + +configureHelp(mcp); + +const mcpServer = mcp + .command("server") + .description("Manage remote MCP servers (SSE and streamable HTTP)") + .action(() => { + mcpServer.help(); + }); + +configureHelp(mcpServer); + +withOutputOption( +mcpServer + .command("list") + .description("List all MCP servers") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/mcp/servers`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const servers = await response.json() as Array<{ _id: string; name: string; type: string; url: string; description?: string }>; + if (servers.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No MCP servers found.")); + return; + } + if (!isMachineReadable(format)) { + console.log(label(`Found ${servers.length} MCP server(s):\n`)); + } + const displayFields: DisplayField[] = [ + { key: '_id', label: 'Slug', tableFormatter: (s: any) => value(s._id) }, + { key: 'name', label: 'Name' }, + { key: 'type', label: 'Type' }, + { key: 'url', label: 'URL' }, + { key: 'description', label: 'Description', formatter: (s: any) => s.description || 'ā€”' }, + ]; + console.log(formatData(servers, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +mcpServer + .command("get") + .description("Get details of an MCP server") + .requiredOption("-i, --id ", "MCP server slug") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/mcp/servers/${encodeURIComponent(options.id)}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const server = await response.json(); + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: '_id', label: 'Slug' }, + { key: 'name', label: 'Name' }, + { key: 'type', label: 'Type' }, + { key: 'url', label: 'URL' }, + { key: 'description', label: 'Description', formatter: (s: any) => s.description || '' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([server], fields, format)); + return; + } + + console.log(`${label('Slug:')} ${value(server._id)}`); + console.log(`${label('Name:')} ${value(server.name)}`); + console.log(`${label('Type:')} ${value(server.type)}`); + console.log(`${label('URL:')} ${value(server.url)}`); + if (server.description) console.log(`${label('Description:')} ${server.description}`); + if (server.headers && server.headers.length > 0) { + console.log(`${label('Headers:')}`); + for (const h of server.headers) { + console.log(` ${h.name}: ${h.value}`); + } + } + console.log(`${label('Created:')} ${new Date(server.createdAt).toLocaleString()}`); + if (server.updatedAt) console.log(`${label('Updated:')} ${new Date(server.updatedAt).toLocaleString()}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +mcpServer + .command("create") + .description("Create a new MCP server") + .requiredOption("--id ", "Slug identifier (lowercase, hyphens allowed)") + .requiredOption("--name ", "Display name") + .requiredOption("--type ", "Transport type (sse or http)") + .requiredOption("--url ", "Server URL") + .option("--description ", "Description") + .option("--header ", "Headers in name:value format (repeatable)") + .option("-u, --api-url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const headers = options.header?.map((h: string) => { + const idx = h.indexOf(':'); + if (idx === -1) { + console.error(errorText(`Invalid header format: "${h}". Expected name:value`)); + process.exit(1); + } + return { name: h.substring(0, idx).trim(), value: h.substring(idx + 1).trim() }; + }); + + const body: Record = { + _id: options.id, + name: options.name, + type: options.type, + url: options.url, + }; + if (options.description) body.description = options.description; + if (headers && headers.length > 0) body.headers = headers; + + const response = await fetch(`${normalizeUrl(options.apiUrl)}/api/v1/mcp/servers`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const created = await response.json(); + console.log(successText(`MCP server "${created._id}" created.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +mcpServer + .command("update") + .description("Update an MCP server") + .requiredOption("-i, --id ", "MCP server slug") + .option("--name ", "Display name") + .option("--type ", "Transport type (sse or http)") + .option("--url ", "Server URL") + .option("--description ", "Description") + .option("--header ", "Headers in name:value format (replaces all headers)") + .option("-u, --api-url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = {}; + if (options.name) body.name = options.name; + if (options.type) body.type = options.type; + if (options.url) body.url = options.url; + if (options.description) body.description = options.description; + if (options.header) { + body.headers = options.header.map((h: string) => { + const idx = h.indexOf(':'); + if (idx === -1) { + console.error(errorText(`Invalid header format: "${h}". Expected name:value`)); + process.exit(1); + } + return { name: h.substring(0, idx).trim(), value: h.substring(idx + 1).trim() }; + }); + } + if (Object.keys(body).length === 0) { + console.error(errorText("Error: provide at least one field to update")); + process.exit(1); + } + const response = await fetch(`${normalizeUrl(options.apiUrl)}/api/v1/mcp/servers/${encodeURIComponent(options.id)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const updated = await response.json(); + console.log(successText(`MCP server "${updated._id}" updated.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +mcpServer + .command("delete") + .description("Delete an MCP server (soft-delete)") + .requiredOption("-i, --id ", "MCP server slug") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/mcp/servers/${encodeURIComponent(options.id)}`, { + method: "DELETE", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`MCP server "${options.id}" deleted.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Skill management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const skill = program + .command("skill") + .description("Manage agent skills (Agent Skills Specification)") + .action(() => { + skill.help(); + }); + +configureHelp(skill); + +withOutputOption( +skill + .command("list") + .description("List all imported skills") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const skills = await response.json() as Array<{ _id: string; name: string; source: string; description?: string; origin: string }>; + if (skills.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No skills found.")); + return; + } + if (!isMachineReadable(format)) { + console.log(label(`Found ${skills.length} skill(s):\n`)); + } + const displayFields: DisplayField[] = [ + { key: '_id', label: 'Slug', tableFormatter: (s: any) => value(s._id) }, + { key: 'name', label: 'Name' }, + { key: 'source', label: 'Source' }, + { key: 'origin', label: 'Origin' }, + { key: 'description', label: 'Description', formatter: (s: any) => s.description || 'ā€”' }, + ]; + console.log(formatData(skills, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +skill + .command("search") + .description("Search skills (internal + skills.sh registry)") + .requiredOption("-q, --query ", "Search query") + .option("--limit ", "Maximum results", parseInt) + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = new URLSearchParams({ q: options.query }); + if (options.limit) params.set('limit', String(options.limit)); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills/search?${params}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const results = await response.json() as Array<{ id: string; name: string; source: string; description?: string; internal: boolean; installs?: number }>; + if (results.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No skills found.")); + return; + } + if (!isMachineReadable(format)) { + console.log(label(`Found ${results.length} result(s):\n`)); + } + const displayFields: DisplayField[] = [ + { key: 'id', label: 'Slug', tableFormatter: (s: any) => value(s.id) }, + { key: 'name', label: 'Name' }, + { key: 'source', label: 'Source' }, + { key: 'internal', label: 'Imported', formatter: (s: any) => s.internal ? 'Yes' : 'No' }, + { key: 'installs', label: 'Installs', formatter: (s: any) => s.installs != null ? String(s.installs) : 'ā€”' }, + { key: 'description', label: 'Description', formatter: (s: any) => s.description || 'ā€”' }, + ]; + console.log(formatData(results, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +skill + .command("get") + .description("Get details of a skill") + .requiredOption("-i, --id ", "Skill slug (e.g. vercel-labs/agent-skills/my-skill)") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills/${options.id}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const s = await response.json(); + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: '_id', label: 'Slug' }, + { key: 'name', label: 'Name' }, + { key: 'source', label: 'Source' }, + { key: 'skillName', label: 'Skill Name' }, + { key: 'origin', label: 'Origin' }, + { key: 'description', label: 'Description', formatter: (sk: any) => sk.description || '' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([s], fields, format)); + return; + } + + console.log(`${label('Slug:')} ${value(s._id)}`); + console.log(`${label('Name:')} ${value(s.name)}`); + console.log(`${label('Source:')} ${value(s.source)}`); + console.log(`${label('Skill Name:')} ${value(s.skillName)}`); + console.log(`${label('Origin:')} ${value(s.origin)}`); + if (s.description) console.log(`${label('Description:')} ${s.description}`); + console.log(`${label('Created:')} ${new Date(s.createdAt).toLocaleString()}`); + if (s.updatedAt) console.log(`${label('Updated:')} ${new Date(s.updatedAt).toLocaleString()}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +skill + .command("import") + .description("Import a skill from a GitHub repository") + .requiredOption("--source ", "GitHub repo (e.g. vercel-labs/agent-skills)") + .requiredOption("--skill-name ", "Skill name within the repo") + .requiredOption("--name ", "Display name") + .option("--description ", "Description") + .option("--origin ", "Origin: skills-sh or manual", "manual") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record = { + source: options.source, + skillName: options.skillName, + name: options.name, + origin: options.origin, + }; + if (options.description) body.description = options.description; + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const created = await response.json(); + console.log(successText(`Skill "${created._id}" imported.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +skill + .command("delete") + .description("Delete a skill (soft-delete)") + .requiredOption("-i, --id ", "Skill slug") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills/${options.id}`, { + method: "DELETE", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Skill "${options.id}" deleted.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +skill + .command("resolve") + .description("Resolve a skill from GitHub (fetch latest version and create a revision)") + .requiredOption("-i, --id ", "Skill slug") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills/${options.id}/resolve`, { + method: "POST", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const revision = await response.json(); + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'ref', label: 'Ref' }, + { key: 'commitHash', label: 'Commit' }, + { key: 'description', label: 'Description', formatter: (r: any) => r.description || '' }, + { key: 'archiveUrl', label: 'Archive URL', formatter: (r: any) => r.archiveUrl || '' }, + ]; + console.log(formatData([revision], fields, format)); + return; + } + + console.log(successText(`Skill resolved to revision:`)); + console.log(`${label('Ref:')} ${value(revision.ref)}`); + console.log(`${label('Commit:')} ${value(revision.commitHash)}`); + console.log(`${label('Description:')} ${revision.description || 'ā€”'}`); + if (revision.archiveUrl) console.log(`${label('Archive:')} ${value(revision.archiveUrl)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +skill + .command("revisions") + .description("List revisions for a skill") + .requiredOption("-i, --id ", "Skill slug") + .option("--limit ", "Maximum results", parseInt) + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = options.limit ? `?limit=${options.limit}` : ''; + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/skills/${options.id}/revisions${params}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const revisions = await response.json() as Array<{ ref: string; commitHash: string; name: string; resolvedAt: string }>; + if (revisions.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No revisions found.")); + return; + } + if (!isMachineReadable(format)) { + console.log(label(`Found ${revisions.length} revision(s):\n`)); + } + const displayFields: DisplayField[] = [ + { key: 'ref', label: 'Ref', tableFormatter: (r: any) => value(r.ref) }, + { key: 'commitHash', label: 'Commit', formatter: (r: any) => r.commitHash.substring(0, 8) }, + { key: 'name', label: 'Name' }, + { key: 'resolvedAt', label: 'Resolved', formatter: (r: any) => new Date(r.resolvedAt).toLocaleString() }, + ]; + console.log(formatData(revisions, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Insight management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const insight = program + .command("insight") + .description("Manage insights discovered during report analysis") + .action(() => { + insight.help(); + }); + +configureHelp(insight); + +withOutputOption( +insight + .command("list") + .description("List all insights") + .option("-q, --query ", "Search by keyword") + .option("--blocked", "Show only blocked insights") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = new URLSearchParams(); + if (options.query) params.set("q", options.query); + if (options.blocked) params.set("blocked", "true"); + const qs = params.toString(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights${qs ? `?${qs}` : ""}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const insights = await response.json(); + if (insights.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No insights found.")); + return; + } + const displayFields: DisplayField[] = [ + { key: '_id', label: 'ID', tableFormatter: (r: any) => value(r._id.slice(0, 8) + 'ā€¦') }, + { key: 'title', label: 'Title', formatter: (r: any) => r.title.length > 60 ? r.title.slice(0, 57) + 'ā€¦' : r.title }, + { key: 'category', label: 'Category', formatter: (r: any) => r.category || 'ā€”' }, + { key: 'referenceCount', label: 'Refs', formatter: (r: any) => String(r.referenceCount) }, + { key: 'votes', label: 'Votes', formatter: (r: any) => String(r.upvotes - r.downvotes) }, + { key: 'blocked', label: 'Blocked', formatter: (r: any) => r.blocked ? 'āœ—' : '' }, + { key: 'createdBy', label: 'Source' }, + ]; + console.log(formatData(insights, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +insight + .command("get") + .description("Get details of an insight (renders markdown description)") + .requiredOption("-i, --id ", "Insight ID") + .option("-u, --url ", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +, ['markdown']) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}`); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const insightDoc = await response.json(); + + if (format === 'markdown') { + console.log(insightDoc.description); + return; + } + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: '_id', label: 'ID' }, + { key: 'title', label: 'Title' }, + { key: 'category', label: 'Category', formatter: (r: any) => r.category || '' }, + { key: 'tags', label: 'Tags', formatter: (r: any) => (r.tags ?? []).join(', ') }, + { key: 'votes', label: 'Votes', formatter: (r: any) => String(r.upvotes - r.downvotes) }, + { key: 'referenceCount', label: 'References', formatter: (r: any) => String(r.referenceCount) }, + { key: 'blocked', label: 'Blocked', formatter: (r: any) => r.blocked ? 'Yes' : 'No' }, + { key: 'createdBy', label: 'Created By' }, + { key: 'description', label: 'Description' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + ]; + console.log(formatData([insightDoc], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(insightDoc._id)}`); + console.log(`${label('Title:')} ${value(insightDoc.title)}`); + if (insightDoc.category) console.log(`${label('Category:')} ${insightDoc.category}`); + if (insightDoc.tags?.length) console.log(`${label('Tags:')} ${insightDoc.tags.join(', ')}`); + console.log(`${label('Votes:')} ā–²${insightDoc.upvotes} ā–¼${insightDoc.downvotes} (net: ${insightDoc.upvotes - insightDoc.downvotes})`); + console.log(`${label('References:')} ${insightDoc.referenceCount} reports`); + console.log(`${label('Blocked:')} ${insightDoc.blocked ? 'Yes' : 'No'}`); + console.log(`${label('Created by:')} ${insightDoc.createdBy}`); + if (insightDoc.sourceReportId) console.log(`${label('Source report:')} ${insightDoc.sourceReportId}`); + console.log(`${label('Created:')} ${new Date(insightDoc.createdAt).toLocaleString()}`); + if (insightDoc.updatedAt) console.log(`${label('Updated:')} ${new Date(insightDoc.updatedAt).toLocaleString()}`); + console.log(`\n${label('Description:')}\n${insightDoc.description}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("create") + .description("Create a new insight") + .requiredOption("--title ", "Short summary (one line)") + .requiredOption("--description <description>", "Markdown description") + .option("--category <category>", "Category tag") + .option("--tags <tags>", "Comma-separated tags") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record<string, unknown> = { + title: options.title, + description: options.description, + createdBy: "user", + }; + if (options.category) body.category = options.category; + if (options.tags) body.tags = options.tags.split(",").map((t: string) => t.trim()).filter(Boolean); + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const created = await response.json(); + console.log(successText(`Insight created: ${created._id}`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("update") + .description("Update an insight") + .requiredOption("-i, --id <id>", "Insight ID") + .option("--title <title>", "New title") + .option("--description <description>", "New markdown description") + .option("--category <category>", "New category") + .option("--tags <tags>", "New comma-separated tags") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const body: Record<string, unknown> = {}; + if (options.title) body.title = options.title; + if (options.description) body.description = options.description; + if (options.category) body.category = options.category; + if (options.tags) body.tags = options.tags.split(",").map((t: string) => t.trim()).filter(Boolean); + if (Object.keys(body).length === 0) { + console.error(errorText("Error: provide at least one field to update")); + process.exit(1); + } + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Insight ${options.id} updated.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("delete") + .description("Delete an insight (soft-delete)") + .requiredOption("-i, --id <id>", "Insight ID") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}`, { + method: "DELETE", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Insight ${options.id} deleted.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("upvote") + .description("Upvote an insight") + .requiredOption("-i, --id <id>", "Insight ID") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}/upvote`, { + method: "POST", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const updated = await response.json(); + console.log(successText(`Upvoted. Score: ā–²${updated.upvotes} ā–¼${updated.downvotes}`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("downvote") + .description("Downvote an insight") + .requiredOption("-i, --id <id>", "Insight ID") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}/downvote`, { + method: "POST", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + const updated = await response.json(); + console.log(successText(`Downvoted. Score: ā–²${updated.upvotes} ā–¼${updated.downvotes}`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("block") + .description("Block an insight") + .requiredOption("-i, --id <id>", "Insight ID") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}/block`, { + method: "POST", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Insight ${options.id} blocked.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +insight + .command("unblock") + .description("Unblock an insight") + .requiredOption("-i, --id <id>", "Insight ID") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/insights/${encodeURIComponent(options.id)}/unblock`, { + method: "POST", + }); + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + console.log(successText(`Insight ${options.id} unblocked.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +// ā”€ā”€ā”€ Task Prompt management ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const taskPrompt = program + .command("task-prompt") + .description("Manage task prompts (content-addressed, immutable prompt entities)") + .action(() => { + taskPrompt.help(); + }); + +configureHelp(taskPrompt); + +withOutputOption( +taskPrompt + .command("list") + .description("List all task prompts") + .option("-s, --search <search>", "Filter by text content") + .option("-l, --limit <n>", "Maximum number of results", "50") + .option("--offset <n>", "Number of results to skip", "0") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const params = new URLSearchParams(); + if (options.search) params.set("search", options.search); + if (options.limit) params.set("limit", options.limit); + if (options.offset) params.set("offset", options.offset); + const qs = params.toString(); + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts${qs ? `?${qs}` : ""}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const data = await response.json() as { items: Array<{ _id: string; text: string; features?: Array<{ featureId: string; detected: boolean; evaluated: boolean }>; createdAt: string }>; total: number }; + if (data.items.length === 0) { + if (!isMachineReadable(format)) console.log(warnBanner("No task prompts found.")); + return; + } + + if (!isMachineReadable(format)) { + console.log(label(`Found ${data.items.length} of ${data.total} task prompts:\n`)); + } + + const displayFields: DisplayField[] = [ + { key: '_id', label: 'ID', + formatter: (tp: any) => tp._id.substring(0, 8) + 'ā€¦', + tableFormatter: (tp: any) => value(tp._id.substring(0, 8) + 'ā€¦'), + }, + { key: 'text', label: 'Text', formatter: (tp: any) => { + const text = tp.text.replace(/\n/g, ' '); + return text.length > 60 ? text.substring(0, 60) + 'ā€¦' : text; + }, tableFormatter: (tp: any) => { + const text = tp.text.replace(/\n/g, ' '); + const truncated = text.length > 60 ? text.substring(0, 60) + 'ā€¦' : text; + return dimTimestamp(truncated); + }}, + { key: 'features', label: 'Features', formatter: (tp: any) => { + if (!tp.features) return 'ā€”'; + const detected = tp.features.filter((f: any) => f.detected).length; + return `${detected}/${tp.features.length}`; + }}, + { key: 'createdAt', label: 'Created', formatter: (tp: any) => tp.createdAt ? new Date(tp.createdAt).toLocaleDateString() : 'ā€”' }, + ]; + + console.log(formatData(data.items, displayFields, format)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +taskPrompt + .command("get") + .description("Get details of a single task prompt") + .requiredOption("-i, --id <id>", "Task prompt ID (UUID)") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts/${encodeURIComponent(options.id)}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const tp = await response.json() as { + _id: string; text: string; + features?: Array<{ featureId: string; detected: boolean; evaluated: boolean }>; + featuresExtractedAt?: string; + createdAt: string; deletedAt?: string; + }; + + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: '_id', label: 'ID' }, + { key: 'text', label: 'Text' }, + { key: 'features', label: 'Features', formatter: (item: any) => { + if (!item.features) return '(not extracted)'; + const detected = item.features.filter((f: any) => f.detected).length; + return `${detected}/${item.features.length} detected`; + }}, + { key: 'featuresExtractedAt', label: 'Extracted At' }, + { key: 'createdAt', label: 'Created' }, + { key: 'deletedAt', label: 'Deleted' }, + ]; + console.log(formatData([tp], fields, format)); + return; + } + + console.log(`${label('ID:')} ${value(tp._id)}`); + console.log(`${label('Created:')} ${value(tp.createdAt)}`); + if (tp.deletedAt) console.log(`${label('Deleted:')} ${value(tp.deletedAt)}`); + console.log(`${label('Text:')}`); + for (const line of tp.text.trim().split('\n')) { + console.log(` ${line}`); + } + + if (tp.features && tp.features.length > 0) { + const detected = tp.features.filter(f => f.detected); + const notDetected = tp.features.filter(f => !f.detected && f.evaluated); + const skipped = tp.features.filter(f => !f.evaluated); + + console.log(`\n${label('Features:')} ${value(String(detected.length))} detected, ${dimTimestamp(String(notDetected.length))} not detected, ${dimTimestamp(String(skipped.length))} skipped`); + if (tp.featuresExtractedAt) console.log(`${label('Extracted:')} ${value(tp.featuresExtractedAt)}`); + + if (detected.length > 0) { + console.log(` ${successText('Detected:')}`); + for (const r of detected) { + console.log(` ${criterionIcon(true, true)} ${value(r.featureId)}`); + } + } + if (notDetected.length > 0) { + console.log(` ${dimTimestamp('Not detected:')}`); + for (const r of notDetected) { + console.log(` ${criterionIcon(true, false)} ${dimTimestamp(r.featureId)}`); + } + } + } else { + console.log(`\n${label('Features:')} ${dimTimestamp('(not extracted)')}`); + } + + console.log(`\n${label('View runs:')} ${dimTimestamp(`pnpm cli run list --task-prompt-id ${tp._id}`)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +taskPrompt + .command("create") + .description("Register a task prompt (idempotent ā€” same text returns existing entity)") + .option("-t, --text <text>", "Task prompt text") + .option("-f, --file <path>", "Read task prompt text from file") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + let text = options.text; + if (!text && options.file) { + const absPath = resolve(options.file); + if (!existsSync(absPath)) { + console.error(errorText(`File not found: ${absPath}`)); + process.exit(1); + } + text = readFileSync(absPath, 'utf-8'); + } + if (!text) { + console.error(errorText("Error: provide --text or --file")); + process.exit(1); + } + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const tp = await response.json() as { _id: string; text: string; createdAt: string }; + console.log(successText(`Task prompt registered.`)); + console.log(`${label('ID:')} ${value(tp._id)}`); + console.log(`${label('Created:')} ${value(tp.createdAt)}`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +taskPrompt + .command("delete") + .description("Soft-delete a task prompt") + .requiredOption("-i, --id <id>", "Task prompt ID (UUID)") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") + .action(async (options) => { + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts/${encodeURIComponent(options.id)}`, { + method: "DELETE", + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + console.log(successText(`Task prompt ${options.id} deleted.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +withOutputOption( +taskPrompt + .command("extract-features") + .description("Extract prompt features for a task prompt") + .requiredOption("-i, --id <id>", "Task prompt ID (UUID)") + .option("--model <model>", "LLM model to use for extraction") + .option("--force", "Force re-extraction even if already extracted") + .option("-u, --url <url>", "API base URL", process.env.SCOPE_MT_API_URL || "http://localhost:3100") +) + .action(async (options) => { + const format = (options.output || 'table') as OutputFormat; + try { + if (!isMachineReadable(format)) { + console.log(`${label('Extracting prompt features for task prompt')} ${value(options.id)}${label('...')}`); + } + + const qs = options.force ? "?force=true" : ""; + const body: Record<string, unknown> = {}; + if (options.model) body.model = options.model; + + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/task-prompts/${encodeURIComponent(options.id)}/extract-features${qs}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const extraction = await response.json() as { + taskPromptId: string; + features: Array<{ featureId: string; detected: boolean; evaluated: boolean }>; + featuresExtractedAt: string; + cached: boolean; + }; + + if (isMachineReadable(format)) { + console.log(format === 'json' ? JSON.stringify(extraction, null, 2) : format === 'yaml' ? yamlStringify(extraction).trimEnd() : JSON.stringify(extraction)); + return; + } + + const detected = extraction.features.filter(r => r.detected); + const notDetected = extraction.features.filter(r => !r.detected && r.evaluated); + const skipped = extraction.features.filter(r => !r.evaluated); + + if (extraction.cached) { + console.log(dimTimestamp('(cached ā€” use --force to re-extract)')); + } + + console.log(`\n${label('Results:')}`); + if (detected.length > 0) { + console.log(` ${successText('Detected:')}`); + for (const r of detected) { + console.log(` ${criterionIcon(true, true)} ${value(r.featureId)}`); + } + } + if (notDetected.length > 0) { + console.log(` ${dimTimestamp('Not detected:')}`); + for (const r of notDetected) { + console.log(` ${criterionIcon(true, false)} ${dimTimestamp(r.featureId)}`); + } + } + if (skipped.length > 0) { + console.log(` ${warnBanner('Skipped (not evaluated):')}`); + for (const r of skipped) { + console.log(` ā—‹ ${dimTimestamp(r.featureId)}`); + } + } + + console.log(`\n${label('Summary:')} ${value(String(detected.length))} detected, ${dimTimestamp(String(notDetected.length))} not detected, ${dimTimestamp(String(skipped.length))} skipped`); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } + }); + +program.parse(); + diff --git a/apps/cli/src/run-get-action.ts b/apps/cli/src/run-get-action.ts new file mode 100644 index 000000000..2c20218ad --- /dev/null +++ b/apps/cli/src/run-get-action.ts @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Action handler for `run get` subcommand ā€” extracted for testability. + */ +import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon } from "./utils/style.js"; +import { formatData, isMachineReadable } from "./utils/formatters.js"; +import type { OutputFormat, DisplayField } from "./utils/types.js"; + +/** Strip trailing slashes from a URL */ +const normalizeUrl = (url: string): string => url.replace(/\/+$/, ''); + +export interface RunGetOptions { + id: string; + url: string; + output: string; +} + +export async function runGetAction(options: RunGetOptions): Promise<void> { + const format = (options.output || 'table') as OutputFormat; + try { + const response = await fetch(`${normalizeUrl(options.url)}/api/v1/requests/${options.id}`); + + if (!response.ok) { + const error = await response.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exit(1); + } + + const run = await response.json(); + + // Machine-readable output + if (isMachineReadable(format)) { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'workerType', label: 'Worker' }, + { key: 'model', label: 'Model' }, + { key: 'status', label: 'Status' }, + { key: 'maxIterations', label: 'Max Iterations' }, + { key: 'turnsCount', label: 'Turns' }, + { key: 'passed', label: 'Passed' }, + { key: 'task', label: 'Task' }, + { key: 'criteriaCount', label: 'Criteria' }, + { key: 'logsCount', label: 'Logs' }, + { key: 'createdAt', label: 'Created' }, + { key: 'updatedAt', label: 'Updated' }, + { key: 'error', label: 'Error' }, + ]; + const row = { + ...run, + turnsCount: run.turns?.length ?? 0, + passed: run.status === 'completed' ? 'yes' : run.status === 'failed' || run.status === 'exhausted' ? 'no' : '-', + task: run.scenario?.task ?? '', + criteriaCount: run.scenario?.criteria?.length ?? 0, + logsCount: run.logs?.length ?? 0, + }; + console.log(formatData([row], fields, format)); + return; + } + + // Human-readable output + console.log(`${label('ID:')} ${value(run.id)}`); + console.log(`${label('Worker:')} ${value(run.workerType)}`); + if (run.model) console.log(`${label('Model:')} ${value(run.model)}`); + + const statusColor = run.status === 'completed' ? successText + : (run.status === 'failed' || run.status === 'exhausted') ? errorText + : value; + console.log(`${label('Status:')} ${statusColor(run.status)}`); + + if (run.maxIterations != null) console.log(`${label('Max Iterations:')} ${value(String(run.maxIterations))}`); + if (run.createdAt) console.log(`${label('Created:')} ${dimTimestamp(new Date(run.createdAt).toLocaleString())}`); + if (run.updatedAt) console.log(`${label('Updated:')} ${dimTimestamp(new Date(run.updatedAt).toLocaleString())}`); + if (run.completedAt) console.log(`${label('Completed:')} ${dimTimestamp(new Date(run.completedAt).toLocaleString())}`); + + // Persona + if (run.persona) { + const p = run.persona; + console.log(`${label('Persona:')} ${value(`${p.personality} / ${p.experience} / ${p.verbosity} / ${p.type}`)}`); + } + + // Scenario + if (run.scenario) { + console.log(`${label('Task:')}`); + for (const line of (run.scenario.task ?? '').trim().split('\n')) { + console.log(` ${line}`); + } + if (run.scenario.criteria?.length > 0) { + console.log(`${label('Criteria:')} ${value(String(run.scenario.criteria.length))} criterion/criteria`); + // Build a lookup from the last turn's criteria results + const lastTurn = run.turns?.length ? run.turns[run.turns.length - 1] : null; + const resultsMap = new Map<string, { passed: boolean; evaluated: boolean }>(); + if (lastTurn?.criteriaResults) { + for (const cr of lastTurn.criteriaResults) { + resultsMap.set(cr.criterionId, { passed: cr.passed, evaluated: cr.evaluated }); + } + } + for (const c of run.scenario.criteria) { + const r = resultsMap.get(c); + const icon = r ? criterionIcon(r.evaluated, r.passed) : ' '; + console.log(` ${icon} ${c}`); + } + } + } + + // Turns summary + if (run.turns?.length > 0) { + console.log(`\n${banner('ā”€ā”€ā”€ Turns ā”€ā”€ā”€')}`); + for (const turn of run.turns) { + const passIcon = criterionIcon(true, turn.passed); + const criteriaStr = turn.criteriaResults?.length + ? ` ā€” ${turn.criteriaResults.filter((cr: { passed: boolean }) => cr.passed).length}/${turn.criteriaResults.length} criteria passed` + : ''; + console.log(` ${label(`Turn ${turn.iteration}:`)} ${passIcon}${criteriaStr}`); + } + } + + // Error + if (run.error) { + console.log(`${label('Error:')} ${errorText(run.error)}`); + } + + // Logs count + if (run.logs?.length != null) { + console.log(`${label('Logs:')} ${value(String(run.logs.length))} entries`); + } + + // Prompt feature extraction + if (run.promptFeatureExtractionId) { + console.log(`${label('Prompt Features:')} ${value(run.promptFeatureExtractionId)}`); + } + + // Soft-deleted + if (run.deletedAt) { + console.log(`${label('Deleted:')} ${warnBanner(new Date(run.deletedAt).toLocaleString())}`); + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exit(1); + } +} diff --git a/apps/cli/src/run-get.test.ts b/apps/cli/src/run-get.test.ts new file mode 100644 index 000000000..6bc1f7422 --- /dev/null +++ b/apps/cli/src/run-get.test.ts @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; + +// ā”€ā”€ Fixtures ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** Minimal run ā€” only required fields, no turns/persona/model */ +const minimalRun = { + id: 'run-001', + workerType: 'coder-acp-copilot', + status: 'pending', + scenario: { task: 'Add a button', criteria: ['has_button'] }, + logs: [], + createdAt: '2025-06-01T10:00:00Z', +}; + +/** Full run with all optional fields populated */ +const fullRun = { + id: 'run-002', + workerType: 'coder-acp-claude-code', + model: 'claude-sonnet-4-20250514', + status: 'completed', + scenario: { + version: 'v2' as const, + task: 'Implement login form\nwith validation', + criteria: ['has_form', 'has_validation', 'has_tests'], + }, + maxIterations: 3, + persona: { + personality: 'demanding' as const, + experience: 'senior' as const, + verbosity: 'brief' as const, + type: 'traditional' as const, + }, + turns: [ + { + iteration: 1, + codingAgentResponse: 'code v1', + judgeFeedback: 'needs tests', + snapshotUrl: 'https://snap/1', + passed: false, + timestamp: '2025-06-01T10:05:00Z', + criteriaResults: [ + { criterionId: 'has_form', passed: true, evaluated: true }, + { criterionId: 'has_validation', passed: true, evaluated: true }, + { criterionId: 'has_tests', passed: false, evaluated: true }, + ], + }, + { + iteration: 2, + codingAgentResponse: 'code v2', + judgeFeedback: 'all good', + snapshotUrl: 'https://snap/2', + passed: true, + timestamp: '2025-06-01T10:10:00Z', + criteriaResults: [ + { criterionId: 'has_form', passed: true, evaluated: true }, + { criterionId: 'has_validation', passed: true, evaluated: true }, + { criterionId: 'has_tests', passed: true, evaluated: true }, + ], + }, + ], + logs: [ + { timestamp: '2025-06-01T10:00:00Z', level: 'info', message: 'started' }, + { timestamp: '2025-06-01T10:10:00Z', level: 'info', message: 'done' }, + ], + createdAt: '2025-06-01T10:00:00Z', + updatedAt: '2025-06-01T10:10:00Z', + promptFeatureExtractionId: 'pfe-abc', +}; + +// ā”€ā”€ Test helpers ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Build a stripped-down Commander program that registers only the `run get` + * subcommand so we can invoke it programmatically. + * + * We dynamically import the real CLI module to pick up the registered command, + * but that also boots every other command and runs dotenv, etc. Instead we + * replicate the minimal wiring here ā€” it tests the same display logic because + * we call it through Commander's `.parseAsync()`. + */ +async function buildRunGetCommand() { + // Re-import the module each time so vi.stubGlobal('fetch') is picked up. + // Dynamic import with cache-bust to avoid stale module. + const mod = await import('./index.js'); + // The default export is the program; grab the 'run' parent command. + // Commander nests: program ā†’ run ā†’ get + const program: Command = (mod as unknown as { default: Command }).default ?? (mod as unknown as { program: Command }).program; + + // If the module doesn't export program directly, we build our own mini program + // that mirrors the real `run get` logic ā€” but that defeats the purpose. + // Instead, we test the output formatting inline. + return program; +} + +/** Capture all console.log calls and return them joined by newlines. */ +function captureConsole() { + const lines: string[] = []; + const spy = vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(' ')); + }); + const errLines: string[] = []; + const errSpy = vi.spyOn(console, 'error').mockImplementation((...args: unknown[]) => { + errLines.push(args.map(String).join(' ')); + }); + return { + lines, + errLines, + restore: () => { spy.mockRestore(); errSpy.mockRestore(); }, + }; +} + +// Stub process.exit to prevent vitest from dying +const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => { + throw new Error('process.exit called'); +}) as never); + +// ā”€ā”€ Tests ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('run get', () => { + beforeEach(() => { + vi.stubGlobal('fetch', vi.fn()); + exitSpy.mockClear(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('human-readable output', () => { + it('displays all fields for a full run', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(fullRun), + }); + vi.stubGlobal('fetch', mockFetch); + + const { lines, restore } = captureConsole(); + try { + // Import fresh to pick up stubbed fetch + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output: 'table' }); + } finally { + restore(); + } + + const output = lines.join('\n'); + + // Core fields + expect(output).toContain('run-002'); + expect(output).toContain('coder-acp-claude-code'); + expect(output).toContain('claude-sonnet-4-20250514'); + expect(output).toContain('completed'); + + // Persona + expect(output).toContain('demanding'); + expect(output).toContain('senior'); + + // Task + expect(output).toContain('Implement login form'); + + // Criteria listed + expect(output).toContain('has_form'); + expect(output).toContain('has_validation'); + expect(output).toContain('has_tests'); + + // Turns summary + expect(output).toContain('Turn 1'); + expect(output).toContain('Turn 2'); + expect(output).toContain('3/3 criteria passed'); + + // Logs count + expect(output).toContain('2'); + expect(output).toContain('entries'); + + // Prompt feature extraction + expect(output).toContain('pfe-abc'); + }); + + it('displays minimal run without optional fields', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(minimalRun), + }); + vi.stubGlobal('fetch', mockFetch); + + const { lines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-001', url: 'http://localhost:3100', output: 'table' }); + } finally { + restore(); + } + + const output = lines.join('\n'); + + expect(output).toContain('run-001'); + expect(output).toContain('coder-acp-copilot'); + expect(output).toContain('pending'); + expect(output).toContain('Add a button'); + + // Should NOT contain optional fields + expect(output).not.toContain('Model:'); + expect(output).not.toContain('Persona:'); + expect(output).not.toContain('Turn '); + expect(output).not.toContain('Prompt Features:'); + }); + }); + + describe('JSON output', () => { + it('outputs the full run as formatted JSON', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(fullRun), + }); + vi.stubGlobal('fetch', mockFetch); + + const { lines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output: 'json' }); + } finally { + restore(); + } + + const output = lines.join('\n'); + // Should contain valid JSON with key fields + expect(output).toContain('"id"'); + expect(output).toContain('"run-002"'); + expect(output).toContain('"workerType"'); + }); + }); + + describe('error handling', () => { + it('exits with error on 404', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: false, + json: () => Promise.resolve({ error: 'Request not found' }), + }); + vi.stubGlobal('fetch', mockFetch); + + const { errLines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'nonexistent', url: 'http://localhost:3100', output: 'table' }); + } catch { + // process.exit throws + } finally { + restore(); + } + + const errOutput = errLines.join('\n'); + expect(errOutput).toContain('Request not found'); + }); + + it('exits with error on network failure', async () => { + const mockFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED')); + vi.stubGlobal('fetch', mockFetch); + + const { errLines, restore } = captureConsole(); + try { + const { runGetAction } = await import('./run-get-action.js'); + await runGetAction({ id: 'run-001', url: 'http://localhost:3100', output: 'table' }); + } catch { + // process.exit throws + } finally { + restore(); + } + + const errOutput = errLines.join('\n'); + expect(errOutput).toContain('ECONNREFUSED'); + }); + }); +}); diff --git a/apps/cli/src/utils/formatters.test.ts b/apps/cli/src/utils/formatters.test.ts new file mode 100644 index 000000000..9851c0785 --- /dev/null +++ b/apps/cli/src/utils/formatters.test.ts @@ -0,0 +1,358 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect } from 'vitest'; +import { + formatAsTable, + formatAsTSV, + formatAsJSON, + formatAsYAML, + formatData, + isMachineReadable, + getValueOrNA, + formatDate, + maxColumnWidth, +} from './formatters.js'; +import type { DisplayField, TableColumn } from './types.js'; + +// ā”€ā”€ Helper data ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const sampleItems = [ + { id: 'abc-123', name: 'First', status: 'completed' }, + { id: 'def-456', name: 'Second', status: 'failed' }, + { id: 'ghi-789', name: 'Third', status: 'pending' }, +]; + +const displayFields: DisplayField[] = [ + { key: 'id', label: 'ID' }, + { key: 'name', label: 'Name' }, + { key: 'status', label: 'Status' }, +]; + +// ā”€ā”€ getValueOrNA ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('getValueOrNA', () => { + it('returns string value for non-null input', () => { + expect(getValueOrNA('hello')).toBe('hello'); + expect(getValueOrNA(42)).toBe('42'); + expect(getValueOrNA(true)).toBe('true'); + }); + + it('returns N/A for null, undefined, and empty string', () => { + expect(getValueOrNA(null)).toBe('N/A'); + expect(getValueOrNA(undefined)).toBe('N/A'); + expect(getValueOrNA('')).toBe('N/A'); + }); +}); + +// ā”€ā”€ formatDate ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('formatDate', () => { + it('formats a valid ISO date string', () => { + const result = formatDate('2025-01-15T10:30:00Z'); + expect(result).not.toBe('N/A'); + expect(result.length).toBeGreaterThan(0); + }); + + it('returns N/A for undefined', () => { + expect(formatDate(undefined)).toBe('N/A'); + }); + + it('returns N/A for empty string', () => { + expect(formatDate('')).toBe('N/A'); + }); +}); + +// ā”€ā”€ maxColumnWidth ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('maxColumnWidth', () => { + it('calculates width from data', () => { + const data = [{ name: 'abc' }, { name: 'abcdef' }]; + // 'abcdef' = 6 chars + 2 padding = 8 + expect(maxColumnWidth(data, 'name')).toBe(8); + }); + + it('respects minWidth', () => { + const data = [{ name: 'ab' }]; + // minWidth 10 + padding 2 = 12 + expect(maxColumnWidth(data, 'name', 10)).toBe(12); + }); + + it('handles empty arrays', () => { + expect(maxColumnWidth([], 'name')).toBe(12); // default 10 + 2 padding + }); +}); + +// ā”€ā”€ formatAsTable ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('formatAsTable', () => { + it('produces bordered table output', () => { + const columns: TableColumn[] = [ + { key: 'id', header: 'ID' }, + { key: 'name', header: 'Name' }, + { key: 'status', header: 'Status' }, + ]; + + const result = formatAsTable(sampleItems, columns); + + // Check headers present + expect(result).toContain('ID'); + expect(result).toContain('Name'); + expect(result).toContain('Status'); + + // Check data present + expect(result).toContain('abc-123'); + expect(result).toContain('Second'); + expect(result).toContain('pending'); + + // Check table borders (cli-table3 uses ā”€, ā”‚, etc.) + expect(result).toContain('ā”€'); + }); + + it('uses custom formatters', () => { + const columns: TableColumn[] = [ + { key: 'id', header: 'ID', formatter: (item) => `[${item.id}]` }, + { key: 'status', header: 'Status' }, + ]; + + const result = formatAsTable(sampleItems, columns); + expect(result).toContain('[abc-123]'); + }); + + it('prefers tableFormatter over formatter for table output', () => { + const columns: TableColumn[] = [ + { + key: 'id', + header: 'ID', + formatter: (item) => item.id, + tableFormatter: (item) => `**${item.id}**`, + }, + ]; + + const result = formatAsTable(sampleItems, columns); + expect(result).toContain('**abc-123**'); + expect(result).not.toMatch(/ā”‚\s*abc-123\s*ā”‚/); // plain version should not appear + }); + + it('handles empty array', () => { + const columns: TableColumn[] = [ + { key: 'id', header: 'ID' }, + ]; + + const result = formatAsTable([], columns); + // Should produce table with headers but no data rows + expect(result).toContain('ID'); + }); + + it('handles missing properties with N/A', () => { + const items = [{ id: 'test' }]; // no 'name' property + const columns: TableColumn[] = [ + { key: 'id', header: 'ID' }, + { key: 'name', header: 'Name' }, + ]; + + const result = formatAsTable(items, columns); + expect(result).toContain('N/A'); + }); +}); + +// ā”€ā”€ formatAsTSV ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('formatAsTSV', () => { + it('produces tab-separated output without headers', () => { + const result = formatAsTSV(sampleItems, displayFields); + + const lines = result.split('\n'); + expect(lines).toHaveLength(3); + + // Check tab separation + const firstLine = lines[0].split('\t'); + expect(firstLine).toHaveLength(3); + expect(firstLine[0]).toBe('abc-123'); + expect(firstLine[1]).toBe('First'); + expect(firstLine[2]).toBe('completed'); + }); + + it('uses custom formatters', () => { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID', formatter: (item) => item.id.toUpperCase() }, + ]; + + const result = formatAsTSV(sampleItems, fields); + expect(result).toContain('ABC-123'); + }); + + it('handles empty array', () => { + const result = formatAsTSV([], displayFields); + expect(result).toBe(''); + }); + + it('handles null/undefined values', () => { + const items = [{ id: 'test', name: null, status: undefined }]; + const result = formatAsTSV(items as any, displayFields); + expect(result).toContain('N/A'); + }); + + it('no trailing newline for clean piping', () => { + const result = formatAsTSV(sampleItems, displayFields); + expect(result.endsWith('\n')).toBe(false); + }); +}); + +// ā”€ā”€ formatAsJSON ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('formatAsJSON', () => { + it('produces valid JSON array', () => { + const result = formatAsJSON(sampleItems, displayFields); + const parsed = JSON.parse(result); + + expect(Array.isArray(parsed)).toBe(true); + expect(parsed).toHaveLength(3); + expect(parsed[0]).toEqual({ id: 'abc-123', name: 'First', status: 'completed' }); + }); + + it('applies custom formatters to JSON values', () => { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID', formatter: (item) => `prefix:${item.id}` }, + ]; + + const result = formatAsJSON(sampleItems, fields); + const parsed = JSON.parse(result); + expect(parsed[0].id).toBe('prefix:abc-123'); + }); + + it('handles empty array', () => { + const result = formatAsJSON([], displayFields); + const parsed = JSON.parse(result); + expect(parsed).toEqual([]); + }); + + it('is pretty-printed with 2-space indent', () => { + const result = formatAsJSON(sampleItems, displayFields); + // Pretty-printed JSON starts with [\n { + expect(result).toMatch(/^\[\n {2}\{/); + }); +}); + +// ā”€ā”€ formatAsYAML ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('formatAsYAML', () => { + it('produces valid YAML output', () => { + const result = formatAsYAML(sampleItems, displayFields); + + // Should contain keys from displayFields + expect(result).toContain('id: abc-123'); + expect(result).toContain('name: First'); + expect(result).toContain('status: completed'); + }); + + it('produces a YAML sequence (list) with one entry per item', () => { + const result = formatAsYAML(sampleItems, displayFields); + + // YAML list items start with "- " + const listEntries = result.split('\n').filter((l: string) => l.startsWith('- ')); + expect(listEntries).toHaveLength(3); + }); + + it('applies custom formatters', () => { + const fields: DisplayField[] = [ + { key: 'id', label: 'ID', formatter: (item) => `prefix:${item.id}` }, + ]; + + const result = formatAsYAML(sampleItems, fields); + expect(result).toContain('prefix:abc-123'); + }); + + it('handles empty array', () => { + const result = formatAsYAML([], displayFields); + expect(result).toBe('[]'); + }); + + it('no trailing newline for clean piping', () => { + const result = formatAsYAML(sampleItems, displayFields); + expect(result.endsWith('\n')).toBe(false); + }); + + it('handles null/undefined values', () => { + const items = [{ id: 'test', name: null, status: undefined }]; + const result = formatAsYAML(items as any, displayFields); + expect(result).toContain('N/A'); + }); +}); + +// ā”€ā”€ formatData (dispatcher) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('formatData', () => { + it('dispatches to table format', () => { + const result = formatData(sampleItems, displayFields, 'table'); + // Table format has borders + expect(result).toContain('ā”€'); + expect(result).toContain('abc-123'); + }); + + it('dispatches to tsv format', () => { + const result = formatData(sampleItems, displayFields, 'tsv'); + // TSV: no borders, tab-separated + expect(result).not.toContain('ā”€'); + expect(result).toContain('\t'); + expect(result).toContain('abc-123'); + }); + + it('dispatches to json format', () => { + const result = formatData(sampleItems, displayFields, 'json'); + const parsed = JSON.parse(result); + expect(parsed[0].id).toBe('abc-123'); + }); + + it('dispatches to yaml format', () => { + const result = formatData(sampleItems, displayFields, 'yaml'); + expect(result).toContain('id: abc-123'); + expect(result).toContain('name: First'); + }); + + it('throws on unsupported format', () => { + expect(() => formatData(sampleItems, displayFields, 'csv' as any)).toThrow( + 'Unsupported output format: csv', + ); + }); + + it('uses tableFormatter for table format, plain formatter for tsv/json/yaml', () => { + const fields: DisplayField[] = [ + { + key: 'id', + label: 'ID', + formatter: (item: any) => item.id, + tableFormatter: (item: any) => `STYLED:${item.id}`, + }, + ]; + + const tableResult = formatData(sampleItems, fields, 'table'); + expect(tableResult).toContain('STYLED:abc-123'); + + const tsvResult = formatData(sampleItems, fields, 'tsv'); + expect(tsvResult).toContain('abc-123'); + expect(tsvResult).not.toContain('STYLED:'); + + const jsonResult = formatData(sampleItems, fields, 'json'); + expect(jsonResult).toContain('abc-123'); + expect(jsonResult).not.toContain('STYLED:'); + + const yamlResult = formatData(sampleItems, fields, 'yaml'); + expect(yamlResult).toContain('abc-123'); + expect(yamlResult).not.toContain('STYLED:'); + }); +}); + +// ā”€ā”€ isMachineReadable ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe('isMachineReadable', () => { + it('returns true for tsv, json, and yaml', () => { + expect(isMachineReadable('tsv')).toBe(true); + expect(isMachineReadable('json')).toBe(true); + expect(isMachineReadable('yaml')).toBe(true); + }); + + it('returns false for table', () => { + expect(isMachineReadable('table')).toBe(false); + }); +}); diff --git a/apps/cli/src/utils/formatters.ts b/apps/cli/src/utils/formatters.ts new file mode 100644 index 000000000..ec10a1b6b --- /dev/null +++ b/apps/cli/src/utils/formatters.ts @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Central formatting engine for CLI output. + * Supports table, TSV, JSON, and YAML output formats. + * Ported from ca-geo/packages/cli reference implementation, with JSON and YAML added. + */ + +import Table from 'cli-table3'; +import { stringify } from 'csv-stringify/sync'; +import { stringify as yamlStringify } from 'yaml'; +import type { TableColumn, ListItem, OutputFormat, DisplayField } from './types.js'; + +// ā”€ā”€ Helpers ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Get value or return 'N/A' placeholder. + */ +export function getValueOrNA(value: any): string { + if (value === null || value === undefined || value === '') return 'N/A'; + return String(value); +} + +/** + * Format a date string or Date object to a locale string. + */ +export function formatDate(date?: string | Date): string { + if (!date) return 'N/A'; + try { + return new Date(date).toLocaleString(); + } catch { + return 'N/A'; + } +} + +/** + * Calculate the maximum width needed for a table column based on the data. + * + * @param data - Array of data objects + * @param field - Field name to calculate width for + * @param minWidth - Optional minimum width (defaults to data-driven width) + * @param padding - Additional padding to add (default: 2) + * @returns Maximum column width + */ +export function maxColumnWidth<T>( + data: T[], + field: keyof T | string, + minWidth?: number, + padding: number = 2, +): number { + if (data.length === 0) return (minWidth ?? 10) + padding; + const dataMaxLength = Math.max( + ...data.map((item) => { + const value = item[field as keyof T]; + return String(value || '').length; + }), + ); + const maxLength = + minWidth !== undefined ? Math.max(dataMaxLength, minWidth) : dataMaxLength; + return maxLength + padding; +} + +// ā”€ā”€ Format: Table ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Format items as a bordered table using cli-table3. + */ +export function formatAsTable<T extends ListItem>( + items: T[], + columns: TableColumn<T>[], +): string { + const hasWidths = columns.some((col) => col.width !== undefined); + const tableOptions: any = { + head: columns.map((col) => col.header), + truncate: 'ā€¦', + wordWrap: true, + }; + + if (hasWidths) { + tableOptions.colWidths = columns.map((col) => col.width ?? null); + } + + const table = new Table(tableOptions); + + for (const item of items) { + const row = columns.map((col) => { + // Prefer tableFormatter (styled) over formatter (plain) for table output + if (col.tableFormatter) return col.tableFormatter(item); + if (col.formatter) return col.formatter(item); + return getValueOrNA(item[col.key]); + }); + table.push(row); + } + + return table.toString(); +} + +// ā”€ā”€ Format: TSV ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Format items as TSV (tab-separated values) without headers. + * Clean output suitable for piping to cut, awk, grep, xargs, etc. + */ +export function formatAsTSV<T extends ListItem>( + items: T[], + fields: DisplayField<T>[], +): string { + const rows = items.map((item) => + fields.map((field) => { + if (field.formatter) return field.formatter(item); + return getValueOrNA(item[field.key]); + }), + ); + + const output = stringify(rows, { + delimiter: '\t', + header: false, + }); + + // Remove trailing newline for cleaner piping + return output.trimEnd(); +} + +// ā”€ā”€ Format: JSON ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Format items as pretty-printed JSON. + * Uses the display fields' formatters to transform values, keyed by field key. + */ +export function formatAsJSON<T extends ListItem>( + items: T[], + fields: DisplayField<T>[], +): string { + const rows = items.map((item) => { + const obj: Record<string, string> = {}; + for (const field of fields) { + obj[field.key] = field.formatter + ? field.formatter(item) + : getValueOrNA(item[field.key]); + } + return obj; + }); + return JSON.stringify(rows, null, 2); +} + +// ā”€ā”€ Format: YAML ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Format items as YAML. + * Uses the display fields' formatters to transform values, keyed by field key. + */ +export function formatAsYAML<T extends ListItem>( + items: T[], + fields: DisplayField<T>[], +): string { + const rows = items.map((item) => { + const obj: Record<string, string> = {}; + for (const field of fields) { + obj[field.key] = field.formatter + ? field.formatter(item) + : getValueOrNA(item[field.key]); + } + return obj; + }); + return yamlStringify(rows).trimEnd(); +} + +// ā”€ā”€ Dispatcher ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Format data based on output format. + * Routes to the appropriate formatter (table, tsv, json, or yaml). + */ +export function formatData<T extends ListItem>( + items: T[], + fields: DisplayField<T>[], + format: OutputFormat, +): string { + switch (format) { + case 'table': { + const columns: TableColumn<T>[] = fields.map((field) => ({ + key: field.key, + header: field.label, + width: field.width, + formatter: field.formatter, + tableFormatter: field.tableFormatter, + })); + return formatAsTable(items, columns); + } + case 'tsv': + return formatAsTSV(items, fields); + case 'json': + return formatAsJSON(items, fields); + case 'yaml': + return formatAsYAML(items, fields); + default: + throw new Error(`Unsupported output format: ${format}`); + } +} + +/** + * Check if the given format is machine-readable (tsv, json, or yaml). + * Used to suppress decorative output (banners, styled headers) for piping. + */ +export function isMachineReadable(format: OutputFormat): boolean { + return format === 'tsv' || format === 'json' || format === 'yaml'; +} diff --git a/packages/cli/src/utils/helpFormatter.ts b/apps/cli/src/utils/helpFormatter.ts similarity index 78% rename from packages/cli/src/utils/helpFormatter.ts rename to apps/cli/src/utils/helpFormatter.ts index 205f0fce5..c4441f7fa 100644 --- a/packages/cli/src/utils/helpFormatter.ts +++ b/apps/cli/src/utils/helpFormatter.ts @@ -4,6 +4,35 @@ import { Command } from 'commander'; import { styleText } from 'node:util'; +/** Output format definition for help text generation */ +export interface OutputFormatDef { + section: string; + description: string; +} + +/** + * Generate a help text block documenting available output formats. + * Appended to the main program --help via addHelpText('after', ...). + */ +export function generateOutputFormatsHelp( + formats: Record<string, OutputFormatDef>, +): string { + const sections = new Map<string, Array<{ name: string; description: string }>>(); + for (const [name, def] of Object.entries(formats)) { + if (!sections.has(def.section)) sections.set(def.section, []); + sections.get(def.section)!.push({ name, description: def.description }); + } + + let output = '\n' + styleText('bold', 'Output Formats (-o, --output):') + '\n'; + for (const [section, items] of sections) { + output += '\n ' + styleText('italic', section) + '\n'; + for (const item of items) { + output += ` ${styleText('cyan', item.name.padEnd(12))} ${item.description}\n`; + } + } + return output; +} + /** * Recursively collect all commands and subcommands from a Commander program */ diff --git a/packages/cli/src/utils/style.ts b/apps/cli/src/utils/style.ts similarity index 98% rename from packages/cli/src/utils/style.ts rename to apps/cli/src/utils/style.ts index 13d308f58..48f4697cc 100644 --- a/packages/cli/src/utils/style.ts +++ b/apps/cli/src/utils/style.ts @@ -21,7 +21,6 @@ export const LEVEL_COLORS: Record<string, Parameters<typeof styleText>[0]> = { export const WORKER_COLORS: Record<string, Parameters<typeof styleText>[0]> = { 'coder-acp-claude-code': 'cyan', 'coder-acp-copilot': 'magenta', - 'coder-vscode-web': 'blue', }; /** Format a log level with its associated color */ diff --git a/apps/cli/src/utils/types.ts b/apps/cli/src/utils/types.ts new file mode 100644 index 000000000..c150a6947 --- /dev/null +++ b/apps/cli/src/utils/types.ts @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Output format types for CLI list commands. + * Ported from ca-geo/packages/cli reference implementation. + */ + +/** Supported output formats for list commands */ +export type OutputFormat = 'table' | 'tsv' | 'json' | 'yaml' | 'markdown'; + +/** All valid output format values */ +export const OUTPUT_FORMAT_VALUES: OutputFormat[] = ['table', 'tsv', 'json', 'yaml']; + +/** Display field descriptor ā€” unified across all output formats */ +export interface DisplayField<T = any> { + /** Property key on the data object (supports nested access via formatter) */ + key: string; + /** Column header label (used in table headers and list labels) */ + label: string; + /** Optional fixed column width for table format */ + width?: number; + /** Optional custom formatter ā€” receives the full item, returns plain display string (used by tsv/json) */ + formatter?: (item: T) => string; + /** Optional styled formatter ā€” used only for table output to add colors/styling. Falls back to formatter if not provided. */ + tableFormatter?: (item: T) => string; +} + +/** Table column descriptor (used internally by formatAsTable) */ +export interface TableColumn<T = any> { + key: string; + header: string; + width?: number; + formatter?: (item: T) => string; + /** Styled formatter for colored/styled table output. Falls back to formatter if not provided. */ + tableFormatter?: (item: T) => string; +} + +/** Generic list item with flexible properties */ +export interface ListItem { + id?: string; + [key: string]: any; +} diff --git a/packages/cli/tsconfig.json b/apps/cli/tsconfig.json similarity index 100% rename from packages/cli/tsconfig.json rename to apps/cli/tsconfig.json diff --git a/Dockerfile.judge b/apps/judge/Dockerfile similarity index 57% rename from Dockerfile.judge rename to apps/judge/Dockerfile index 8f0b8650a..666da656f 100644 --- a/Dockerfile.judge +++ b/apps/judge/Dockerfile @@ -7,17 +7,28 @@ WORKDIR /app COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ COPY tsconfig.base.json ./ COPY packages/shared/package.json ./packages/shared/ -COPY packages/judge/package.json ./packages/judge/ +COPY apps/judge/package.json ./apps/judge/ RUN pnpm install --frozen-lockfile COPY packages/shared/tsconfig.json ./packages/shared/ COPY packages/shared/src ./packages/shared/src/ -COPY packages/judge/tsconfig.json ./packages/judge/ -COPY packages/judge/src ./packages/judge/src/ +COPY apps/judge/tsconfig.json ./apps/judge/ +COPY apps/judge/src ./apps/judge/src/ RUN pnpm --filter shared build && pnpm --filter judge build +# --- Dev stage: hot reload with tsx watch --- +FROM builder AS dev +ENV NODE_ENV=development +ENV SERVICE_DIR=judge +COPY scripts/dev-entrypoint.sh /app/scripts/dev-entrypoint.sh +COPY config /app/config/ +EXPOSE 80 +ENV PORT=80 +CMD ["/app/scripts/dev-entrypoint.sh"] + +# --- Production stage --- FROM node:22.22.0-slim RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* @@ -27,21 +38,23 @@ WORKDIR /app COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ COPY packages/shared/package.json ./packages/shared/ -COPY packages/judge/package.json ./packages/judge/ +COPY apps/judge/package.json ./apps/judge/ RUN pnpm install --frozen-lockfile --prod COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ -COPY --from=builder /app/packages/judge/dist ./packages/judge/dist/ +COPY --from=builder /app/apps/judge/dist ./apps/judge/dist/ -# Copy criteria definitions -COPY config/criteria ./config/criteria/ +# NOTE: Criteria are loaded at runtime via CRITERIA_API_URL (REST API) or +# CRITERIA_DIR (filesystem fallback). No need to bake criteria into the image. -WORKDIR /app/packages/judge +WORKDIR /app/apps/judge EXPOSE 80 ENV PORT=80 +# In production, set CRITERIA_API_URL to use REST API provider. +# CRITERIA_DIR is kept as fallback for local dev with mounted criteria. ENV CRITERIA_DIR=/app/config/criteria ENV JUDGE_STRATEGY=bundled ENV JUDGE_MAX_PARALLELISM=3 diff --git a/packages/judge/package.json b/apps/judge/package.json similarity index 100% rename from packages/judge/package.json rename to apps/judge/package.json diff --git a/packages/judge/src/feedback-generator.ts b/apps/judge/src/feedback-generator.ts similarity index 93% rename from packages/judge/src/feedback-generator.ts rename to apps/judge/src/feedback-generator.ts index edd9a8297..475de9758 100644 --- a/packages/judge/src/feedback-generator.ts +++ b/apps/judge/src/feedback-generator.ts @@ -2,12 +2,11 @@ // Licensed under the MIT License. import { CopilotClient, SessionEvent } from "@github/copilot-sdk"; -import { CriteriaConfig, CriterionResult } from "shared"; -import { CriteriaGraph } from "shared/criteria-graph"; +import { CriteriaConfig, CriterionResult, DependencyGraph, TokenManagerClient } from "shared"; export interface FeedbackContext { judgeResults: CriterionResult[]; - criteriaGraph: CriteriaGraph; + criteriaGraph: DependencyGraph; criteriaRegistry: Map<string, CriteriaConfig>; personaInstructions?: string; maxCriteria?: number; @@ -55,9 +54,11 @@ Examples of bad feedback: */ export class FeedbackGenerator { private model: string; + private tokenClient: TokenManagerClient; constructor(model?: string) { this.model = model || process.env.FEEDBACK_MODEL || "gpt-4.1"; + this.tokenClient = new TokenManagerClient(); } async generateFeedback( @@ -119,7 +120,7 @@ export class FeedbackGenerator { private buildSystemPrompt( baseInstructions: string, selectedCriteriaIds: string[], - criteriaGraph: CriteriaGraph, + criteriaGraph: DependencyGraph, criteriaRegistry: Map<string, CriteriaConfig>, includeDescendantGuard: boolean ): string { @@ -148,7 +149,7 @@ export class FeedbackGenerator { private getDescendantPrompts( criteriaIds: string[], - graph: CriteriaGraph, + graph: DependencyGraph, registry: Map<string, CriteriaConfig> ): string[] { const descendantPrompts: string[] = []; @@ -177,7 +178,8 @@ export class FeedbackGenerator { systemPrompt: string, failureContext: string ): Promise<string> { - const client = new CopilotClient(); + const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); + const client = new CopilotClient({ githubToken }); let fullResponse = ""; try { diff --git a/packages/judge/src/index.ts b/apps/judge/src/index.ts similarity index 98% rename from packages/judge/src/index.ts rename to apps/judge/src/index.ts index a69fbb417..17ed66900 100644 --- a/packages/judge/src/index.ts +++ b/apps/judge/src/index.ts @@ -28,7 +28,7 @@ const blobStorage = new BlobStorage({ // Redis-only log publisher for real-time criterion progress (optional ā€” no-op if Redis not configured) const redisHost = process.env.REDIS_HOST || ""; -const redisPort = parseInt(process.env.REDIS_PORT || "6379", 10); +const redisPort = parseInt(process.env.REDIS_PORT || "6300", 10); const redisPassword = process.env.REDIS_PASSWORD || ""; let logPublisher: RedisLogPublisher | null = null; diff --git a/packages/judge/src/judge-agent-v1-backup.ts b/apps/judge/src/judge-agent-v1-backup.ts similarity index 100% rename from packages/judge/src/judge-agent-v1-backup.ts rename to apps/judge/src/judge-agent-v1-backup.ts diff --git a/packages/judge/src/judge-agent.ts b/apps/judge/src/judge-agent.ts similarity index 91% rename from packages/judge/src/judge-agent.ts rename to apps/judge/src/judge-agent.ts index bc545048f..24d86c4f7 100644 --- a/packages/judge/src/judge-agent.ts +++ b/apps/judge/src/judge-agent.ts @@ -6,9 +6,10 @@ import { CriterionResult, ConversationTurn, DetailedEvaluationResult, + DependencyGraph, + normalizeCriteria, } from "shared"; -import { getCriteriaRegistry } from "shared/criteria-registry"; -import { CriteriaGraph, normalizeCriteria } from "shared/criteria-graph"; +import { getCriteriaProvider } from "shared/criteria-provider-factory"; import { createJudgeStrategy } from "./judge-strategies.js"; import { FeedbackGenerator } from "./feedback-generator.js"; @@ -56,12 +57,12 @@ export async function evaluateWorkspace( let normalizedCriteria: CriteriaConfig[]; if (scenarioVersion === "v2") { - // v2: criteria are IDs, resolve from registry + // v2: criteria are IDs, resolve from provider (API or filesystem) try { - const registry = getCriteriaRegistry(); - normalizedCriteria = registry.resolveWithAncestors(input.criteria); + const provider = getCriteriaProvider(); + normalizedCriteria = await provider.resolveWithAncestors(input.criteria); console.log( - `[judge-agent] Loaded ${normalizedCriteria.length} criteria (including ancestors) from registry (v2 format)` + `[judge-agent] Loaded ${normalizedCriteria.length} criteria (including ancestors) from provider (v2 format)` ); } catch (error) { console.error("[judge-agent] Failed to resolve criteria:", error); @@ -78,9 +79,9 @@ export async function evaluateWorkspace( } // 4. Build criteria graph (validates DAG) - let criteriaGraph: CriteriaGraph; + let criteriaGraph: DependencyGraph; try { - criteriaGraph = new CriteriaGraph(normalizedCriteria); + criteriaGraph = new DependencyGraph(normalizedCriteria); console.log( `[judge-agent] Built criteria DAG with ${normalizedCriteria.length} nodes` ); diff --git a/packages/judge/src/judge-strategies.ts b/apps/judge/src/judge-strategies.ts similarity index 98% rename from packages/judge/src/judge-strategies.ts rename to apps/judge/src/judge-strategies.ts index 2601d823f..9ad4d9326 100644 --- a/packages/judge/src/judge-strategies.ts +++ b/apps/judge/src/judge-strategies.ts @@ -10,13 +10,14 @@ import { CriterionResult, DetailedEvaluationResult, ConversationTurn, + DependencyGraph, + TokenManagerClient, } from "shared"; -import { CriteriaGraph } from "shared/criteria-graph"; export interface JudgeStrategyContext { workspacePath: string; criteria: CriteriaConfig[]; - criteriaGraph: CriteriaGraph; + criteriaGraph: DependencyGraph; conversationHistory: ConversationTurn[]; personaInstructions?: string; model?: string; @@ -33,10 +34,12 @@ const DEFAULT_JUDGE_TIMEOUT = 300_000; export abstract class JudgeStrategy { protected model: string; protected timeout: number; + protected tokenClient: TokenManagerClient; constructor(model?: string) { this.model = model || process.env.JUDGE_MODEL || "gpt-4.1"; this.timeout = parseInt(process.env.JUDGE_TIMEOUT || String(DEFAULT_JUDGE_TIMEOUT)); + this.tokenClient = new TokenManagerClient(); } abstract evaluate( @@ -229,7 +232,8 @@ export abstract class JudgeStrategy { userPrompt: string ): Promise<string> { const tools = this.createFileTools(workspacePath); - const client = new CopilotClient(); + const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); + const client = new CopilotClient({ githubToken }); let fullResponse = ""; try { diff --git a/packages/judge/tsconfig.json b/apps/judge/tsconfig.json similarity index 100% rename from packages/judge/tsconfig.json rename to apps/judge/tsconfig.json diff --git a/apps/model-scanners/anthropic/Dockerfile b/apps/model-scanners/anthropic/Dockerfile new file mode 100644 index 000000000..76f5c44ab --- /dev/null +++ b/apps/model-scanners/anthropic/Dockerfile @@ -0,0 +1,45 @@ +FROM node:22.22.0-slim AS builder + +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY tsconfig.base.json ./ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/model-scanning/package.json ./packages/model-scanning/ +COPY apps/model-scanners/anthropic/package.json ./apps/model-scanners/anthropic/ + +RUN pnpm install --frozen-lockfile + +COPY packages/shared/tsconfig.json ./packages/shared/ +COPY packages/shared/src ./packages/shared/src/ +COPY packages/model-scanning/tsconfig.json ./packages/model-scanning/ +COPY packages/model-scanning/src ./packages/model-scanning/src/ +COPY apps/model-scanners/anthropic/tsconfig.json ./apps/model-scanners/anthropic/ +COPY apps/model-scanners/anthropic/src ./apps/model-scanners/anthropic/src/ + +RUN pnpm --filter shared build && pnpm --filter model-scanning build && pnpm --filter model-scanner-anthropic build + +# --- Production stage --- +FROM node:22.22.0-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/model-scanning/package.json ./packages/model-scanning/ +COPY apps/model-scanners/anthropic/package.json ./apps/model-scanners/anthropic/ + +RUN pnpm install --frozen-lockfile --prod + +COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ +COPY --from=builder /app/packages/model-scanning/dist ./packages/model-scanning/dist/ +COPY --from=builder /app/apps/model-scanners/anthropic/dist ./apps/model-scanners/anthropic/dist/ + +WORKDIR /app/apps/model-scanners/anthropic + +CMD ["node", "dist/index.js"] diff --git a/apps/model-scanners/anthropic/package.json b/apps/model-scanners/anthropic/package.json new file mode 100644 index 000000000..2269454ee --- /dev/null +++ b/apps/model-scanners/anthropic/package.json @@ -0,0 +1,22 @@ +{ + "name": "model-scanner-anthropic", + "version": "1.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "dev": "tsx src/index.ts", + "start": "node dist/index.js", + "lint": "eslint src/" + }, + "dependencies": { + "shared": "workspace:*", + "model-scanning": "workspace:*", + "dotenv": "^16.4.5" + }, + "devDependencies": { + "@types/node": "^22.7.0", + "tsx": "^4.19.0", + "typescript": "^5.6.2" + } +} diff --git a/apps/model-scanners/anthropic/src/index.ts b/apps/model-scanners/anthropic/src/index.ts new file mode 100644 index 000000000..808ddc7a1 --- /dev/null +++ b/apps/model-scanners/anthropic/src/index.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import dotenv from "dotenv"; +dotenv.config(); + +import { TokenManagerClient } from "shared"; +import { + parseScannerArgs, + waitForApi, + upsertAgent, + reconcileModels, +} from "model-scanning"; +import { scanAnthropicModels } from "./scan.js"; + +const AGENT_ID = "coder-acp-claude-code"; +const AGENT_DEFINITION = { + _id: AGENT_ID, + name: "Claude Code (ACP)", + description: "Anthropic Claude Code coding agent via ACP protocol", + supportedModels: [] as string[], // Will be populated by scan +}; + +async function main(): Promise<void> { + const { dryRun, apiUrl } = parseScannerArgs(); + const tokenClient = new TokenManagerClient(); + + console.log(`Model scanner: anthropic (agent: ${AGENT_ID})`); + console.log(`Mode: ${dryRun ? "dry-run" : "live"}`); + + // Acquire token + console.log("Acquiring token for claude-code-cli capability..."); + const token = await tokenClient.acquireToken("claude-code-cli"); + console.log("Token acquired."); + + // Scan models + console.log("Scanning Anthropic models..."); + const scanResult = await scanAnthropicModels(token); + console.log(`Found ${scanResult.models.length} models:`); + for (const model of scanResult.models) { + console.log(` - ${model.id}`); + } + + if (dryRun) { + // Dry-run: output JSON and exit + console.log("\n--- Dry-run output ---"); + console.log(JSON.stringify(scanResult, null, 2)); + process.exit(0); + } + + // Live mode: wait for API, upsert agent, sync models + console.log(`Waiting for API at ${apiUrl}...`); + await waitForApi(apiUrl); + + // Upsert agent definition (creates if doesn't exist) + console.log("Upserting agent definition..."); + await upsertAgent(apiUrl, { + ...AGENT_DEFINITION, + supportedModels: scanResult.models.map((m) => m.id), + }); + + // Sync models with lifecycle tracking + console.log("Syncing models..."); + const report = await reconcileModels( + apiUrl, + AGENT_ID, + "anthropic", + scanResult, + ); + console.log( + `Sync complete: +${report.added.length} added, -${report.removed.length} removed, =${report.unchanged.length} unchanged`, + ); + if (report.added.length > 0) console.log(` Added: ${report.added.join(", ")}`); + if (report.removed.length > 0) console.log(` Removed: ${report.removed.join(", ")}`); + + console.log("Done."); +} + +main().catch((error) => { + console.error("Model scanner failed:", error); + process.exit(1); +}); diff --git a/apps/model-scanners/anthropic/src/scan.test.ts b/apps/model-scanners/anthropic/src/scan.test.ts new file mode 100644 index 000000000..004afbc38 --- /dev/null +++ b/apps/model-scanners/anthropic/src/scan.test.ts @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { scanAnthropicModels } from "./scan.js"; + +describe("scanAnthropicModels", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("should parse model IDs from { data: [...] } response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "claude-sonnet-4-20250514", type: "model", extra: "ignored" }, + { id: "claude-opus-4-20250514", display_name: "Claude Opus 4" }, + ], + has_more: false, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanAnthropicModels("test-key"); + + expect(result.provider).toBe("anthropic"); + expect(result.models).toHaveLength(2); + expect(result.models.map((m) => m.id)).toEqual([ + "claude-sonnet-4-20250514", + "claude-opus-4-20250514", + ]); + expect(result.scannedAt).toBeInstanceOf(Date); + }); + + it("should extract providerAvailableFrom from created_at (Unix timestamp)", async () => { + const timestamp = 1715000000; // 2024-05-06T16:53:20Z + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "claude-sonnet-4-20250514", created_at: timestamp }, + ], + has_more: false, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanAnthropicModels("test-key"); + + expect(result.models[0].providerAvailableFrom).toEqual( + new Date(timestamp * 1000), + ); + }); + + it("should capture display_name and type in metadata", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { + id: "claude-opus-4-20250514", + display_name: "Claude Opus 4", + type: "model", + }, + ], + has_more: false, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanAnthropicModels("test-key"); + + expect(result.models[0].metadata).toEqual({ + displayName: "Claude Opus 4", + type: "model", + }); + }); + + it("should handle pagination with has_more", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + + // First page + fetchSpy.mockResolvedValueOnce( + new Response( + JSON.stringify({ + data: [ + { id: "model-a" }, + { id: "model-b" }, + ], + has_more: true, + }), + { status: 200 }, + ), + ); + + // Second page + fetchSpy.mockResolvedValueOnce( + new Response( + JSON.stringify({ + data: [ + { id: "model-c" }, + ], + has_more: false, + }), + { status: 200 }, + ), + ); + + const result = await scanAnthropicModels("test-key"); + + expect(result.models).toHaveLength(3); + expect(result.models.map((m) => m.id)).toEqual([ + "model-a", + "model-b", + "model-c", + ]); + + // Second call should include after_id + expect(fetchSpy).toHaveBeenCalledTimes(2); + const secondUrl = fetchSpy.mock.calls[1][0] as string; + expect(secondUrl).toContain("after_id=model-b"); + }); + + it("should skip entries without valid id", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "valid" }, + { id: "" }, + { id: 42 }, + { display_name: "no id" }, + null, + ], + has_more: false, + }), + { status: 200 }, + ), + ); + + const result = await scanAnthropicModels("test-key"); + + expect(result.models).toHaveLength(1); + expect(result.models[0].id).toBe("valid"); + }); + + it("should handle empty data array", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ data: [], has_more: false }), + { status: 200 }, + ), + ); + + const result = await scanAnthropicModels("test-key"); + + expect(result.models).toHaveLength(0); + expect(result.provider).toBe("anthropic"); + }); + + it("should send correct headers", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ data: [], has_more: false }), + { status: 200 }, + ), + ); + + await scanAnthropicModels("my-api-key"); + + const [, init] = fetchSpy.mock.calls[0]; + expect((init?.headers as Record<string, string>)["x-api-key"]).toBe( + "my-api-key", + ); + expect( + (init?.headers as Record<string, string>)["anthropic-version"], + ).toBe("2023-06-01"); + }); + + it("should throw on 401 response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Unauthorized", { status: 401 }), + ); + + await expect(scanAnthropicModels("bad-key")).rejects.toThrow( + "Anthropic models API returned HTTP 401", + ); + }); + + it("should throw on 500 response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Server Error", { status: 500 }), + ); + + await expect(scanAnthropicModels("key")).rejects.toThrow( + "Anthropic models API returned HTTP 500", + ); + }); + + it("should throw on network error", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue( + new Error("Network failure"), + ); + + await expect(scanAnthropicModels("key")).rejects.toThrow( + "Network failure", + ); + }); +}); diff --git a/apps/model-scanners/anthropic/src/scan.ts b/apps/model-scanners/anthropic/src/scan.ts new file mode 100644 index 000000000..0f4ad6b2d --- /dev/null +++ b/apps/model-scanners/anthropic/src/scan.ts @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { ScanResult, ScannedModel } from "model-scanning"; + +const ANTHROPIC_MODELS_URL = "https://api.anthropic.com/v1/models"; + +/** + * Scan models available from the Anthropic API. + * + * Calls GET https://api.anthropic.com/v1/models and handles pagination. + * Permissive: only validates the `id` field on each model, ignores unknown fields. + */ +export async function scanAnthropicModels(token: string): Promise<ScanResult> { + const models: ScannedModel[] = []; + let hasMore = true; + let afterId: string | undefined; + + while (hasMore) { + const url = new URL(ANTHROPIC_MODELS_URL); + url.searchParams.set("limit", "100"); + if (afterId) { + url.searchParams.set("after_id", afterId); + } + + const response = await fetch(url.toString(), { + headers: { + "x-api-key": token, + "anthropic-version": "2023-06-01", + }, + signal: AbortSignal.timeout(30_000), + }); + + if (!response.ok) { + const errorBody = await response.text().catch(() => "unknown error"); + throw new Error( + `Anthropic models API returned HTTP ${response.status}: ${errorBody}`, + ); + } + + const body = await response.json(); + + // Permissive parsing: expect { data: [...], has_more: bool } + const rawModels = Array.isArray(body?.data) + ? body.data + : Array.isArray(body) + ? body + : []; + + for (const m of rawModels) { + if (!m || typeof m.id !== "string" || !m.id) continue; + + models.push({ + id: m.id, + // Anthropic provides created_at as Unix timestamp + ...(m.created_at + ? { + providerAvailableFrom: new Date( + typeof m.created_at === "number" + ? m.created_at * 1000 + : m.created_at, + ), + } + : {}), + // Capture display name and type as metadata + ...(m.display_name || m.type + ? { + metadata: { + ...(m.display_name ? { displayName: m.display_name } : {}), + ...(m.type ? { type: m.type } : {}), + }, + } + : {}), + }); + + afterId = m.id; + } + + // Check pagination + hasMore = body?.has_more === true && rawModels.length > 0; + } + + return { + provider: "anthropic", + models, + scannedAt: new Date(), + }; +} diff --git a/apps/model-scanners/anthropic/tsconfig.json b/apps/model-scanners/anthropic/tsconfig.json new file mode 100644 index 000000000..0e9849c74 --- /dev/null +++ b/apps/model-scanners/anthropic/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/apps/model-scanners/copilot/Dockerfile b/apps/model-scanners/copilot/Dockerfile new file mode 100644 index 000000000..26ff6803e --- /dev/null +++ b/apps/model-scanners/copilot/Dockerfile @@ -0,0 +1,45 @@ +FROM node:22.22.0-slim AS builder + +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY tsconfig.base.json ./ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/model-scanning/package.json ./packages/model-scanning/ +COPY apps/model-scanners/copilot/package.json ./apps/model-scanners/copilot/ + +RUN pnpm install --frozen-lockfile + +COPY packages/shared/tsconfig.json ./packages/shared/ +COPY packages/shared/src ./packages/shared/src/ +COPY packages/model-scanning/tsconfig.json ./packages/model-scanning/ +COPY packages/model-scanning/src ./packages/model-scanning/src/ +COPY apps/model-scanners/copilot/tsconfig.json ./apps/model-scanners/copilot/ +COPY apps/model-scanners/copilot/src ./apps/model-scanners/copilot/src/ + +RUN pnpm --filter shared build && pnpm --filter model-scanning build && pnpm --filter model-scanner-copilot build + +# --- Production stage --- +FROM node:22.22.0-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/model-scanning/package.json ./packages/model-scanning/ +COPY apps/model-scanners/copilot/package.json ./apps/model-scanners/copilot/ + +RUN pnpm install --frozen-lockfile --prod + +COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ +COPY --from=builder /app/packages/model-scanning/dist ./packages/model-scanning/dist/ +COPY --from=builder /app/apps/model-scanners/copilot/dist ./apps/model-scanners/copilot/dist/ + +WORKDIR /app/apps/model-scanners/copilot + +CMD ["node", "dist/index.js"] diff --git a/apps/model-scanners/copilot/package.json b/apps/model-scanners/copilot/package.json new file mode 100644 index 000000000..b228abcb4 --- /dev/null +++ b/apps/model-scanners/copilot/package.json @@ -0,0 +1,22 @@ +{ + "name": "model-scanner-copilot", + "version": "1.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "dev": "tsx src/index.ts", + "start": "node dist/index.js", + "lint": "eslint src/" + }, + "dependencies": { + "shared": "workspace:*", + "model-scanning": "workspace:*", + "dotenv": "^16.4.5" + }, + "devDependencies": { + "@types/node": "^22.7.0", + "tsx": "^4.19.0", + "typescript": "^5.6.2" + } +} diff --git a/apps/model-scanners/copilot/src/index.ts b/apps/model-scanners/copilot/src/index.ts new file mode 100644 index 000000000..b863eaf87 --- /dev/null +++ b/apps/model-scanners/copilot/src/index.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import dotenv from "dotenv"; +dotenv.config(); + +import { TokenManagerClient } from "shared"; +import { + parseScannerArgs, + waitForApi, + upsertAgent, + reconcileModels, +} from "model-scanning"; +import { scanCopilotModels } from "./scan.js"; + +const AGENT_ID = "coder-acp-copilot"; +const AGENT_DEFINITION = { + _id: AGENT_ID, + name: "Copilot (ACP)", + description: "GitHub Copilot coding agent via ACP protocol", + supportedModels: [] as string[], // Will be populated by scan +}; + +async function main(): Promise<void> { + const { dryRun, apiUrl } = parseScannerArgs(); + const tokenClient = new TokenManagerClient(); + + console.log(`Model scanner: copilot (agent: ${AGENT_ID})`); + console.log(`Mode: ${dryRun ? "dry-run" : "live"}`); + + // Acquire token + console.log("Acquiring token for copilot-cli capability..."); + const token = await tokenClient.acquireToken("copilot-cli"); + console.log("Token acquired."); + + // Scan models + console.log("Scanning GitHub Copilot models..."); + const scanResult = await scanCopilotModels(token); + console.log(`Found ${scanResult.models.length} models:`); + for (const model of scanResult.models) { + console.log(` - ${model.id}`); + } + + if (dryRun) { + // Dry-run: output JSON and exit + console.log("\n--- Dry-run output ---"); + console.log(JSON.stringify(scanResult, null, 2)); + process.exit(0); + } + + // Live mode: wait for API, upsert agent, sync models + console.log(`Waiting for API at ${apiUrl}...`); + await waitForApi(apiUrl); + + // Upsert agent definition (creates if doesn't exist) + console.log("Upserting agent definition..."); + await upsertAgent(apiUrl, { + ...AGENT_DEFINITION, + supportedModels: scanResult.models.map((m) => m.id), + }); + + // Sync models with lifecycle tracking + console.log("Syncing models..."); + const report = await reconcileModels( + apiUrl, + AGENT_ID, + "github-copilot", + scanResult, + ); + console.log( + `Sync complete: +${report.added.length} added, -${report.removed.length} removed, =${report.unchanged.length} unchanged`, + ); + if (report.added.length > 0) console.log(` Added: ${report.added.join(", ")}`); + if (report.removed.length > 0) console.log(` Removed: ${report.removed.join(", ")}`); + + console.log("Done."); +} + +main().catch((error) => { + console.error("Model scanner failed:", error); + process.exit(1); +}); diff --git a/apps/model-scanners/copilot/src/scan.test.ts b/apps/model-scanners/copilot/src/scan.test.ts new file mode 100644 index 000000000..15a31fcf0 --- /dev/null +++ b/apps/model-scanners/copilot/src/scan.test.ts @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { scanCopilotModels } from "./scan.js"; + +describe("scanCopilotModels", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("should parse model IDs from .data[] response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "gpt-4.1", name: "GPT-4.1", extra_field: "ignored" }, + { id: "claude-sonnet-4", object: "model", another: 123 }, + { id: "o3-mini", created_at: "2025-01-15T00:00:00Z" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("test-token"); + + expect(result.provider).toBe("github-copilot"); + expect(result.models).toHaveLength(3); + expect(result.models.map((m) => m.id)).toEqual([ + "gpt-4.1", + "claude-sonnet-4", + "o3-mini", + ]); + expect(result.scannedAt).toBeInstanceOf(Date); + }); + + it("should extract providerAvailableFrom from created_at", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "gpt-4.1", created_at: "2025-06-01T00:00:00Z" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("test-token"); + + expect(result.models[0].providerAvailableFrom).toEqual(new Date("2025-06-01T00:00:00Z")); + }); + + it("should extract providerEndOfLife from deprecation_date", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "gpt-4o", deprecation_date: "2026-12-31T00:00:00Z" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("test-token"); + + expect(result.models[0].providerEndOfLife).toEqual(new Date("2026-12-31T00:00:00Z")); + }); + + it("should skip entries without a valid id", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "valid-model" }, + { id: "" }, + { id: 123 }, + { name: "no-id" }, + null, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("test-token"); + + expect(result.models).toHaveLength(1); + expect(result.models[0].id).toBe("valid-model"); + }); + + it("should handle empty data array", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ data: [] }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("test-token"); + + expect(result.models).toHaveLength(0); + expect(result.provider).toBe("github-copilot"); + }); + + it("should handle response as plain array (fallback)", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify([ + { id: "model-a" }, + { id: "model-b" }, + ]), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("test-token"); + + expect(result.models).toHaveLength(2); + expect(result.models.map((m) => m.id)).toEqual(["model-a", "model-b"]); + }); + + it("should send correct headers", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ data: [] }), { status: 200 }), + ); + + await scanCopilotModels("my-github-token"); + + const [url, init] = fetchSpy.mock.calls[0]; + expect(url).toBe("https://api.githubcopilot.com/models"); + expect((init?.headers as Record<string, string>)["Authorization"]).toBe( + "Bearer my-github-token", + ); + expect( + (init?.headers as Record<string, string>)["Copilot-Integration-Id"], + ).toBe("vscode-chat"); + }); + + it("should throw on 401 response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Unauthorized", { status: 401 }), + ); + + await expect(scanCopilotModels("bad-token")).rejects.toThrow( + "Copilot models API returned HTTP 401", + ); + }); + + it("should throw on 500 response", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Server Error", { status: 500 }), + ); + + await expect(scanCopilotModels("token")).rejects.toThrow( + "Copilot models API returned HTTP 500", + ); + }); + + it("should throw on network error", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("Network failure")); + + await expect(scanCopilotModels("token")).rejects.toThrow("Network failure"); + }); + + it("should capture name and version in metadata", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + data: [ + { id: "gpt-4.1", name: "GPT 4.1", version: "2025-04-14" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + const result = await scanCopilotModels("token"); + + expect(result.models[0].metadata).toEqual({ + name: "GPT 4.1", + version: "2025-04-14", + }); + }); +}); diff --git a/apps/model-scanners/copilot/src/scan.ts b/apps/model-scanners/copilot/src/scan.ts new file mode 100644 index 000000000..0f5a7a467 --- /dev/null +++ b/apps/model-scanners/copilot/src/scan.ts @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { ScanResult } from "model-scanning"; + +const COPILOT_MODELS_URL = "https://api.githubcopilot.com/models"; + +/** + * Scan models available from the GitHub Copilot API. + * + * Calls GET https://api.githubcopilot.com/models and parses the response. + * Permissive: only validates the `id` field on each model, ignores unknown fields. + * + * @see https://aider.chat/docs/llms/github.html + */ +export async function scanCopilotModels(token: string): Promise<ScanResult> { + const response = await fetch(COPILOT_MODELS_URL, { + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "Copilot-Integration-Id": "vscode-chat", + }, + signal: AbortSignal.timeout(30_000), + }); + + if (!response.ok) { + const errorBody = await response.text().catch(() => "unknown error"); + throw new Error( + `Copilot models API returned HTTP ${response.status}: ${errorBody}`, + ); + } + + const body = await response.json(); + + // Permissive parsing: expect { data: [...] } but handle variations + const rawModels = Array.isArray(body?.data) + ? body.data + : Array.isArray(body) + ? body + : []; + + const models = rawModels + .filter( + (m: Record<string, unknown>) => m && typeof m.id === "string" && m.id, + ) + .map((m: Record<string, unknown>) => ({ + id: m.id as string, + // Extract availability/EOL dates if the API provides them + ...(m.created_at + ? { providerAvailableFrom: new Date(m.created_at as string) } + : {}), + ...(m.deprecation_date || m.end_of_life + ? { + providerEndOfLife: new Date( + (m.deprecation_date || m.end_of_life) as string, + ), + } + : {}), + // Keep a subset of useful metadata + ...(m.name || m.version + ? { + metadata: { + ...(m.name ? { name: m.name } : {}), + ...(m.version ? { version: m.version } : {}), + }, + } + : {}), + })); + + return { + provider: "github-copilot", + models, + scannedAt: new Date(), + }; +} diff --git a/apps/model-scanners/copilot/tsconfig.json b/apps/model-scanners/copilot/tsconfig.json new file mode 100644 index 000000000..0e9849c74 --- /dev/null +++ b/apps/model-scanners/copilot/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/apps/portal/Dockerfile b/apps/portal/Dockerfile new file mode 100644 index 000000000..61cffec70 --- /dev/null +++ b/apps/portal/Dockerfile @@ -0,0 +1,31 @@ +# Stage 1: Build +FROM node:20-alpine AS builder +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +# Version information (passed as build args, used by Vite at build time) +ARG GIT_COMMIT=unknown +ARG BUILD_TIME=unknown +ENV GIT_COMMIT=$GIT_COMMIT +ENV BUILD_TIME=$BUILD_TIME + +WORKDIR /app +COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./ +COPY apps/portal/package.json apps/portal/ +RUN pnpm install --frozen-lockfile --filter portal + +COPY apps/portal/ apps/portal/ +RUN pnpm --filter portal build + +# --- Dev stage: Vite dev server with HMR --- +FROM builder AS dev +ENV NODE_ENV=development +WORKDIR /app/apps/portal +EXPOSE 80 +CMD ["npx", "vite", "--host", "0.0.0.0", "--port", "80"] + +# Stage 2: Serve with nginx +FROM nginx:alpine +COPY --from=builder /app/apps/portal/dist /usr/share/nginx/html +COPY apps/portal/nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 +CMD ["nginx", "-g", "daemon off;"] diff --git a/packages/portal/components.json b/apps/portal/components.json similarity index 100% rename from packages/portal/components.json rename to apps/portal/components.json diff --git a/packages/portal/index.html b/apps/portal/index.html similarity index 82% rename from packages/portal/index.html rename to apps/portal/index.html index 10030f9c1..ae64eb8ca 100644 --- a/packages/portal/index.html +++ b/apps/portal/index.html @@ -2,7 +2,7 @@ <html lang="en"> <head> <meta charset="UTF-8" /> - <link rel="icon" type="image/svg+xml" href="/vite.svg" /> + <link rel="icon" type="image/svg+xml" href="/favicon.svg" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <title>Scope MT ā€” Run Management diff --git a/packages/portal/nginx.conf b/apps/portal/nginx.conf similarity index 100% rename from packages/portal/nginx.conf rename to apps/portal/nginx.conf diff --git a/packages/portal/package.json b/apps/portal/package.json similarity index 83% rename from packages/portal/package.json rename to apps/portal/package.json index 5305de970..695b59258 100644 --- a/packages/portal/package.json +++ b/apps/portal/package.json @@ -12,26 +12,33 @@ "dependencies": { "@radix-ui/react-alert-dialog": "^1.1.4", "@radix-ui/react-checkbox": "^1.1.4", + "@radix-ui/react-dialog": "^1.1.4", "@radix-ui/react-label": "^2.1.1", "@radix-ui/react-scroll-area": "^1.2.2", "@radix-ui/react-select": "^2.1.5", "@radix-ui/react-separator": "^1.1.1", "@radix-ui/react-slot": "^1.1.1", + "@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-tabs": "^1.1.2", "@tanstack/react-query": "^5.62.0", + "@xyflow/react": "^12.6.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", + "dagre": "^0.8.5", "lucide-react": "^0.469.0", "react": "^19.0.0", "react-dom": "^19.0.0", + "react-markdown": "^9.0.1", + "react-router-dom": "^7.1.0", "recharts": "^2.15.0", + "rehype-raw": "^7.0.0", + "remark-gfm": "^4.0.0", + "remark-github-markdown-alerts": "^1.2.2", "sonner": "^1.7.0", - "@xyflow/react": "^12.6.0", - "dagre": "^0.8.5", - "react-router-dom": "^7.1.0", "tailwind-merge": "^2.6.0" }, "devDependencies": { + "@tailwindcss/typography": "^0.5.19", "@types/dagre": "^0.7.52", "@types/react": "^19.0.0", "@types/react-dom": "^19.0.0", diff --git a/packages/portal/postcss.config.js b/apps/portal/postcss.config.js similarity index 100% rename from packages/portal/postcss.config.js rename to apps/portal/postcss.config.js diff --git a/apps/portal/public/favicon.svg b/apps/portal/public/favicon.svg new file mode 100644 index 000000000..c3c664690 --- /dev/null +++ b/apps/portal/public/favicon.svg @@ -0,0 +1,3 @@ + + + diff --git a/apps/portal/src/App.tsx b/apps/portal/src/App.tsx new file mode 100644 index 000000000..ee269409f --- /dev/null +++ b/apps/portal/src/App.tsx @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Routes, Route, Navigate } from "react-router-dom"; +import { Layout } from "@/components/Layout"; +import { RunsList } from "@/pages/RunsList"; +import { RunDetail } from "@/pages/RunDetail"; +import { SubmitRun } from "@/pages/SubmitRun"; +import { CriteriaList } from "@/pages/CriteriaList"; +import { CriterionDetail } from "@/pages/CriterionDetail"; +import { CreateCriterion } from "@/pages/CreateCriterion"; +import { CriteriaGraphView } from "@/pages/CriteriaGraphView"; +import { Statistics } from "@/pages/Statistics"; +import { PromptFeatureList } from "@/pages/PromptFeatureList"; +import { PromptFeatureDetail } from "@/pages/PromptFeatureDetail"; +import { CreatePromptFeature } from "@/pages/CreatePromptFeature"; +import { PromptFeatureGraphView } from "@/pages/PromptFeatureGraphView"; +import { ReportsList } from "@/pages/ReportsList"; +import { ReportDetail } from "@/pages/ReportDetail"; +import { ReportTemplateList } from "@/pages/ReportTemplateList"; +import { ReportTemplateDetail } from "@/pages/ReportTemplateDetail"; +import { CreateReportTemplate } from "@/pages/CreateReportTemplate"; +import { TokenList } from "@/pages/TokenList"; +import { CreateToken } from "@/pages/CreateToken"; +import { TokenDetail } from "@/pages/TokenDetail"; +import { AgentList } from "@/pages/AgentList"; +import { AgentDetail } from "@/pages/AgentDetail"; +import { McpServerList } from "@/pages/McpServerList"; +import { CreateMcpServer } from "@/pages/CreateMcpServer"; +import { McpServerDetail } from "@/pages/McpServerDetail"; +import { SkillList } from "@/pages/SkillList"; +import { SkillDetail } from "@/pages/SkillDetail"; +import { InsightsList } from "@/pages/InsightsList"; +import { InsightDetail } from "@/pages/InsightDetail"; +import { CriteriaMdpView } from "@/pages/CriteriaMdpView"; +import { ModelList } from "@/pages/ModelList"; +import { ModelDetail } from "@/pages/ModelDetail"; +import { TaskPromptList } from "@/pages/TaskPromptList"; +import { TaskPromptDetail } from "@/pages/TaskPromptDetail"; +import { Admin } from "@/pages/Admin"; +import { FeatureRoute } from "@/components/FeatureRoute"; + +export function App() { + return ( + + }> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + ); +} diff --git a/apps/portal/src/components/ConversationView.tsx b/apps/portal/src/components/ConversationView.tsx new file mode 100644 index 000000000..b47400992 --- /dev/null +++ b/apps/portal/src/components/ConversationView.tsx @@ -0,0 +1,339 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState } from "react"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/utils"; +import { Bot, Scale, CheckCircle2, AlertCircle, Brain, Wrench, ChevronRight, ChevronDown, Loader2 } from "lucide-react"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import rehypeRaw from "rehype-raw"; +import type { ConversationTurn, ToolCall } from "@/types"; +import { useHarExtraction, type ConversationSegment } from "@/hooks/useHarExtraction"; + +interface ConversationViewProps { + turns: ConversationTurn[]; + /** The original task / user prompt, shown as the first message */ + task?: string; + /** Run ID ā€” needed to fetch HAR data for each turn */ + runId: string; +} + +/** + * Chat-style conversation view showing agent responses and judge feedback. + * + * Layout: + * - Task prompt (top, full-width, neutral) + * - For each turn: + * - Thinking content (collapsible, muted) + * - Agent response (right-aligned, primary tint) + * - Tool calls (inline, collapsible) + * - Judge feedback (left-aligned, amber tint) + */ +export function ConversationView({ turns, task, runId }: ConversationViewProps) { + if (turns.length === 0 && !task) { + return ( +
+ No conversation data available. +
+ ); + } + + return ( +
+ {/* Task prompt */} + {task && ( +
+ + +
+ + Task + +
+
+ {task} +
+ + +
+ )} + + {/* Turn messages */} + {turns.map((turn) => ( + + ))} +
+ ); +} + +/** Collapsible section with a toggle header */ +function CollapsibleSection({ + label, + icon: Icon, + iconClassName, + count, + timestamp, + preview, + defaultOpen = false, + children, +}: { + label: string; + icon: React.ComponentType<{ className?: string }>; + iconClassName?: string; + count?: number; + timestamp?: string; + /** One-line preview shown when collapsed */ + preview?: string; + defaultOpen?: boolean; + children: React.ReactNode; +}) { + const [open, setOpen] = useState(defaultOpen); + return ( +
+ + {open &&
{children}
} +
+ ); +} + +/** Inline tool call display */ +function ToolCallInline({ tc }: { tc: ToolCall }) { + const [expanded, setExpanded] = useState(false); + const hasResponse = !!tc.response; + const argsStr = JSON.stringify(tc.arguments, null, 2); + const argsOneLine = JSON.stringify(tc.arguments); + + return ( +
+ + {expanded && ( +
+ {argsStr !== "{}" && ( +
+ Arguments + 
{argsStr}
+
+ )} + {hasResponse && ( +
+ Response + 
{tc.response}
+
+ )} +
+ )} +
+ ); +} + +function TurnMessages({ turn, runId }: { turn: ConversationTurn; runId: string }) { + const hasHar = !!turn.harUrl; + const { data: harData, isLoading: harLoading } = useHarExtraction(runId, turn.iteration, hasHar); + + const segments = harData?.segments ?? []; + const hasContentSegment = segments.some((s) => s.type === "content"); + + return ( + <> + {/* Iteration divider */} +
+
+ + Iteration {turn.iteration} + {turn.passed && ( + + )} + +
+
+ + {/* Loading indicator */} + {harLoading && hasHar && ( +
+
+ + Loading HAR dataā€¦ +
+
+ )} + + {/* Chronological segments from HAR */} + {segments.map((segment, idx) => ( + + ))} + + {/* Fallback: show DB agent response if no content segments from HAR */} + {!hasContentSegment && !harLoading && ( + + )} + + {/* Judge feedback ā€” left aligned */} +
+ + +
+ + Judge + {turn.passed ? ( + + Passed + + ) : ( + + Incomplete + + )} +
+
+ {turn.judgeFeedback} +
+ + {/* Criteria results inline */} + {turn.criteriaResults && turn.criteriaResults.length > 0 && ( +
+ {turn.criteriaResults.map((cr) => ( +
+ {cr.passed ? ( + + ) : ( + + )} + {cr.criterionId} +
+ ))} +
+ )} + + +
+ + ); +} + +/** Render a single chronological segment */ +function SegmentBlock({ segment, turn }: { segment: ConversationSegment; turn: ConversationTurn }) { + switch (segment.type) { + case "thinking": + return ( +
+
+ + + +
+ {segment.content} +
+ + + +
+
+ ); + + case "content": + return ; + + case "tool_calls": + return ( +
+
+ +
+ {segment.toolCalls.map((tc, idx) => ( + + ))} +
+ +
+
+ ); + } +} + +/** Agent response card ā€” right aligned */ +function AgentResponseBlock({ content, turn }: { content: string; turn: ConversationTurn }) { + return ( +
+ + +
+ + Coding Agent + + {new Date(turn.timestamp).toLocaleTimeString()} + +
+
+ {content} +
+ + +
+ ); +} diff --git a/apps/portal/src/components/CriteriaFilterBar.tsx b/apps/portal/src/components/CriteriaFilterBar.tsx new file mode 100644 index 000000000..3c1dff4f0 --- /dev/null +++ b/apps/portal/src/components/CriteriaFilterBar.tsx @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Filter, X } from "lucide-react"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; + +interface CriteriaFilterBarProps { + availableCriteria: string[]; + selectedCriteria: string[]; + onToggle: (id: string) => void; + onClear: () => void; + /** Override the card title (default: "Success Criteria Filter") */ + title?: string; + /** Override the description when nothing is selected */ + emptyDescription?: string; + /** Override the description template when criteria are selected (receives count) */ + selectedDescription?: (count: number) => string; +} + +export function CriteriaFilterBar({ + availableCriteria, + selectedCriteria, + onToggle, + onClear, + title = "Success Criteria Filter", + emptyDescription = "Click criteria to filter runs and redefine success. Default: all criteria in each run must pass.", + selectedDescription = (count: number) => + `Success = all ${count} selected criteria pass. Runs without these criteria are excluded.`, +}: CriteriaFilterBarProps) { + if (availableCriteria.length === 0) { + return null; + } + + const selectedSet = new Set(selectedCriteria); + + return ( + + +
+
+ + {title} +
+ {selectedCriteria.length > 0 && ( + + )} +
+ + {selectedCriteria.length === 0 + ? emptyDescription + : selectedDescription(selectedCriteria.length)} + + + +
+ {availableCriteria.map((id) => { + const isSelected = selectedSet.has(id); + return ( + onToggle(id)} + > + {id} + {isSelected && } + + ); + })} +
+ + + ); +} diff --git a/packages/portal/src/components/CriteriaGraphView.tsx b/apps/portal/src/components/CriteriaGraphView.tsx similarity index 100% rename from packages/portal/src/components/CriteriaGraphView.tsx rename to apps/portal/src/components/CriteriaGraphView.tsx diff --git a/packages/portal/src/components/CriteriaPicker.tsx b/apps/portal/src/components/CriteriaPicker.tsx similarity index 100% rename from packages/portal/src/components/CriteriaPicker.tsx rename to apps/portal/src/components/CriteriaPicker.tsx diff --git a/apps/portal/src/components/FeatureRoute.tsx b/apps/portal/src/components/FeatureRoute.tsx new file mode 100644 index 000000000..387d73e86 --- /dev/null +++ b/apps/portal/src/components/FeatureRoute.tsx @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Navigate } from "react-router-dom"; +import { useFeatureFlags } from "@/contexts/FeatureFlagContext"; +import type { ReactNode } from "react"; + +interface FeatureRouteProps { + featureKey: string; + children: ReactNode; +} + +/** Route guard that redirects to /statistics when a feature flag is disabled */ +export function FeatureRoute({ featureKey, children }: FeatureRouteProps) { + const { isFeatureEnabled } = useFeatureFlags(); + + if (!isFeatureEnabled(featureKey)) { + return ; + } + + return <>{children}; +} diff --git a/apps/portal/src/components/HarNetworkViewer.tsx b/apps/portal/src/components/HarNetworkViewer.tsx new file mode 100644 index 000000000..02c9316c3 --- /dev/null +++ b/apps/portal/src/components/HarNetworkViewer.tsx @@ -0,0 +1,466 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState, useMemo } from "react"; +import { useHarData } from "@/hooks/useHarExtraction"; +import { api } from "@/lib/api"; +import { Card, CardContent } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { cn } from "@/lib/utils"; +import { Download, Search, X, ChevronDown, ChevronRight } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +// --------------------------------------------------------------------------- +// Types (inline HAR 1.2 subset ā€” no shared dep needed for the portal) +// --------------------------------------------------------------------------- +interface HarNameValue { name: string; value: string } +interface HarRequest { + method: string; + url: string; + headers: HarNameValue[]; + postData?: { mimeType: string; text?: string }; +} +interface HarResponse { + status: number; + statusText: string; + headers: HarNameValue[]; + content: { size: number; mimeType: string; text?: string; encoding?: string }; +} +interface HarEntry { + startedDateTime: string; + time: number; + request: HarRequest; + response: HarResponse; +} +interface HarFile { + log: { entries: HarEntry[] }; +} + +// --------------------------------------------------------------------------- +// Props +// --------------------------------------------------------------------------- +interface HarNetworkViewerProps { + runId: string; + /** If provided, fetches HAR for a specific iteration */ + iteration?: number; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function formatBytes(bytes: number): string { + if (bytes === 0) return "0 B"; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function formatMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)} ms`; + return `${(ms / 1000).toFixed(2)} s`; +} + +function methodColor(method: string): string { + switch (method.toUpperCase()) { + case "GET": return "text-emerald-600 bg-emerald-500/10"; + case "POST": return "text-blue-600 bg-blue-500/10"; + case "PUT": return "text-amber-600 bg-amber-500/10"; + case "PATCH": return "text-orange-600 bg-orange-500/10"; + case "DELETE": return "text-red-600 bg-red-500/10"; + default: return "text-muted-foreground bg-muted"; + } +} + +function statusColor(status: number): string { + if (status < 300) return "text-emerald-600"; + if (status < 400) return "text-amber-600"; + return "text-red-600"; +} + +/** Extract a short path from a full URL for display. */ +function shortUrl(url: string): string { + try { + const u = new URL(url); + return u.pathname + u.search; + } catch { + return url; + } +} + +/** Extract the host from a URL. */ +function hostFromUrl(url: string): string { + try { return new URL(url).host; } catch { return ""; } +} + +/** Get content type category for the "Type" column. */ +function contentCategory(entry: HarEntry): string { + const ct = entry.response.content.mimeType || ""; + if (ct.includes("json")) return "json"; + if (ct.includes("html")) return "html"; + if (ct.includes("xml")) return "xml"; + if (ct.includes("javascript") || ct.includes("ecmascript")) return "js"; + if (ct.includes("css")) return "css"; + if (ct.includes("image")) return "img"; + if (ct.includes("font")) return "font"; + if (ct.includes("text/event-stream")) return "sse"; + if (ct.includes("text")) return "text"; + return ct.split("/").pop()?.split(";")[0] ?? "other"; +} + +/** + * Re-encode a string from Latin-1 code points back to UTF-8. + * Fixes "mojibake" where UTF-8 bytes were stored as Latin-1 characters. + */ +function repairMojibake(text: string): string { + if (!/\xc2[\x80-\xbf]|\xc3[\x80-\xbf]|\xe2[\x80-\xbf]/.test(text)) return text; + try { + const bytes = Uint8Array.from(text, (c) => c.charCodeAt(0)); + return new TextDecoder("utf-8").decode(bytes); + } catch { + return text; + } +} + +/** Decode response body text (handle base64 and mojibake). */ +function decodeBody(content: HarResponse["content"]): string | null { + if (!content.text) return null; + if (content.encoding === "base64") { + try { + const binary = atob(content.text); + const bytes = Uint8Array.from(binary, (c) => c.charCodeAt(0)); + return new TextDecoder("utf-8").decode(bytes); + } catch { return null; } + } + return repairMojibake(content.text); +} + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- +export function HarNetworkViewer({ runId, iteration }: HarNetworkViewerProps) { + const [filter, setFilter] = useState(""); + const [selectedIdx, setSelectedIdx] = useState(null); + + const { data: har, isLoading, error } = useHarData(runId, iteration); + + const entries = useMemo(() => { + if (!har) return []; + const all = har.log.entries; + if (!filter) return all; + const lf = filter.toLowerCase(); + return all.filter( + (e) => + e.request.url.toLowerCase().includes(lf) || + e.request.method.toLowerCase().includes(lf) || + String(e.response.status).includes(lf), + ); + }, [har, filter]); + + // Summary stats + const totalTime = useMemo( + () => entries.reduce((sum, e) => sum + (e.time || 0), 0), + [entries], + ); + const totalSize = useMemo( + () => entries.reduce((sum, e) => sum + (e.response.content.size || 0), 0), + [entries], + ); + + if (isLoading) { + return ( +
+ + +
+ ); + } + + if (error) { + return ( +
+ Failed to load HAR data: {error instanceof Error ? error.message : String(error)} +
+ ); + } + + if (!har || entries.length === 0) { + return ( +
+ No network requests captured. +
+ ); + } + + const selected = selectedIdx !== null ? entries[selectedIdx] : null; + + return ( +
+ {/* Summary bar */} +
+ {entries.length} requests + {formatBytes(totalSize)} transferred + {formatMs(totalTime)} total +
+ +
+ + {/* Filter bar */} +
+ + setFilter(e.target.value)} + placeholder="Filter by URL, method, or statusā€¦" + className="w-full rounded-md border border-input bg-background px-9 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring" + /> + {filter && ( + + )} +
+ + {/* Request table + detail panel */} + + +
+ {/* Request list (left side) */} +
+ + + + + + + + + + + + + {entries.map((entry, idx) => ( + setSelectedIdx(selectedIdx === idx ? null : idx)} + className={cn( + "border-b last:border-0 cursor-pointer hover:bg-muted/30 transition-colors", + selectedIdx === idx && "bg-primary/5", + )} + > + + + + + + + + ))} + +
MethodURLStatusTypeSizeTime
+ + {entry.request.method} + + + + {shortUrl(entry.request.url)} + + + {entry.response.status} + + {contentCategory(entry)} + + {formatBytes(entry.response.content.size)} + + {formatMs(entry.time)} +
+
+ + {/* Detail panel (right side) */} + {selected && ( + setSelectedIdx(null)} /> + )} +
+ + +
+ ); +} + +// --------------------------------------------------------------------------- +// Detail Panel +// --------------------------------------------------------------------------- + +function DetailPanel({ entry, onClose }: { entry: HarEntry; onClose: () => void }) { + const [tab, setTab] = useState<"headers" | "request" | "response">("headers"); + + const requestBody = entry.request.postData?.text ?? null; + const responseBody = decodeBody(entry.response.content); + + return ( +
+ {/* Panel header */} +
+
+ + {entry.request.method} + + + {entry.response.status} {entry.response.statusText} + +
+ +
+ + {/* URL */} +
+ {hostFromUrl(entry.request.url)}{shortUrl(entry.request.url)} +
+ + {/* Sub-tabs */} +
+ {(["headers", "request", "response"] as const).map((t) => ( + + ))} +
+ + {/* Tab content */} +
+ {tab === "headers" && } + {tab === "request" && } + {tab === "response" && } +
+
+ ); +} + +// --------------------------------------------------------------------------- +// Headers sub-tab +// --------------------------------------------------------------------------- + +function HeadersTab({ entry }: { entry: HarEntry }) { + return ( +
+ + +
+
General
+
+ + + + + +
+
+
+ ); +} + +function HeaderSection({ title, headers }: { title: string; headers: HarNameValue[] }) { + const [collapsed, setCollapsed] = useState(false); + + if (headers.length === 0) return null; + + return ( +
+ + {!collapsed && ( +
+ {headers.map((h, i) => ( + + ))} +
+ )} +
+ ); +} + +function HeaderRow({ name, value }: { name: string; value: string }) { + return ( +
+ {name}: + {value} +
+ ); +} + +// --------------------------------------------------------------------------- +// Body sub-tab +// --------------------------------------------------------------------------- + +function BodyTab({ body, mimeType }: { body: string | null; mimeType?: string }) { + if (!body) { + return
No body
; + } + + // Try to pretty-print JSON + if (mimeType?.includes("json") || mimeType?.includes("text/event-stream")) { + // Handle SSE: try to pretty-print each data line + if (body.includes("data: ")) { + return ( + 
+          {body.split("\n").map((line) => {
+            if (line.startsWith("data: ") && line !== "data: [DONE]") {
+              try {
+                const parsed = JSON.parse(line.slice(6));
+                return `data: ${JSON.stringify(parsed, null, 2)}`;
+              } catch {
+                return line;
+              }
+            }
+            return line;
+          }).join("\n")}
+
+ ); + } + + try { + const parsed = JSON.parse(body); + return ( + 
+          {JSON.stringify(parsed, null, 2)}
+
+ ); + } catch { + // fall through to raw display + } + } + + return ( + 
+      {body}
+
+ ); +} diff --git a/apps/portal/src/components/KbdBadge.tsx b/apps/portal/src/components/KbdBadge.tsx new file mode 100644 index 000000000..2a9c676c8 --- /dev/null +++ b/apps/portal/src/components/KbdBadge.tsx @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { cn } from "@/lib/utils"; +import { isMac } from "@/hooks/useCommandEnter"; + +interface KbdBadgeProps { + className?: string; +} + +/** + * Small keyboard badge showing the platform-appropriate shortcut + * for "Cmd+Enter" (Mac) or "Ctrl+Enter" (Windows/Linux). + * + * Designed to sit inside a ` + {overflowOpen && ( +
+ {overflowItems.map((item) => { + const isActive = location.pathname.startsWith(item.to); + return ( + + + {item.label} + + ); + })} +
+ )} +
+ )} + + {/* Spacer on very small screens */} +
+ + {/* Admin gear icon */} + + + + + {/* Mobile hamburger ā€” visible on xs only (below sm) */} + + + + + + Navigation + + + +
+ + + {/* Main content */} +
+ +
+ + {/* Version footer */} + +
+ ); +} diff --git a/packages/portal/src/components/LogViewer.tsx b/apps/portal/src/components/LogViewer.tsx similarity index 89% rename from packages/portal/src/components/LogViewer.tsx rename to apps/portal/src/components/LogViewer.tsx index e784a9b4f..8e4708cb6 100644 --- a/packages/portal/src/components/LogViewer.tsx +++ b/apps/portal/src/components/LogViewer.tsx @@ -119,6 +119,14 @@ export function LogViewer({ [{log.source}] )} {log.message} + {log.data && Object.keys(log.data).filter(k => k !== "iteration" && k !== "final").length > 0 && ( + + {Object.entries(log.data) + .filter(([k]) => k !== "iteration" && k !== "final") + .map(([k, v]) => `${k}=${typeof v === "object" ? JSON.stringify(v) : v}`) + .join(" ")} + + )}
); diff --git a/apps/portal/src/components/PromptFeaturePicker.tsx b/apps/portal/src/components/PromptFeaturePicker.tsx new file mode 100644 index 000000000..6436188e4 --- /dev/null +++ b/apps/portal/src/components/PromptFeaturePicker.tsx @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState, useMemo, useRef, useEffect, useCallback } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { api } from "@/lib/api"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; +import { X, Sparkles } from "lucide-react"; + +interface PromptFeaturePickerProps { + selected: string[]; + onChange: (ids: string[]) => void; + /** IDs that were AI-suggested ā€” shown with a sparkle indicator */ + aiSuggested?: string[]; +} + +export function PromptFeaturePicker({ selected, onChange, aiSuggested = [] }: PromptFeaturePickerProps) { + const [query, setQuery] = useState(""); + const [open, setOpen] = useState(false); + const [highlightIdx, setHighlightIdx] = useState(0); + const containerRef = useRef(null); + const inputRef = useRef(null); + + const { data: features = [], isLoading } = useQuery({ + queryKey: ["prompt-features"], + queryFn: () => api.listPromptFeatures(), + }); + + // Filter: show unselected features matching query + const suggestions = useMemo(() => { + const available = features.filter((f) => !selected.includes(f.id)); + if (!query.trim()) return available.slice(0, 8); + const q = query.toLowerCase(); + return available.filter( + (f) => f.id.toLowerCase().includes(q) || f.prompt.toLowerCase().includes(q), + ); + }, [features, selected, query]); + + // Reset highlight when suggestions change + useEffect(() => { + setHighlightIdx(0); + }, [suggestions]); + + // Close dropdown on outside click + useEffect(() => { + const handler = (e: MouseEvent) => { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + setOpen(false); + } + }; + document.addEventListener("mousedown", handler); + return () => document.removeEventListener("mousedown", handler); + }, []); + + const addItem = useCallback( + (id: string) => { + if (!selected.includes(id)) { + onChange([...selected, id]); + } + setQuery(""); + setOpen(false); + inputRef.current?.focus(); + }, + [selected, onChange], + ); + + const removeItem = (id: string) => { + onChange(selected.filter((s) => s !== id)); + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "ArrowDown") { + e.preventDefault(); + setHighlightIdx((i) => Math.min(i + 1, suggestions.length - 1)); + } else if (e.key === "ArrowUp") { + e.preventDefault(); + setHighlightIdx((i) => Math.max(i - 1, 0)); + } else if (e.key === "Enter") { + e.preventDefault(); + if (suggestions[highlightIdx]) { + addItem(suggestions[highlightIdx].id); + } + } else if (e.key === "Escape") { + setOpen(false); + } else if (e.key === "Backspace" && !query && selected.length > 0) { + removeItem(selected[selected.length - 1]); + } + }; + + if (isLoading) { + return ; + } + + return ( +
+ {/* Selected badges */} + {selected.length > 0 && ( +
+ {selected.map((id) => ( + + {aiSuggested.includes(id) && ( + + )} + {id} + removeItem(id)} + /> + + ))} +
+ )} + + {/* Typeahead input */} + { + setQuery(e.target.value); + setOpen(true); + }} + onFocus={() => setOpen(true)} + onKeyDown={handleKeyDown} + placeholder="Type to search prompt featuresā€¦" + className="h-9 font-mono text-sm" + /> + + {/* Dropdown suggestions */} + {open && suggestions.length > 0 && ( +
+
+ {suggestions.map((f, idx) => ( + + ))} +
+
+ )} + + {open && query && suggestions.length === 0 && ( +
+ No matching prompt features +
+ )} +
+ ); +} diff --git a/apps/portal/src/components/PromptFeatureWizard.tsx b/apps/portal/src/components/PromptFeatureWizard.tsx new file mode 100644 index 000000000..c98572f7a --- /dev/null +++ b/apps/portal/src/components/PromptFeatureWizard.tsx @@ -0,0 +1,491 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState, useMemo, useCallback } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { api } from "@/lib/api"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import { Badge } from "@/components/ui/badge"; +import { Separator } from "@/components/ui/separator"; +import { Stepper } from "@/components/Stepper"; +import { PromptFeaturePicker } from "@/components/PromptFeaturePicker"; +import type { PromptFeatureDocument } from "@/types"; +import { + ArrowLeft, + ArrowRight, + Loader2, + Check, + X, + Sparkles, + RefreshCw, + Pencil, +} from "lucide-react"; +import { useCommandEnter } from "@/hooks/useCommandEnter"; +import { KbdBadge } from "@/components/KbdBadge"; + +const STEPS = ["Define Feature", "Review & Create"]; + +/** Convert a behavior description to a snake_case ID suggestion */ +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/['']/g, "") + .replace(/[^a-z0-9]+/g, "_") + .replace(/^[^a-z]+/, "") + .replace(/_+/g, "_") + .replace(/_$/, "") + .slice(0, 40); +} + +export interface PromptFeatureWizardProps { + /** Pre-fill behavior description */ + initialBehavior?: string; + /** Pre-fill feature ID */ + initialId?: string; + /** Pre-fill detection prompt (skips to step 2 when provided) */ + initialPrompt?: string; + /** Called after successful feature creation */ + onCreated?: (feature: PromptFeatureDocument) => void; + /** Called when user clicks Cancel (step 1) or Back at step 1 */ + onCancel?: () => void; +} + +export function PromptFeatureWizard({ + initialBehavior = "", + initialId = "", + initialPrompt = "", + onCreated, + onCancel, +}: PromptFeatureWizardProps) { + const queryClient = useQueryClient(); + + // Wizard step (1 or 2) ā€” skip to 2 if prompt pre-filled + const [step, setStep] = useState(initialPrompt ? 2 : 1); + + // Step 1 fields + const [behavior, setBehavior] = useState(initialBehavior); + const [id, setId] = useState(initialId); + const [idManuallyEdited, setIdManuallyEdited] = useState(!!initialId); + const [idEditMode, setIdEditMode] = useState(false); + const [dependsOn, setDependsOn] = useState([]); + + // Step 2 fields + const [prompt, setPrompt] = useState(initialPrompt); + const [aiGenerated, setAiGenerated] = useState(false); + const [aiModel, setAiModel] = useState(""); + + // AI-suggested dependencies + const [suggestedParents, setSuggestedParents] = useState([]); + const [suggestedChildren, setSuggestedChildren] = useState([]); + const [acceptedChildren, setAcceptedChildren] = useState([]); + + // ID validation + const idValid = useMemo(() => /^[a-z][a-z0-9_]*$/.test(id), [id]); + const canContinue = behavior.trim().length > 0 && id.trim().length > 0 && idValid; + + // Auto-suggest ID from behavior (unless manually edited) + const handleBehaviorChange = useCallback( + (value: string) => { + setBehavior(value); + if (!idManuallyEdited) { + setId(slugify(value)); + } + }, + [idManuallyEdited], + ); + + // Generate prompt mutation + const generateMutation = useMutation({ + mutationFn: (behaviorText: string) => api.generatePromptFeaturePrompt(behaviorText), + onSuccess: (data) => { + setPrompt(data.prompt); + setAiGenerated(true); + setAiModel("AI Generated"); + if (!idManuallyEdited && data.suggestedId) { + setId(data.suggestedId); + } + if (data.suggestedParents?.length) { + setSuggestedParents(data.suggestedParents); + setDependsOn((prev) => [...new Set([...prev, ...data.suggestedParents])]); + } else { + setSuggestedParents([]); + } + if (data.suggestedChildren?.length) { + setSuggestedChildren(data.suggestedChildren); + setAcceptedChildren(data.suggestedChildren); + } else { + setSuggestedChildren([]); + setAcceptedChildren([]); + } + }, + onError: () => { + setPrompt(""); + setAiGenerated(false); + setSuggestedParents([]); + setSuggestedChildren([]); + setAcceptedChildren([]); + }, + }); + + // Create prompt feature mutation + const createMutation = useMutation({ + mutationFn: api.createPromptFeature, + onSuccess: async (data) => { + for (const childId of acceptedChildren) { + try { + const child = await api.getPromptFeature(childId); + const existingDeps = child.dependsOn ?? []; + if (!existingDeps.includes(data.id)) { + await api.updatePromptFeature(childId, { + dependsOn: [...existingDeps, data.id], + }); + } + } catch { + console.warn(`Failed to update child feature ${childId}`); + } + } + queryClient.invalidateQueries({ queryKey: ["prompt-features"] }); + onCreated?.(data); + }, + }); + + // Step 1 ā†’ Step 2 + const handleContinue = () => { + setStep(2); + generateMutation.mutate(behavior.trim()); + }; + + // Step 2 ā†’ Submit + const handleCreate = () => { + if (!id.trim() || !prompt.trim()) return; + createMutation.mutate({ + id: id.trim(), + prompt: prompt.trim(), + dependsOn: dependsOn.length > 0 ? dependsOn : undefined, + }); + }; + + // Cmd+Enter / Ctrl+Enter shortcut for primary action + useCommandEnter( + step === 1 ? handleContinue : handleCreate, + step === 1 ? canContinue : !!prompt.trim() && !createMutation.isPending, + ); + + // Regenerate prompt + const handleRegenerate = () => { + setSuggestedParents([]); + setSuggestedChildren([]); + setAcceptedChildren([]); + generateMutation.mutate(behavior.trim()); + }; + + return ( +
+ {step === 1 ? ( + /* ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ Step 1: Define Feature ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ */ + <> +
+
+
+ +
+
+

Define Prompt Feature

+

+ Describe the characteristic to detect in task prompts +

+
+
+ +
+ + + +
+ +