From 1bd1dbff00236fce7dd9e56b788aeaefb5c42174 Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Tue, 13 Oct 2026 16:47:17 +0200 Subject: [PATCH 1/2] ci(security): temporarily make release Codex scans advisory Signed-off-by: Adrien Langou --- .agents/skills/watch-github-actions/SKILL.md | 6 ++++++ .github/workflows/codex-security.yml | 13 ++++++++++++- .github/workflows/release-tag.yml | 2 ++ .github/workflows/security-scan.yml | 7 ++++++- CI.md | 20 ++++++++++++++------ 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 41b9a3fcff..4a8ac758e3 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -125,6 +125,12 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId ## View Job Logs +`Release Tag` temporarily enables `codex-advisory`: Codex findings, incomplete +coverage, and execution failures do not block release qualification. The +`OpenShell / Codex Security (advisory)` check reports this exception; inspect +the scan job and diagnostics for the actual outcome. Manual scans remain strict +about execution failures, and other qualification suites keep their existing gates. + For `Codex Security`, an exit code of 2 with partial coverage is an incomplete scan, not a HIGH/CRITICAL threshold failure. Check the execution diagnostics in the job summary and the `codex-security-diagnostics--` artifact on diff --git a/.github/workflows/codex-security.yml b/.github/workflows/codex-security.yml index 91426db468..1bfce6e301 100644 --- a/.github/workflows/codex-security.yml +++ b/.github/workflows/codex-security.yml @@ -24,6 +24,10 @@ on: description: Fail on HIGH/CRITICAL security findings default: false type: boolean + advisory: + description: Keep findings and execution failures non-blocking + default: false + type: boolean upload_sarif: description: Upload results to Code Scanning when called from a manual workflow default: true @@ -87,6 +91,7 @@ jobs: # so its preflight never scopes the diff and it seals no draft. runs-on: ubuntu-latest timeout-minutes: 120 + continue-on-error: ${{ inputs.advisory || false }} outputs: base_sha: ${{ steps.range.outputs.base_sha }} candidate_sha: ${{ steps.range.outputs.candidate_sha }} @@ -313,7 +318,7 @@ jobs: fi result: - name: ${{ inputs.fail-on-findings && 'OpenShell / Codex Security' || 'OpenShell / Codex Security (informational)' }} + name: ${{ inputs.advisory && 'OpenShell / Codex Security (advisory)' || inputs.fail-on-findings && 'OpenShell / Codex Security' || 'OpenShell / Codex Security (informational)' }} if: ${{ always() }} needs: analyze runs-on: ubuntu-latest @@ -322,8 +327,14 @@ jobs: - name: Evaluate scanner execution env: ANALYZE_RESULT: ${{ needs.analyze.result }} + ADVISORY: ${{ inputs.advisory || false }} run: | set -euo pipefail + if [ "$ADVISORY" = "true" ]; then + echo "::warning::Codex Security is advisory; findings and execution failures do not block release qualification." + echo "Codex Security is advisory. Inspect the scan job and execution diagnostics for its actual outcome." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi if [ "$ANALYZE_RESULT" != "success" ]; then echo "::error::Codex Security release qualification did not complete successfully." exit 1 diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index f31c16a4fc..d3ad2654a0 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -141,6 +141,8 @@ jobs: with: candidate_ref: ${{ inputs.tag || github.ref_name }} fail-on-codex-findings: true + # Temporary (#4466): retain Codex evidence while reliability is investigated. + codex-advisory: true fail-on-static-findings: false images: | ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.source_sha }} diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 36a7fec8c3..a836346e6c 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -36,6 +36,10 @@ on: type: string workflow_call: inputs: + codex-advisory: + description: Keep Codex findings and execution failures non-blocking + default: false + type: boolean candidate_ref: description: Release tag to scan; Codex runs only for pre-release tags required: true @@ -73,7 +77,7 @@ on: permissions: contents: read -# Only the finding threshold is overridable; execution failures remain fatal. +# Execution failures remain fatal unless the caller opts into Codex advisory mode. jobs: codex: name: Codex Security @@ -88,6 +92,7 @@ jobs: stable_ref: ${{ inputs.stable_ref }} allow_full_bootstrap: ${{ inputs.allow_full_bootstrap }} fail-on-findings: ${{ inputs.fail-on-codex-findings }} + advisory: ${{ inputs.codex-advisory || false }} upload_sarif: true secrets: CODEX_SECURITY_API_KEY: ${{ secrets.CODEX_SECURITY_API_KEY }} diff --git a/CI.md b/CI.md index f2305b963c..a21a12fd2f 100644 --- a/CI.md +++ b/CI.md @@ -238,12 +238,17 @@ Actions or call it from another workflow. All applicable children analyze the candidate snapshot. Cargo Deny uses its existing NVIDIA self-hosted runner and CI container. -Tagged releases treat Cargo Deny and Codex Security findings as failures of the -currently implemented qualification profile. A profile failure prevents both -pre-release and stable publication. CodeQL, Trivy, and Zizmor findings are -temporarily informational for tagged releases: the existing findings were +Tagged releases treat Cargo Deny findings as failures of the currently +implemented qualification profile. Codex Security is temporarily advisory: +findings, incomplete coverage, and execution failures do not block publication. +The scan still runs and retains its existing reports and public execution +diagnostics; its advisory check does not assert a successful scan. Restore +Codex enforcement by removing `codex-advisory: true` from `Release Tag`. +A profile failure prevents both pre-release and stable publication. +CodeQL, Trivy, and Zizmor findings are temporarily informational for tagged +releases: the existing findings were reviewed and accepted for v0.1.0 and will be addressed in 0.1.x releases. -Scanner failures still fail qualification. +Other scanner failures still fail qualification. Failed pre-release builds send notifications via Slack using a webhook and at-mentioning the triage engineer with a link to the failure. The webhook is @@ -315,7 +320,10 @@ on HIGH/CRITICAL findings from CodeQL, Trivy, and Zizmor when `fail-on-static-findings` is `true`. Both inputs default to `true`. Set either input to `false` to make only that finding class informational. Scanner setup, execution, and report publication errors still fail. Reports are published -before the finding threshold is enforced. +before the finding threshold is enforced. Reusable callers can explicitly set +`codex-advisory: true` to tolerate all Codex job failures; `Release Tag` +temporarily enables this exception. It defaults to `false` and is not exposed +on manual dispatches. Codex uses each finding's severity; CodeQL uses the rule's security score (at least 7.0); Trivy uses `HIGH,CRITICAL`, including vulnerabilities without an From 36a30a30fc679845ea717ba925dac0c51002142c Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Tue, 13 Oct 2026 17:17:40 +0200 Subject: [PATCH 2/2] ci(security): remove Codex from release scans Signed-off-by: Adrien Langou --- .agents/skills/watch-github-actions/SKILL.md | 8 ++- .github/workflows/codex-security.yml | 13 +---- .github/workflows/release-tag.yml | 4 -- .github/workflows/security-scan.yml | 54 ++----------------- CI.md | 56 ++++++++------------ 5 files changed, 29 insertions(+), 106 deletions(-) diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 4a8ac758e3..8ba30a8f54 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -125,11 +125,9 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId ## View Job Logs -`Release Tag` temporarily enables `codex-advisory`: Codex findings, incomplete -coverage, and execution failures do not block release qualification. The -`OpenShell / Codex Security (advisory)` check reports this exception; inspect -the scan job and diagnostics for the actual outcome. Manual scans remain strict -about execution failures, and other qualification suites keep their existing gates. +`Release Tag` temporarily omits Codex Security from its `Security Scan` job. +Other scanners and qualification gates still run. Use the standalone +`Codex Security Release Qualification` workflow for manual investigation. For `Codex Security`, an exit code of 2 with partial coverage is an incomplete scan, not a HIGH/CRITICAL threshold failure. Check the execution diagnostics in diff --git a/.github/workflows/codex-security.yml b/.github/workflows/codex-security.yml index 1bfce6e301..91426db468 100644 --- a/.github/workflows/codex-security.yml +++ b/.github/workflows/codex-security.yml @@ -24,10 +24,6 @@ on: description: Fail on HIGH/CRITICAL security findings default: false type: boolean - advisory: - description: Keep findings and execution failures non-blocking - default: false - type: boolean upload_sarif: description: Upload results to Code Scanning when called from a manual workflow default: true @@ -91,7 +87,6 @@ jobs: # so its preflight never scopes the diff and it seals no draft. runs-on: ubuntu-latest timeout-minutes: 120 - continue-on-error: ${{ inputs.advisory || false }} outputs: base_sha: ${{ steps.range.outputs.base_sha }} candidate_sha: ${{ steps.range.outputs.candidate_sha }} @@ -318,7 +313,7 @@ jobs: fi result: - name: ${{ inputs.advisory && 'OpenShell / Codex Security (advisory)' || inputs.fail-on-findings && 'OpenShell / Codex Security' || 'OpenShell / Codex Security (informational)' }} + name: ${{ inputs.fail-on-findings && 'OpenShell / Codex Security' || 'OpenShell / Codex Security (informational)' }} if: ${{ always() }} needs: analyze runs-on: ubuntu-latest @@ -327,14 +322,8 @@ jobs: - name: Evaluate scanner execution env: ANALYZE_RESULT: ${{ needs.analyze.result }} - ADVISORY: ${{ inputs.advisory || false }} run: | set -euo pipefail - if [ "$ADVISORY" = "true" ]; then - echo "::warning::Codex Security is advisory; findings and execution failures do not block release qualification." - echo "Codex Security is advisory. Inspect the scan job and execution diagnostics for its actual outcome." >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi if [ "$ANALYZE_RESULT" != "success" ]; then echo "::error::Codex Security release qualification did not complete successfully." exit 1 diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index d3ad2654a0..7ced9d4252 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -140,16 +140,12 @@ jobs: uses: ./.github/workflows/security-scan.yml with: candidate_ref: ${{ inputs.tag || github.ref_name }} - fail-on-codex-findings: true - # Temporary (#4466): retain Codex evidence while reliability is investigated. - codex-advisory: true fail-on-static-findings: false images: | ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.source_sha }} ghcr.io/nvidia/openshell/sandbox:${{ needs.compute-versions.outputs.source_sha }} ghcr.io/nvidia/openshell/supervisor:${{ needs.compute-versions.outputs.source_sha }} secrets: - CODEX_SECURITY_API_KEY: ${{ secrets.CODEX_SECURITY_API_KEY }} CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} prepare-integration: diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index a836346e6c..f224f8ab14 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -7,21 +7,9 @@ on: workflow_dispatch: inputs: candidate_ref: - description: Release tag to scan; Codex runs only for pre-release tags + description: Release tag to scan required: true type: string - stable_ref: - description: Optional previous stable tag override for Codex - default: "" - type: string - allow_full_bootstrap: - description: Allow a full Codex scan when no previous stable tag exists - default: false - type: boolean - fail-on-codex-findings: - description: Fail when Codex Security reports findings - default: true - type: boolean fail-on-static-findings: description: Fail when CodeQL, Trivy, or Zizmor reports HIGH/CRITICAL findings default: true @@ -36,26 +24,10 @@ on: type: string workflow_call: inputs: - codex-advisory: - description: Keep Codex findings and execution failures non-blocking - default: false - type: boolean candidate_ref: - description: Release tag to scan; Codex runs only for pre-release tags + description: Release tag to scan required: true type: string - stable_ref: - description: Optional previous stable tag override for Codex - default: "" - type: string - allow_full_bootstrap: - description: Allow a full Codex scan when no previous stable tag exists - default: false - type: boolean - fail-on-codex-findings: - description: Fail when Codex Security reports findings - default: true - type: boolean fail-on-static-findings: description: Fail when CodeQL, Trivy, or Zizmor reports HIGH/CRITICAL findings default: true @@ -69,34 +41,14 @@ on: default: "" type: string secrets: - CODEX_SECURITY_API_KEY: - required: true CACHIX_AUTH_TOKEN: required: false permissions: contents: read -# Execution failures remain fatal unless the caller opts into Codex advisory mode. +# Only the finding threshold is overridable; execution failures remain fatal. jobs: - codex: - name: Codex Security - if: contains(inputs.candidate_ref, '-pre.') - uses: ./.github/workflows/codex-security.yml - permissions: - actions: read - contents: read - security-events: write - with: - candidate_ref: ${{ inputs.candidate_ref }} - stable_ref: ${{ inputs.stable_ref }} - allow_full_bootstrap: ${{ inputs.allow_full_bootstrap }} - fail-on-findings: ${{ inputs.fail-on-codex-findings }} - advisory: ${{ inputs.codex-advisory || false }} - upload_sarif: true - secrets: - CODEX_SECURITY_API_KEY: ${{ secrets.CODEX_SECURITY_API_KEY }} - codeql: name: CodeQL uses: ./.github/workflows/codeql.yml diff --git a/CI.md b/CI.md index a21a12fd2f..f60fe0a783 100644 --- a/CI.md +++ b/CI.md @@ -201,9 +201,10 @@ scanners upload results directly: E2E test code are excluded. It runs nightly on `main`, remains manually dispatchable for diagnostics, uploads results to Code Scanning, and retains workflow artifacts. -- `Codex Security Release Qualification` analyzes each `vX.Y.Z-pre.N` - candidate against the previous stable release. Every candidate in a release - train therefore rescans the cumulative stable-to-candidate diff. It calls +- `Codex Security Release Qualification` can manually analyze a `vX.Y.Z-pre.N` + candidate against the previous stable release. It is temporarily excluded + from tagged-release qualification. Each scan reviews the cumulative + stable-to-candidate diff. It calls `https://inference-api.nvidia.com/v1` with `openai/openai/gpt-5.6-sol` at medium reasoning effort. The `CODEX_SECURITY_API_KEY` secret must contain an API key authorized for that @@ -232,23 +233,18 @@ nix develop --command zizmor --offline --persona=regular --min-severity=high --n `Security Scan` (`.github/workflows/security-scan.yml`) calls CodeQL, Trivy, Cargo Deny, and Workflow Security Reports (Actionlint/Zizmor) in parallel for a -release tag. It additionally calls Codex Security for `vX.Y.Z-pre.N` tags, which -Codex compares with the previous stable release. Run the parent manually from +release tag. Codex Security is temporarily excluded; its standalone workflow +remains available for manual investigation. Run the parent manually from Actions or call it from another workflow. All applicable children analyze the candidate snapshot. Cargo Deny uses its existing NVIDIA self-hosted runner and CI container. -Tagged releases treat Cargo Deny findings as failures of the currently -implemented qualification profile. Codex Security is temporarily advisory: -findings, incomplete coverage, and execution failures do not block publication. -The scan still runs and retains its existing reports and public execution -diagnostics; its advisory check does not assert a successful scan. Restore -Codex enforcement by removing `codex-advisory: true` from `Release Tag`. -A profile failure prevents both pre-release and stable publication. -CodeQL, Trivy, and Zizmor findings are temporarily informational for tagged -releases: the existing findings were +Tagged releases treat Cargo Deny findings as failures of the +currently implemented qualification profile. A profile failure prevents both +pre-release and stable publication. CodeQL, Trivy, and Zizmor findings are +temporarily informational for tagged releases: the existing findings were reviewed and accepted for v0.1.0 and will be addressed in 0.1.x releases. -Other scanner failures still fail qualification. +Scanner failures still fail qualification. Failed pre-release builds send notifications via Slack using a webhook and at-mentioning the triage engineer with a link to the failure. The webhook is @@ -260,7 +256,6 @@ webhook skips sending; an unset mention sends without a mention. ```shell gh workflow run security-scan.yml --ref main \ -f candidate_ref=v0.1.1-pre.1 \ - -F fail-on-codex-findings=false \ -F fail-on-static-findings=false ``` @@ -282,15 +277,12 @@ jobs: security-events: write with: candidate_ref: ${{ needs.build.outputs.candidate_tag }} - stable_ref: v0.1.0 # Optional; otherwise resolved automatically. images: | ${{ needs.build.outputs.gateway_image }} ${{ needs.build.outputs.sandbox_image }} charts: ${{ needs.build.outputs.chart_ref }} - fail-on-codex-findings: true fail-on-static-findings: true secrets: - CODEX_SECURITY_API_KEY: ${{ secrets.CODEX_SECURITY_API_KEY }} CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} ``` @@ -310,32 +302,28 @@ The summary contains qualification results and policy coverage only; artifact identity belongs in the release manifest. The run ID and attempt distinguish reruns without overwriting earlier evidence. -`CODEX_SECURITY_API_KEY` is required; `CACHIX_AUTH_TOKEN` is optional. The parent -publishes SARIF, including Codex results for manual parent runs. Codex keeps its -release-train category on `main`; CodeQL, Trivy, and workflow reports publish -against the candidate tag and commit. Existing standalone triggers stay active. +`CACHIX_AUTH_TOKEN` is optional. The parent publishes SARIF from CodeQL, Trivy, +and workflow reports against the candidate tag and commit. Existing standalone +triggers stay active. -The parent fails on Codex findings when `fail-on-codex-findings` is `true` and -on HIGH/CRITICAL findings from CodeQL, Trivy, and Zizmor when -`fail-on-static-findings` is `true`. Both inputs default to `true`. Set either -input to `false` to make only that finding class informational. Scanner setup, +The parent fails on HIGH/CRITICAL findings from CodeQL, Trivy, and Zizmor when +`fail-on-static-findings` is `true`, which is the default. Set it to `false` +to make those findings informational. Scanner setup, execution, and report publication errors still fail. Reports are published -before the finding threshold is enforced. Reusable callers can explicitly set -`codex-advisory: true` to tolerate all Codex job failures; `Release Tag` -temporarily enables this exception. It defaults to `false` and is not exposed -on manual dispatches. +before the finding threshold is enforced. -Codex uses each finding's severity; CodeQL uses the rule's security score +CodeQL uses the rule's security score (at least 7.0); Trivy uses `HIGH,CRITICAL`, including vulnerabilities without an upstream fix; Zizmor uses its High severity. Actionlint remains informational. Existing scanner exceptions still apply. Cargo Deny runs only `cargo deny check advisories` in the parent. It keeps its -native failure behavior and configured exceptions regardless of either finding +native failure behavior and configured exceptions regardless of the finding threshold input; it has no HIGH/CRITICAL filter. Its standalone runs still check all dependency policies. -Codex scans the cumulative diff from `stable_ref` to `candidate_ref`. Both inputs +The standalone Codex workflow scans the cumulative diff from `stable_ref` to +`candidate_ref`. Both inputs are tag names, not arbitrary commit SHAs. Omit `stable_ref` to resolve the previous stable automatically; set `allow_full_bootstrap: true` to permit a full scan when no previous stable exists.