From ac1bcbc37ebfa1a5ac8c819160b2ac4b5bc9e62d Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 13 May 2026 17:24:00 -0700 Subject: [PATCH 1/6] ci(release): drop continue-on-error from changesets publish step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `Create Release Pull Request or Publish to npm` step had `continue-on-error: true`, which masks publish failures and lets the workflow complete green. In release run 25827737203 the npm publish for `@hyperdx/cli@0.4.1` failed with `ENEEDAUTH` (a separate npm trusted-publisher config typo, fixed in the npm UI), but because `changeset publish` exits non-zero, `changesets/action` aborts before its `createRelease` loop — so neither the `@hyperdx/cli@0.4.1` nor the four private-package GitHub Releases (api/app/common-utils/otel-collector @ 2.25.0/0.19.0) ever got created, and we only noticed because they were absent from the Releases page. Removing the flag means the next failure surfaces immediately instead of silently shipping a half-release. The subsequent jobs that should still run on a no-op release (`check_version`, Docker builds, downstream notifications) are already gated on `check_changesets.outputs.changeset_outputs_hasChangesets == 'false'`, not on the step succeeding, so dropping `continue-on-error` does not change their behavior on changeset-PR commits. --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c62c307464..a3bf37a33b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,6 @@ jobs: run: make ci-build - name: Create Release Pull Request or Publish to npm if: always() - continue-on-error: true id: changesets uses: changesets/action@v1 with: From 31d2c3f07fd706543955c1bf3414d5e26026ef9a Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 13 May 2026 17:31:59 -0700 Subject: [PATCH 2/6] ci(release): upgrade npm to enable OIDC trusted publishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check_changesets job runs on Node 22.21.1, which ships npm 10.9.4. Native client-side OIDC trusted publishing for `npm publish` only landed in npm 11.5.1 — npm 10.x has no OIDC code path, so when changesets/action hands off to npm publish, it falls back to looking for NODE_AUTH_TOKEN (which is deliberately not set, to avoid conflicting with OIDC), then exits ENEEDAUTH. The April 23 release that originally introduced OIDC (commit 3ae1b85d8) only appeared to work because @hyperdx/cli@0.4.0 had been published manually out of band — the workflow's own npm publish E404'd, then a manual npm publish landed, and the next workflow run on the OIDC commit saw "already published" and silently exited zero. Today's 0.4.1 release had no out-of-band publish to fall back on, so the underlying ENEEDAUTH surfaced. Pinning to `npm@latest` installs >= 11.5.1, which can perform the OIDC handshake using ACTIONS_ID_TOKEN_REQUEST_URL / ACTIONS_ID_TOKEN_REQUEST_TOKEN that GHA already injects when `permissions: id-token: write` is set on the workflow. --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3bf37a33b..0f8c6c6cf1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,11 @@ jobs: cache: 'yarn' # Note: Do NOT set registry-url here — it creates a NODE_AUTH_TOKEN # that conflicts with npm OIDC trusted publishing + # npm OIDC trusted publishing requires npm >= 11.5.1 client-side. Node 22 + # LTS ships npm 10.x, which has no OIDC code path and falls back to + # NODE_AUTH_TOKEN, failing ENEEDAUTH. + - name: Upgrade npm for OIDC trusted publishing + run: npm install -g npm@latest - name: Configure npm registry run: npm config set registry https://registry.npmjs.org/ - name: Install root dependencies From c78c0b062111b66e9e3fd8ebb43f9096726cd765 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 13 May 2026 17:35:50 -0700 Subject: [PATCH 3/6] ci(release): allow manual workflow_dispatch trigger Needed so we can manually re-run the release workflow against a clean main (e.g. after this PR merges, to retry the cli@0.4.1 publish + the five missing 2.25.0 / 0.19.0 GH Releases). gh run rerun uses the original commit's workflow file and so cannot test fixes added later. --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f8c6c6cf1..570355691c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,6 +2,7 @@ name: Release on: push: branches: [main] + workflow_dispatch: permissions: contents: write packages: write From 44abbb9396998a135a315a8ba7314c77f1beb1b9 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 13 May 2026 17:38:01 -0700 Subject: [PATCH 4/6] ci(release): add slack-notify-failure job Mirrors the slack-notify-failure pattern already present in release-nightly.yml: a job gated on `if: failure() && always()` that waits on every major job, lists the failed ones via listJobsForWorkflowRun, and posts a danger-colored Slack message to SLACK_WEBHOOK_URL_ENG_NOTIFS with the commit SHA + author. Combined with the continue-on-error removal in this PR, the next release.yml failure surfaces on Slack instead of going green-and-silent. --- .github/workflows/release.yml | 74 +++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 570355691c..03d2b1cb7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -692,6 +692,79 @@ jobs: } }); + # --------------------------------------------------------------------------- + # Failure notification + # --------------------------------------------------------------------------- + slack-notify-failure: + name: Notify Slack on Failure + needs: + [ + check_changesets, + publish-app, + publish-otel-collector, + publish-local, + publish-all-in-one, + release-cli, + notify_helm_charts, + notify_ch, + notify_clickhouse_clickstack, + ] + runs-on: ubuntu-24.04 + if: failure() && always() + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Get failed jobs + id: get_failed_jobs + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const response = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + const failedJobs = response.data.jobs + .filter(job => job.status === 'completed' && job.conclusion === 'failure') + .map(job => job.name) + .join(', '); + + core.setOutput('failed_jobs', failedJobs); + - name: Slack Notification + uses: 8398a7/action-slack@v3 + with: + status: custom + fields: repo,workflow,commit,author + custom_payload: | + { + "text": "Release Failed", + "attachments": [{ + "color": "danger", + "fields": [ + { + "title": "Failed Build", + "value": "${{ steps.get_failed_jobs.outputs.failed_jobs }}", + "short": false + }, + { + "title": "Commit", + "value": "", + "short": false + }, + { + "title": "Author", + "value": "${{ github.actor }}", + "short": true + } + ] + }] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_ENG_NOTIFS }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + otel-cicd-action: if: always() name: OpenTelemetry Export Trace @@ -707,6 +780,7 @@ jobs: notify_helm_charts, notify_ch, notify_clickhouse_clickstack, + slack-notify-failure, ] steps: - name: Export workflow From cb78e1de5fa057e8e244cbe08b8e5d1c5a93137f Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 13 May 2026 17:58:32 -0700 Subject: [PATCH 5/6] ci(release): simplify slack-notify-failure to match Pull Upstream style Drops the actions/github-script step that listed failed job names and the Checkout step. Keeps the needs: list (still required so the job evaluates after the workflow's terminal jobs) and switches the gate to a bare `if: failure()` matching the pattern used by the Pull Upstream workflow. Adds a Run-URL field so the alert links straight back to the failing workflow run. --- .github/workflows/release.yml | 39 ++++++++--------------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03d2b1cb7c..a54a9e1c59 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -696,7 +696,6 @@ jobs: # Failure notification # --------------------------------------------------------------------------- slack-notify-failure: - name: Notify Slack on Failure needs: [ check_changesets, @@ -710,53 +709,33 @@ jobs: notify_clickhouse_clickstack, ] runs-on: ubuntu-24.04 - if: failure() && always() + if: failure() steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Get failed jobs - id: get_failed_jobs - uses: actions/github-script@v9 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const response = await github.rest.actions.listJobsForWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - const failedJobs = response.data.jobs - .filter(job => job.status === 'completed' && job.conclusion === 'failure') - .map(job => job.name) - .join(', '); - - core.setOutput('failed_jobs', failedJobs); - name: Slack Notification uses: 8398a7/action-slack@v3 with: status: custom - fields: repo,workflow,commit,author + fields: repo,workflow custom_payload: | { - "text": "Release Failed", + "text": "Release workflow failed!", "attachments": [{ "color": "danger", "fields": [ - { - "title": "Failed Build", - "value": "${{ steps.get_failed_jobs.outputs.failed_jobs }}", - "short": false - }, { "title": "Commit", "value": "", "short": false }, { - "title": "Author", + "title": "Actor", "value": "${{ github.actor }}", "short": true + }, + { + "title": "Run", + "value": "", + "short": true } ] }] From 3266d0ce44baea26916242d757d2c6af3a38cae8 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 13 May 2026 18:00:55 -0700 Subject: [PATCH 6/6] ci(release): include failed step name in slack alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Get failed steps script that calls listJobsForWorkflowRun, walks each failed job's steps, and emits a 'Job → Step' list. The fallback '(unknown step)' covers job-level failures where no individual step conclusion is 'failure' (runner timeout, infra cancellation, etc.) so the alert never lands with an empty Failed step(s) field. For today's outage the alert would have read: Check Changesets → Create Release Pull Request or Publish to npm --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a54a9e1c59..b899b51239 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -711,6 +711,29 @@ jobs: runs-on: ubuntu-24.04 if: failure() steps: + - name: Get failed steps + id: get_failed_steps + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + }); + + const failedSteps = data.jobs + .filter(job => job.conclusion === 'failure') + .flatMap(job => { + const stepNames = (job.steps ?? []) + .filter(step => step.conclusion === 'failure') + .map(step => `${job.name} → ${step.name}`); + return stepNames.length > 0 ? stepNames : [`${job.name} → (unknown step)`]; + }) + .join(', '); + + core.setOutput('failed_steps', failedSteps || 'unknown'); - name: Slack Notification uses: 8398a7/action-slack@v3 with: @@ -722,6 +745,11 @@ jobs: "attachments": [{ "color": "danger", "fields": [ + { + "title": "Failed step(s)", + "value": "${{ steps.get_failed_steps.outputs.failed_steps }}", + "short": false + }, { "title": "Commit", "value": "",