From a71480524956aaa960e23a0005d7ce9199e9101c Mon Sep 17 00:00:00 2001
From: Ralf Juengling <rjuengling@nvidia.com>
Date: Fri, 30 Oct 2026 11:01:48 -0700
Subject: [PATCH 1/3] ci: route all artifact downloads through
 download-artifact-retry

Follow-on to #3037.  A job testing against the prior branch recently
hit:

  HTTP 503: Egress is over the account limit
  (productionresultssa17.blob.core.windows.net)

The failing step used raw 'gh run download' with no retry.  Some
sibling steps were already protected by the download-artifact-retry
composite action introduced in #3037, but that action only accepted an
exact artifact name, and several steps bypassed it entirely.

This commit:

* Extends the action with 'pattern' and 'merge-multiple' inputs so it
  can replace glob-pattern gh run download calls.  Adds mutual-
  exclusion validation (exactly one of 'name' or 'pattern' must be
  supplied).  Adds a third attempt with twice the base delay (15s ->
  30s) to give egress throttling more time to clear.

* Splits the backport lookup-then-download steps in
  test-wheel-linux.yml, test-wheel-windows.yml, and build-wheel.yml
  into two steps each: a lookup step that emits run-id and
  artifact-pattern as step outputs, then an action download step.
  Adds a verify step in the test-wheel workflows to confirm exactly
  one bindings wheel arrived (previously guaranteed by compgen -G).

* Repoints all existing pinned @f20d661 callers in the test-wheel
  workflows to $/ so they pick up the new third attempt.

* Swaps every direct actions/download-artifact call in build-wheel.yml,
  build-docs.yml, and coverage.yml for the action.

* Adds an internal retry loop (5 attempts, 15s/30s/60s/120s/120s
  delays) to ci/tools/download-wheels so the release path is also
  covered.

* Adds build-docs.yml to the actionlint.yaml $/ suppression list.
---
 .github/actionlint.yaml                       |  3 +
 .../download-artifact-retry/action.yml        | 78 ++++++++++++++++---
 .github/workflows/build-docs.yml              | 18 ++---
 .github/workflows/build-wheel.yml             | 37 ++++-----
 .github/workflows/coverage.yml                | 10 +--
 .github/workflows/test-wheel-linux.yml        | 67 +++++++++-------
 .github/workflows/test-wheel-windows.yml      | 66 +++++++++-------
 ci/tools/download-wheels                      | 23 +++++-
 8 files changed, 205 insertions(+), 97 deletions(-)

diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
index 57724681f83..25cc1e40959 100644
--- a/.github/actionlint.yaml
+++ b/.github/actionlint.yaml
@@ -15,6 +15,9 @@ paths:
   ".github/workflows/build-docs.yml":
     ignore:
       - 'unexpected key "queue" for "concurrency" section'
+      # GitHub supports self-repository ($/) action references, but the latest
+      # actionlint release does not yet recognize them (rhysd/actionlint#711).
+      - 'specifying action "\$/[^"]+" in invalid format because ref is missing'
   ".github/workflows/cleanup-pr-previews.yml":
     ignore:
       - 'unexpected key "queue" for "concurrency" section'
diff --git a/.github/actions/download-artifact-retry/action.yml b/.github/actions/download-artifact-retry/action.yml
index df7be01c8c4..335257d2890 100644
--- a/.github/actions/download-artifact-retry/action.yml
+++ b/.github/actions/download-artifact-retry/action.yml
@@ -5,16 +5,33 @@
 name: Download artifact (with retry)
 
 description: >-
-  Wrapper around actions/download-artifact that retries once after a short
-  delay when the first attempt fails.  Designed to absorb transient
-  infrastructure flakes such as "other side closed" without failing the job.
+  Wrapper around actions/download-artifact that retries up to three times with
+  exponential backoff when a download fails.  Handles both transient TCP errors
+  such as "other side closed" and storage-layer throttling such as
+  "HTTP 503: Egress is over the account limit".  Accepts either an exact
+  artifact name or a glob pattern; supply exactly one.
 
 inputs:
   name:
-    description: "Name of the artifact to download"
-    required: true
+    description: >-
+      Exact name of the artifact to download.  Mutually exclusive with
+      'pattern'; supply exactly one of the two.
+    required: false
+    default: ""
+  pattern:
+    description: >-
+      Glob pattern matching one or more artifact names.  Mutually exclusive
+      with 'name'; supply exactly one of the two.
+    required: false
+    default: ""
+  merge-multiple:
+    description: >-
+      When true and 'pattern' is set, merge files from all matched artifacts
+      directly into 'path' instead of placing each in its own subdirectory.
+    required: false
+    default: "false"
   path:
-    description: "Destination path for the downloaded artifact"
+    description: "Destination path for the downloaded artifact(s)"
     required: true
   run-id:
     description: >-
@@ -32,39 +49,80 @@ inputs:
     required: false
     default: ""
   retry-delay-seconds:
-    description: "Seconds to wait between the first failure and the retry"
+    description: "Seconds to wait between the first failure and the second attempt"
     required: false
     default: "15"
 
 runs:
   using: composite
   steps:
+    - name: Validate inputs
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        INPUT_NAME: ${{ inputs.name }}
+        INPUT_PATTERN: ${{ inputs.pattern }}
+      run: |
+        if [[ -n "${INPUT_NAME}" && -n "${INPUT_PATTERN}" ]]; then
+          echo "::error::Specify exactly one of 'name' or 'pattern', not both"
+          exit 1
+        fi
+        if [[ -z "${INPUT_NAME}" && -z "${INPUT_PATTERN}" ]]; then
+          echo "::error::Specify exactly one of 'name' or 'pattern'"
+          exit 1
+        fi
+
     - name: Download artifact (attempt 1)
       id: attempt1
       continue-on-error: true
       uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
       with:
         name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: ${{ inputs.merge-multiple }}
         path: ${{ inputs.path }}
         run-id: ${{ inputs.run-id }}
         github-token: ${{ inputs.github-token }}
 
-    - name: Wait before retry
+    - name: Wait before retry (attempt 2)
       if: ${{ steps.attempt1.outcome == 'failure' }}
       shell: bash --noprofile --norc -euo pipefail {0}
       env:
-        # Pass through env instead of interpolating the input into the shell
-        # script, so a non-numeric value cannot inject commands.
         RETRY_DELAY_SECONDS: ${{ inputs.retry-delay-seconds }}
       run: |
         echo "::warning::Artifact download failed (attempt 1); retrying in ${RETRY_DELAY_SECONDS}s"
         sleep "$RETRY_DELAY_SECONDS"
 
     - name: Download artifact (attempt 2)
+      id: attempt2
       if: ${{ steps.attempt1.outcome == 'failure' }}
+      continue-on-error: true
+      uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+      with:
+        name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: ${{ inputs.merge-multiple }}
+        path: ${{ inputs.path }}
+        run-id: ${{ inputs.run-id }}
+        github-token: ${{ inputs.github-token }}
+
+    - name: Wait before retry (attempt 3)
+      if: ${{ steps.attempt1.outcome == 'failure' && steps.attempt2.outcome == 'failure' }}
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        RETRY_DELAY_SECONDS: ${{ inputs.retry-delay-seconds }}
+      run: |
+        # Double the delay for the second retry to give egress throttling more time to clear.
+        RETRY_DELAY_SECONDS=$(( RETRY_DELAY_SECONDS * 2 ))
+        echo "::warning::Artifact download failed (attempt 2); retrying in ${RETRY_DELAY_SECONDS}s"
+        sleep "$RETRY_DELAY_SECONDS"
+
+    - name: Download artifact (attempt 3)
+      if: ${{ steps.attempt1.outcome == 'failure' && steps.attempt2.outcome == 'failure' }}
       uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
       with:
         name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: ${{ inputs.merge-multiple }}
         path: ${{ inputs.path }}
         run-id: ${{ inputs.run-id }}
         github-token: ${{ inputs.github-token }}
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
index b1c029f62cf..b734fdd8c73 100644
--- a/.github/workflows/build-docs.yml
+++ b/.github/workflows/build-docs.yml
@@ -144,7 +144,7 @@ jobs:
           echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/$CUDA_BINDINGS_SOURCE_DIR/dist")" >> $GITHUB_ENV
 
       - name: Download cuda-python build artifacts
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-python-wheel
           path: .
@@ -157,7 +157,7 @@ jobs:
           ls -lahR .
 
       - name: Download cuda-pathfinder build artifacts
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-pathfinder-wheel
           path: ./cuda_pathfinder
@@ -171,17 +171,17 @@ jobs:
 
       - name: Download cuda.bindings build artifacts
         if: ${{ !inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
 
       - name: Download cuda.bindings build artifacts
         if: ${{ inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           pattern: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
-          merge-multiple: true
+          merge-multiple: "true"
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
           run-id: ${{ inputs.run-id }}
           github-token: ${{ github.token }}
@@ -193,17 +193,17 @@ jobs:
 
       - name: Download cuda.core build artifacts
         if: ${{ !inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
 
       - name: Download cuda.core build artifacts
         if: ${{ inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           pattern: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
-          merge-multiple: true
+          merge-multiple: "true"
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
           run-id: ${{ inputs.run-id }}
           github-token: ${{ github.token }}
@@ -366,7 +366,7 @@ jobs:
           ref: ${{ inputs.git-tag }}
 
       - name: Download docs deployment payload
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ needs.build.outputs.deployment-artifact-name }}
           path: artifacts/docs/
diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml
index 9b2be767828..03dd258fa19 100644
--- a/.github/workflows/build-wheel.yml
+++ b/.github/workflows/build-wheel.yml
@@ -170,7 +170,7 @@ jobs:
 
       - name: Download reusable cuda.pathfinder wheel
         if: ${{ env.BUILD_PATHFINDER != 'true' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-pathfinder-wheel
           path: cuda_pathfinder
@@ -271,7 +271,7 @@ jobs:
 
       - name: Download reusable cuda.bindings wheel
         if: ${{ env.BUILD_BINDINGS != 'true' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
@@ -395,7 +395,7 @@ jobs:
 
       - name: Download reusable cuda.core wheel
         if: ${{ env.BUILD_CORE != 'true' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
@@ -413,7 +413,7 @@ jobs:
 
       - name: Download reusable cuda-python wheel
         if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-python-wheel
           path: cuda_python
@@ -542,14 +542,15 @@ jobs:
             ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.lib
           if-no-files-found: error
 
-      - name: Download cuda.bindings build artifacts from the prior branch
+      - name: Look up prior-branch cuda.bindings run ID
+        id: prior-bindings
         if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           if ! (command -v gh 2>&1 >/dev/null); then
             # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
-            # gh is needed for artifact fetching.
+            # gh is needed for lookup-run-id, which calls gh api.
             mkdir -p -m 755 /etc/apt/keyrings \
                   && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
                   && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
@@ -565,18 +566,18 @@ jobs:
             --branch "${OLD_BRANCH}" \
             --artifact "${OLD_ARTIFACT_PATTERN}" \
             NVIDIA/cuda-python "CI")
-          PREV_BINDINGS_DIR="cuda_bindings_13/dist-prev"
-
-          gh run download \
-            "${LATEST_PRIOR_RUN_ID}" \
-            -p "${OLD_ARTIFACT_PATTERN}" \
-            -R NVIDIA/cuda-python
-          OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
-          test -d "${OLD_ARTIFACT_DIR}"
-          ls -al "${OLD_ARTIFACT_DIR}"
-          mkdir -p "${PREV_BINDINGS_DIR}"
-          mv "${OLD_ARTIFACT_DIR}"/*.whl "${PREV_BINDINGS_DIR}"
-          rmdir "${OLD_ARTIFACT_DIR}"
+          echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}"
+          echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}"
+
+      - name: Download prior-branch cuda.bindings build artifacts
+        if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}
+        uses: $/.github/actions/download-artifact-retry
+        with:
+          pattern: ${{ steps.prior-bindings.outputs.artifact-pattern }}
+          merge-multiple: "true"
+          path: cuda_bindings_13/dist-prev
+          run-id: ${{ steps.prior-bindings.outputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel
         if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 577e08c93b3..88d1f11d2da 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -386,7 +386,7 @@ jobs:
           cuda-version: ${{ env.CUDA_VER }}
 
       - name: Download Windows wheel artifacts
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-windows-wheels
           path: ./wheels/
@@ -485,19 +485,19 @@ jobs:
           pip install coverage[toml] Cython
 
       - name: Download Linux coverage data
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-data-linux
           path: ./
 
       - name: Download cuda source code
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-source-linux
           path: ./cuda/
 
       - name: Download Windows coverage data
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-data-windows
           path: ./
@@ -583,7 +583,7 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
 
       - name: Download combined coverage results
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-combined
           path: docs/coverage/
diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml
index c5266302400..001f54288a8 100644
--- a/.github/workflows/test-wheel-linux.yml
+++ b/.github/workflows/test-wheel-linux.yml
@@ -185,7 +185,7 @@ jobs:
 
       - name: Download cuda-pathfinder build artifacts
         if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-pathfinder-wheel
           path: ./cuda_pathfinder
@@ -194,7 +194,7 @@ jobs:
 
       - name: Download cuda-python build artifacts
         if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-python-wheel
           path: .
@@ -204,21 +204,22 @@ jobs:
       - name: Download cuda.bindings build artifacts
         if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                 env.BINDINGS_SOURCE == 'main' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
           run-id: ${{ inputs.run-id || github.run_id }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Download cuda-python & cuda.bindings build artifacts from the prior branch
+      - name: Look up prior-branch artifacts run ID
+        id: prior
         if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                 env.BINDINGS_SOURCE == 'backport' }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
-          # gh is needed for artifact fetching.
+          # gh is needed for lookup-run-id, which calls gh api.
           mkdir -p -m 755 /etc/apt/keyrings \
                 && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
                 && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
@@ -238,27 +239,39 @@ jobs:
           fi
           LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \
             "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI")
+          echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}"
+          echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}"
 
-          gh run download \
-            "${LATEST_PRIOR_RUN_ID}" \
-            -p "${OLD_ARTIFACT_PATTERN}" \
-            -R NVIDIA/cuda-python
-          OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
-          test -d "${OLD_ARTIFACT_DIR}"
-          ls -al "${OLD_ARTIFACT_DIR}"
-          mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
-          mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
-          rmdir "${OLD_ARTIFACT_DIR}"
+      - name: Download prior-branch cuda.bindings build artifacts
+        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
+                env.BINDINGS_SOURCE == 'backport' }}
+        uses: $/.github/actions/download-artifact-retry
+        with:
+          pattern: ${{ steps.prior.outputs.artifact-pattern }}
+          merge-multiple: "true"
+          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+          run-id: ${{ steps.prior.outputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
-          if ${{ env.TEST_PYTHON == 'true' }}; then
-            gh run download \
-              "${LATEST_PRIOR_RUN_ID}" \
-              -p cuda-python-wheel \
-              -R NVIDIA/cuda-python
-            ls -al cuda-python-wheel
-            mv cuda-python-wheel/*.whl .
-            rmdir cuda-python-wheel
+      - name: Download prior-branch cuda-python wheel
+        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'backport' }}
+        uses: $/.github/actions/download-artifact-retry
+        with:
+          name: cuda-python-wheel
+          path: .
+          run-id: ${{ steps.prior.outputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Verify prior-branch artifact layout
+        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
+                env.BINDINGS_SOURCE == 'backport' }}
+        run: |
+          bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl)
+          if [[ "${#bindings_wheels[@]}" -ne 1 ]] || [[ ! -f "${bindings_wheels[0]}" ]]; then
+            echo "::error::Expected exactly one cuda_bindings wheel in ${CUDA_BINDINGS_ARTIFACTS_DIR}; found: ${bindings_wheels[*]}"
+            exit 1
           fi
+          ls -al "${CUDA_BINDINGS_ARTIFACTS_DIR}"
 
       - name: Display structure of downloaded cuda-python artifacts
         if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }}
@@ -275,7 +288,7 @@ jobs:
 
       - name: Download cuda.bindings Cython tests
         if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests
           path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}
@@ -290,7 +303,7 @@ jobs:
 
       - name: Download cuda.core build artifacts
         if: ${{ env.TEST_CORE == 'true' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
@@ -305,7 +318,7 @@ jobs:
 
       - name: Download cuda.core Cython tests
         if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests
           path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}
@@ -320,7 +333,7 @@ jobs:
 
       - name: Download cuda.core test binaries
         if: ${{ env.TEST_CORE == 'true' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries
           path: ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}
diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml
index 0b8f2802f3b..54f709226c5 100644
--- a/.github/workflows/test-wheel-windows.yml
+++ b/.github/workflows/test-wheel-windows.yml
@@ -176,7 +176,7 @@ jobs:
 
       - name: Download cuda-pathfinder build artifacts
         if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-pathfinder-wheel
           path: ./cuda_pathfinder
@@ -185,7 +185,7 @@ jobs:
 
       - name: Download cuda-python build artifacts
         if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-python-wheel
           path: .
@@ -195,14 +195,15 @@ jobs:
       - name: Download cuda.bindings build artifacts
         if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                 env.BINDINGS_SOURCE == 'main' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
           run-id: ${{ inputs.run-id || github.run_id }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Download cuda-python & cuda.bindings build artifacts from the prior branch
+      - name: Look up prior-branch artifacts run ID
+        id: prior
         if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                 env.BINDINGS_SOURCE == 'backport' }}
         env:
@@ -220,27 +221,40 @@ jobs:
           fi
           LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \
             "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI")
+          echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}"
+          echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}"
 
-          gh run download \
-            "${LATEST_PRIOR_RUN_ID}" \
-            -p "${OLD_ARTIFACT_PATTERN}" \
-            -R NVIDIA/cuda-python
-          OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
-          test -d "${OLD_ARTIFACT_DIR}"
-          ls -al "${OLD_ARTIFACT_DIR}"
-          mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
-          mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
-          rmdir "${OLD_ARTIFACT_DIR}"
+      - name: Download prior-branch cuda.bindings build artifacts
+        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
+                env.BINDINGS_SOURCE == 'backport' }}
+        uses: $/.github/actions/download-artifact-retry
+        with:
+          pattern: ${{ steps.prior.outputs.artifact-pattern }}
+          merge-multiple: "true"
+          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+          run-id: ${{ steps.prior.outputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
-          if ${{ env.TEST_PYTHON == 'true' }}; then
-            gh run download \
-              "${LATEST_PRIOR_RUN_ID}" \
-              -p cuda-python-wheel \
-              -R NVIDIA/cuda-python
-            ls -al cuda-python-wheel
-            mv cuda-python-wheel/*.whl .
-            rmdir cuda-python-wheel
+      - name: Download prior-branch cuda-python wheel
+        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'backport' }}
+        uses: $/.github/actions/download-artifact-retry
+        with:
+          name: cuda-python-wheel
+          path: .
+          run-id: ${{ steps.prior.outputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Verify prior-branch artifact layout
+        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
+                env.BINDINGS_SOURCE == 'backport' }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl)
+          if [[ "${#bindings_wheels[@]}" -ne 1 ]] || [[ ! -f "${bindings_wheels[0]}" ]]; then
+            echo "::error::Expected exactly one cuda_bindings wheel in ${CUDA_BINDINGS_ARTIFACTS_DIR}; found: ${bindings_wheels[*]}"
+            exit 1
           fi
+          ls -al "${CUDA_BINDINGS_ARTIFACTS_DIR}"
 
       - name: Display structure of downloaded cuda-python artifacts
         if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }}
@@ -257,7 +271,7 @@ jobs:
 
       - name: Download cuda.bindings Cython tests
         if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests
           path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}
@@ -272,7 +286,7 @@ jobs:
 
       - name: Download cuda.core build artifacts
         if: ${{ env.TEST_CORE == 'true' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
@@ -287,7 +301,7 @@ jobs:
 
       - name: Download cuda.core Cython tests
         if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests
           path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}
@@ -302,7 +316,7 @@ jobs:
 
       - name: Download cuda.core test binaries
         if: ${{ env.TEST_CORE == 'true' }}
-        uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f  # introduced in #3037
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries
           path: ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}
diff --git a/ci/tools/download-wheels b/ci/tools/download-wheels
index 58e8a213226..73094bba744 100755
--- a/ci/tools/download-wheels
+++ b/ci/tools/download-wheels
@@ -9,6 +9,25 @@
 
 set -euo pipefail
 
+# Retry gh run download with exponential backoff to absorb transient
+# artifact-storage failures such as "HTTP 503: Egress is over the account limit".
+_retry_gh_run_download() {
+    local max_attempts=5
+    local delay=15
+    local max_delay=120
+    local attempt=1
+    until gh run download "$@"; do
+        if (( attempt >= max_attempts )); then
+            echo "::error::gh run download failed after ${attempt} attempt(s): $*" >&2
+            return 1
+        fi
+        echo "::warning::gh run download failed (attempt ${attempt}/${max_attempts}); retrying in ${delay}s" >&2
+        sleep "${delay}"
+        attempt=$(( attempt + 1 ))
+        (( delay = delay * 2 > max_delay ? max_delay : delay * 2 ))
+    done
+}
+
 # Check required arguments
 if [[ $# -lt 3 ]]; then
     echo "Usage: $0 <run-id> <component> <repository> [output-dir]" >&2
@@ -35,9 +54,9 @@ echo "Downloading wheels for component: $COMPONENT from run: $RUN_ID"
 # Download component wheels using the same logic as release.yml
 if [[ "$COMPONENT" == "all" ]]; then
     # Download all component patterns
-    gh run download "$RUN_ID" -p "cuda-*" -R "$REPOSITORY"
+    _retry_gh_run_download "$RUN_ID" -p "cuda-*" -R "$REPOSITORY"
 else
-    gh run download "$RUN_ID" -p "${COMPONENT}*" -R "$REPOSITORY"
+    _retry_gh_run_download "$RUN_ID" -p "${COMPONENT}*" -R "$REPOSITORY"
 fi
 
 # Create output directory

From 405f392e69a7e31f78a4212849b3fd5a38d68ffd Mon Sep 17 00:00:00 2001
From: Ralf Juengling <rjuengling@nvidia.com>
Date: Fri, 30 Oct 2026 11:58:51 -0700
Subject: [PATCH 2/3] ci: fix partial-download blocking retries in
 download-wheels

When gh run download fails mid-run (e.g. HTTP 503 on the second
artifact), it leaves behind the successfully-downloaded artifact
directories from that attempt.  On the next attempt, gh refuses to
re-download those artifacts because the directories already exist,
making the transient failure unrecoverable across all remaining
attempts.

Fix: download into a private mktemp staging directory for each attempt.
On failure the staging directory is removed entirely before the next
attempt.  On success the artifact directories are moved from staging
into the caller's CWD, preserving the existing script interface.

Reported by jpascucci-nv in PR review.
---
 ci/tools/download-wheels | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/ci/tools/download-wheels b/ci/tools/download-wheels
index 73094bba744..09cf293070d 100755
--- a/ci/tools/download-wheels
+++ b/ci/tools/download-wheels
@@ -11,12 +11,21 @@ set -euo pipefail
 
 # Retry gh run download with exponential backoff to absorb transient
 # artifact-storage failures such as "HTTP 503: Egress is over the account limit".
+#
+# Downloads into a private staging directory and promotes the artifacts to CWD
+# on success.  The staging directory is wiped before each retry so that
+# artifact directories left behind by a partial download do not cause gh to
+# refuse to re-download them on subsequent attempts.
 _retry_gh_run_download() {
     local max_attempts=5
     local delay=15
     local max_delay=120
     local attempt=1
-    until gh run download "$@"; do
+    local staging
+    staging=$(mktemp -d)
+
+    until gh run download "$@" -D "${staging}"; do
+        rm -rf "${staging}"
         if (( attempt >= max_attempts )); then
             echo "::error::gh run download failed after ${attempt} attempt(s): $*" >&2
             return 1
@@ -25,7 +34,14 @@ _retry_gh_run_download() {
         sleep "${delay}"
         attempt=$(( attempt + 1 ))
         (( delay = delay * 2 > max_delay ? max_delay : delay * 2 ))
+        staging=$(mktemp -d)
+    done
+
+    # Promote each downloaded artifact directory into the caller's CWD.
+    for artifact_dir in "${staging}"/*/; do
+        mv "${artifact_dir%/}" .
     done
+    rm -rf "${staging}"
 }
 
 # Check required arguments

From 09f3faf6b8d54fe9111c96934f9873125bc2555c Mon Sep 17 00:00:00 2001
From: Ralf Juengling <rjuengling@nvidia.com>
Date: Fri, 30 Oct 2026 12:08:23 -0700
Subject: [PATCH 3/3] ci: stage download-wheels retries under RUNNER_TEMP

The runner empties RUNNER_TEMP at the start and end of every job, so a
staging directory left behind by a killed or cancelled step is still
cleaned up.  Falls back to /tmp outside GitHub Actions.
---
 ci/tools/download-wheels | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ci/tools/download-wheels b/ci/tools/download-wheels
index 09cf293070d..709c81d3d12 100755
--- a/ci/tools/download-wheels
+++ b/ci/tools/download-wheels
@@ -15,14 +15,16 @@ set -euo pipefail
 # Downloads into a private staging directory and promotes the artifacts to CWD
 # on success.  The staging directory is wiped before each retry so that
 # artifact directories left behind by a partial download do not cause gh to
-# refuse to re-download them on subsequent attempts.
+# refuse to re-download them on subsequent attempts.  Staging lives under
+# RUNNER_TEMP, which the runner empties at the start and end of every job, so
+# a killed or cancelled step cannot leak it.
 _retry_gh_run_download() {
     local max_attempts=5
     local delay=15
     local max_delay=120
     local attempt=1
     local staging
-    staging=$(mktemp -d)
+    staging=$(mktemp -d -p "${RUNNER_TEMP:-/tmp}")
 
     until gh run download "$@" -D "${staging}"; do
         rm -rf "${staging}"
@@ -34,7 +36,7 @@ _retry_gh_run_download() {
         sleep "${delay}"
         attempt=$(( attempt + 1 ))
         (( delay = delay * 2 > max_delay ? max_delay : delay * 2 ))
-        staging=$(mktemp -d)
+        staging=$(mktemp -d -p "${RUNNER_TEMP:-/tmp}")
     done
 
     # Promote each downloaded artifact directory into the caller's CWD.