ci: route all artifact downloads through download-artifact-retry

.github/actionlint.yaml

@@ -15,6 +15,9 @@ paths:
   ".github/workflows/build-docs.yml":
     ignore:
       - 'unexpected key "queue" for "concurrency" section'
+      # GitHub supports self-repository ($/) action references, but the latest
+      # actionlint release does not yet recognize them (rhysd/actionlint#711).
+      - 'specifying action "\$/[^"]+" in invalid format because ref is missing'
   ".github/workflows/cleanup-pr-previews.yml":
     ignore:
       - 'unexpected key "queue" for "concurrency" section'

.github/actions/download-artifact-retry/action.yml

@@ -5,16 +5,33 @@
 name: Download artifact (with retry)
 
 description: >-
-  Wrapper around actions/download-artifact that retries once after a short
-  delay when the first attempt fails.  Designed to absorb transient
-  infrastructure flakes such as "other side closed" without failing the job.
+  Wrapper around actions/download-artifact that retries up to three times with
+  exponential backoff when a download fails.  Handles both transient TCP errors
+  such as "other side closed" and storage-layer throttling such as
+  "HTTP 503: Egress is over the account limit".  Accepts either an exact
+  artifact name or a glob pattern; supply exactly one.
 
 inputs:
   name:
-    description: "Name of the artifact to download"
-    required: true
+    description: >-
+      Exact name of the artifact to download.  Mutually exclusive with
+      'pattern'; supply exactly one of the two.
+    required: false
+    default: ""
+  pattern:
+    description: >-
+      Glob pattern matching one or more artifact names.  Mutually exclusive
+      with 'name'; supply exactly one of the two.
+    required: false
+    default: ""
+  merge-multiple:
+    description: >-
+      When true and 'pattern' is set, merge files from all matched artifacts
+      directly into 'path' instead of placing each in its own subdirectory.
+    required: false
+    default: "false"
   path:
-    description: "Destination path for the downloaded artifact"
+    description: "Destination path for the downloaded artifact(s)"
     required: true
   run-id:
     description: >-
@@ -32,39 +49,80 @@ inputs:
     required: false
     default: ""
   retry-delay-seconds:
-    description: "Seconds to wait between the first failure and the retry"
+    description: "Seconds to wait between the first failure and the second attempt"
     required: false
     default: "15"
 
 runs:
   using: composite
   steps:
+    - name: Validate inputs
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        INPUT_NAME: ${{ inputs.name }}
+        INPUT_PATTERN: ${{ inputs.pattern }}
+      run: |
+        if [[ -n "${INPUT_NAME}" && -n "${INPUT_PATTERN}" ]]; then
+          echo "::error::Specify exactly one of 'name' or 'pattern', not both"
+          exit 1
+        fi
+        if [[ -z "${INPUT_NAME}" && -z "${INPUT_PATTERN}" ]]; then
+          echo "::error::Specify exactly one of 'name' or 'pattern'"
+          exit 1
+        fi
+
     - name: Download artifact (attempt 1)
       id: attempt1
       continue-on-error: true
       uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
       with:
         name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: ${{ inputs.merge-multiple }}
         path: ${{ inputs.path }}
         run-id: ${{ inputs.run-id }}
         github-token: ${{ inputs.github-token }}
 
-    - name: Wait before retry
+    - name: Wait before retry (attempt 2)
       if: ${{ steps.attempt1.outcome == 'failure' }}
       shell: bash --noprofile --norc -euo pipefail {0}
       env:
-        # Pass through env instead of interpolating the input into the shell
-        # script, so a non-numeric value cannot inject commands.
         RETRY_DELAY_SECONDS: ${{ inputs.retry-delay-seconds }}
       run: |
         echo "::warning::Artifact download failed (attempt 1); retrying in ${RETRY_DELAY_SECONDS}s"
         sleep "$RETRY_DELAY_SECONDS"
 
     - name: Download artifact (attempt 2)
+      id: attempt2
       if: ${{ steps.attempt1.outcome == 'failure' }}
+      continue-on-error: true
+      uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+      with:
+        name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: ${{ inputs.merge-multiple }}
+        path: ${{ inputs.path }}
+        run-id: ${{ inputs.run-id }}
+        github-token: ${{ inputs.github-token }}
+
+    - name: Wait before retry (attempt 3)
+      if: ${{ steps.attempt1.outcome == 'failure' && steps.attempt2.outcome == 'failure' }}
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        RETRY_DELAY_SECONDS: ${{ inputs.retry-delay-seconds }}
+      run: |
+        # Double the delay for the second retry to give egress throttling more time to clear.
+        RETRY_DELAY_SECONDS=$(( RETRY_DELAY_SECONDS * 2 ))
+        echo "::warning::Artifact download failed (attempt 2); retrying in ${RETRY_DELAY_SECONDS}s"
+        sleep "$RETRY_DELAY_SECONDS"
+
+    - name: Download artifact (attempt 3)
+      if: ${{ steps.attempt1.outcome == 'failure' && steps.attempt2.outcome == 'failure' }}
       uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
       with:
         name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: ${{ inputs.merge-multiple }}
         path: ${{ inputs.path }}
         run-id: ${{ inputs.run-id }}
         github-token: ${{ inputs.github-token }}

.github/workflows/build-docs.yml

@@ -144,7 +144,7 @@ jobs:
           echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/$CUDA_BINDINGS_SOURCE_DIR/dist")" >> $GITHUB_ENV
 
       - name: Download cuda-python build artifacts
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-python-wheel
           path: .
@@ -157,7 +157,7 @@ jobs:
           ls -lahR .
 
       - name: Download cuda-pathfinder build artifacts
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-pathfinder-wheel
           path: ./cuda_pathfinder
@@ -171,17 +171,17 @@ jobs:
 
       - name: Download cuda.bindings build artifacts
         if: ${{ !inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
 
       - name: Download cuda.bindings build artifacts
         if: ${{ inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           pattern: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
-          merge-multiple: true
+          merge-multiple: "true"
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
           run-id: ${{ inputs.run-id }}
           github-token: ${{ github.token }}
@@ -193,17 +193,17 @@ jobs:
 
       - name: Download cuda.core build artifacts
         if: ${{ !inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
 
       - name: Download cuda.core build artifacts
         if: ${{ inputs.is-release }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           pattern: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
-          merge-multiple: true
+          merge-multiple: "true"
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
           run-id: ${{ inputs.run-id }}
           github-token: ${{ github.token }}
@@ -366,7 +366,7 @@ jobs:
           ref: ${{ inputs.git-tag }}
 
       - name: Download docs deployment payload
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ needs.build.outputs.deployment-artifact-name }}
           path: artifacts/docs/

.github/workflows/build-wheel.yml

@@ -170,7 +170,7 @@ jobs:
 
       - name: Download reusable cuda.pathfinder wheel
         if: ${{ env.BUILD_PATHFINDER != 'true' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-pathfinder-wheel
           path: cuda_pathfinder
@@ -271,7 +271,7 @@ jobs:
 
       - name: Download reusable cuda.bindings wheel
         if: ${{ env.BUILD_BINDINGS != 'true' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
@@ -395,7 +395,7 @@ jobs:
 
       - name: Download reusable cuda.core wheel
         if: ${{ env.BUILD_CORE != 'true' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
@@ -413,7 +413,7 @@ jobs:
 
       - name: Download reusable cuda-python wheel
         if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-python-wheel
           path: cuda_python
@@ -542,14 +542,15 @@ jobs:
             ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.lib
           if-no-files-found: error
 
-      - name: Download cuda.bindings build artifacts from the prior branch
+      - name: Look up prior-branch cuda.bindings run ID
+        id: prior-bindings
         if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           if ! (command -v gh 2>&1 >/dev/null); then
             # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
-            # gh is needed for artifact fetching.
+            # gh is needed for lookup-run-id, which calls gh api.
             mkdir -p -m 755 /etc/apt/keyrings \
                   && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
                   && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
@@ -565,18 +566,18 @@ jobs:
             --branch "${OLD_BRANCH}" \
             --artifact "${OLD_ARTIFACT_PATTERN}" \
             NVIDIA/cuda-python "CI")
-          PREV_BINDINGS_DIR="cuda_bindings_13/dist-prev"
-
-          gh run download \
-            "${LATEST_PRIOR_RUN_ID}" \
-            -p "${OLD_ARTIFACT_PATTERN}" \
-            -R NVIDIA/cuda-python
-          OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
-          test -d "${OLD_ARTIFACT_DIR}"
-          ls -al "${OLD_ARTIFACT_DIR}"
-          mkdir -p "${PREV_BINDINGS_DIR}"
-          mv "${OLD_ARTIFACT_DIR}"/*.whl "${PREV_BINDINGS_DIR}"
-          rmdir "${OLD_ARTIFACT_DIR}"
+          echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}"
+          echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}"
+
+      - name: Download prior-branch cuda.bindings build artifacts
+        if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}
+        uses: $/.github/actions/download-artifact-retry
+        with:
+          pattern: ${{ steps.prior-bindings.outputs.artifact-pattern }}
+          merge-multiple: "true"
+          path: cuda_bindings_13/dist-prev
+          run-id: ${{ steps.prior-bindings.outputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel
         if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}

.github/workflows/coverage.yml

@@ -386,7 +386,7 @@ jobs:
           cuda-version: ${{ env.CUDA_VER }}
 
       - name: Download Windows wheel artifacts
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-windows-wheels
           path: ./wheels/
@@ -485,19 +485,19 @@ jobs:
           pip install coverage[toml] Cython
 
       - name: Download Linux coverage data
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-data-linux
           path: ./
 
       - name: Download cuda source code
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: cuda-source-linux
           path: ./cuda/
 
       - name: Download Windows coverage data
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-data-windows
           path: ./
@@ -583,7 +583,7 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
 
       - name: Download combined coverage results
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        uses: $/.github/actions/download-artifact-retry
         with:
           name: coverage-combined
           path: docs/coverage/