diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 57724681f83..25cc1e40959 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -15,6 +15,9 @@ paths: ".github/workflows/build-docs.yml": ignore: - 'unexpected key "queue" for "concurrency" section' + # GitHub supports self-repository ($/) action references, but the latest + # actionlint release does not yet recognize them (rhysd/actionlint#711). + - 'specifying action "\$/[^"]+" in invalid format because ref is missing' ".github/workflows/cleanup-pr-previews.yml": ignore: - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/actions/download-artifact-retry/action.yml b/.github/actions/download-artifact-retry/action.yml index df7be01c8c4..335257d2890 100644 --- a/.github/actions/download-artifact-retry/action.yml +++ b/.github/actions/download-artifact-retry/action.yml @@ -5,16 +5,33 @@ name: Download artifact (with retry) description: >- - Wrapper around actions/download-artifact that retries once after a short - delay when the first attempt fails. Designed to absorb transient - infrastructure flakes such as "other side closed" without failing the job. + Wrapper around actions/download-artifact that retries up to three times with + exponential backoff when a download fails. Handles both transient TCP errors + such as "other side closed" and storage-layer throttling such as + "HTTP 503: Egress is over the account limit". Accepts either an exact + artifact name or a glob pattern; supply exactly one. inputs: name: - description: "Name of the artifact to download" - required: true + description: >- + Exact name of the artifact to download. Mutually exclusive with + 'pattern'; supply exactly one of the two. + required: false + default: "" + pattern: + description: >- + Glob pattern matching one or more artifact names. Mutually exclusive + with 'name'; supply exactly one of the two. + required: false + default: "" + merge-multiple: + description: >- + When true and 'pattern' is set, merge files from all matched artifacts + directly into 'path' instead of placing each in its own subdirectory. + required: false + default: "false" path: - description: "Destination path for the downloaded artifact" + description: "Destination path for the downloaded artifact(s)" required: true run-id: description: >- @@ -32,39 +49,80 @@ inputs: required: false default: "" retry-delay-seconds: - description: "Seconds to wait between the first failure and the retry" + description: "Seconds to wait between the first failure and the second attempt" required: false default: "15" runs: using: composite steps: + - name: Validate inputs + shell: bash --noprofile --norc -euo pipefail {0} + env: + INPUT_NAME: ${{ inputs.name }} + INPUT_PATTERN: ${{ inputs.pattern }} + run: | + if [[ -n "${INPUT_NAME}" && -n "${INPUT_PATTERN}" ]]; then + echo "::error::Specify exactly one of 'name' or 'pattern', not both" + exit 1 + fi + if [[ -z "${INPUT_NAME}" && -z "${INPUT_PATTERN}" ]]; then + echo "::error::Specify exactly one of 'name' or 'pattern'" + exit 1 + fi + - name: Download artifact (attempt 1) id: attempt1 continue-on-error: true uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: ${{ inputs.name }} + pattern: ${{ inputs.pattern }} + merge-multiple: ${{ inputs.merge-multiple }} path: ${{ inputs.path }} run-id: ${{ inputs.run-id }} github-token: ${{ inputs.github-token }} - - name: Wait before retry + - name: Wait before retry (attempt 2) if: ${{ steps.attempt1.outcome == 'failure' }} shell: bash --noprofile --norc -euo pipefail {0} env: - # Pass through env instead of interpolating the input into the shell - # script, so a non-numeric value cannot inject commands. RETRY_DELAY_SECONDS: ${{ inputs.retry-delay-seconds }} run: | echo "::warning::Artifact download failed (attempt 1); retrying in ${RETRY_DELAY_SECONDS}s" sleep "$RETRY_DELAY_SECONDS" - name: Download artifact (attempt 2) + id: attempt2 if: ${{ steps.attempt1.outcome == 'failure' }} + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ inputs.name }} + pattern: ${{ inputs.pattern }} + merge-multiple: ${{ inputs.merge-multiple }} + path: ${{ inputs.path }} + run-id: ${{ inputs.run-id }} + github-token: ${{ inputs.github-token }} + + - name: Wait before retry (attempt 3) + if: ${{ steps.attempt1.outcome == 'failure' && steps.attempt2.outcome == 'failure' }} + shell: bash --noprofile --norc -euo pipefail {0} + env: + RETRY_DELAY_SECONDS: ${{ inputs.retry-delay-seconds }} + run: | + # Double the delay for the second retry to give egress throttling more time to clear. + RETRY_DELAY_SECONDS=$(( RETRY_DELAY_SECONDS * 2 )) + echo "::warning::Artifact download failed (attempt 2); retrying in ${RETRY_DELAY_SECONDS}s" + sleep "$RETRY_DELAY_SECONDS" + + - name: Download artifact (attempt 3) + if: ${{ steps.attempt1.outcome == 'failure' && steps.attempt2.outcome == 'failure' }} uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: ${{ inputs.name }} + pattern: ${{ inputs.pattern }} + merge-multiple: ${{ inputs.merge-multiple }} path: ${{ inputs.path }} run-id: ${{ inputs.run-id }} github-token: ${{ inputs.github-token }} diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index b1c029f62cf..b734fdd8c73 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -144,7 +144,7 @@ jobs: echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/$CUDA_BINDINGS_SOURCE_DIR/dist")" >> $GITHUB_ENV - name: Download cuda-python build artifacts - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: cuda-python-wheel path: . @@ -157,7 +157,7 @@ jobs: ls -lahR . - name: Download cuda-pathfinder build artifacts - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: cuda-pathfinder-wheel path: ./cuda_pathfinder @@ -171,17 +171,17 @@ jobs: - name: Download cuda.bindings build artifacts if: ${{ !inputs.is-release }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - name: Download cuda.bindings build artifacts if: ${{ inputs.is-release }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: pattern: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} - merge-multiple: true + merge-multiple: "true" path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -193,17 +193,17 @@ jobs: - name: Download cuda.core build artifacts if: ${{ !inputs.is-release }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Download cuda.core build artifacts if: ${{ inputs.is-release }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: pattern: ${{ env.CUDA_CORE_ARTIFACT_NAME }} - merge-multiple: true + merge-multiple: "true" path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -366,7 +366,7 @@ jobs: ref: ${{ inputs.git-tag }} - name: Download docs deployment payload - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: ${{ needs.build.outputs.deployment-artifact-name }} path: artifacts/docs/ diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 9b2be767828..03dd258fa19 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -170,7 +170,7 @@ jobs: - name: Download reusable cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: cuda-pathfinder-wheel path: cuda_pathfinder @@ -271,7 +271,7 @@ jobs: - name: Download reusable cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} @@ -395,7 +395,7 @@ jobs: - name: Download reusable cuda.core wheel if: ${{ env.BUILD_CORE != 'true' }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} @@ -413,7 +413,7 @@ jobs: - name: Download reusable cuda-python wheel if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: cuda-python-wheel path: cuda_python @@ -542,14 +542,15 @@ jobs: ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.lib if-no-files-found: error - - name: Download cuda.bindings build artifacts from the prior branch + - name: Look up prior-branch cuda.bindings run ID + id: prior-bindings if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if ! (command -v gh 2>&1 >/dev/null); then # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt. - # gh is needed for artifact fetching. + # gh is needed for lookup-run-id, which calls gh api. mkdir -p -m 755 /etc/apt/keyrings \ && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ @@ -565,18 +566,18 @@ jobs: --branch "${OLD_BRANCH}" \ --artifact "${OLD_ARTIFACT_PATTERN}" \ NVIDIA/cuda-python "CI") - PREV_BINDINGS_DIR="cuda_bindings_13/dist-prev" - - gh run download \ - "${LATEST_PRIOR_RUN_ID}" \ - -p "${OLD_ARTIFACT_PATTERN}" \ - -R NVIDIA/cuda-python - OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") - test -d "${OLD_ARTIFACT_DIR}" - ls -al "${OLD_ARTIFACT_DIR}" - mkdir -p "${PREV_BINDINGS_DIR}" - mv "${OLD_ARTIFACT_DIR}"/*.whl "${PREV_BINDINGS_DIR}" - rmdir "${OLD_ARTIFACT_DIR}" + echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}" + echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}" + + - name: Download prior-branch cuda.bindings build artifacts + if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} + uses: $/.github/actions/download-artifact-retry + with: + pattern: ${{ steps.prior-bindings.outputs.artifact-pattern }} + merge-multiple: "true" + path: cuda_bindings_13/dist-prev + run-id: ${{ steps.prior-bindings.outputs.run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 577e08c93b3..88d1f11d2da 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -386,7 +386,7 @@ jobs: cuda-version: ${{ env.CUDA_VER }} - name: Download Windows wheel artifacts - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: coverage-windows-wheels path: ./wheels/ @@ -485,19 +485,19 @@ jobs: pip install coverage[toml] Cython - name: Download Linux coverage data - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: coverage-data-linux path: ./ - name: Download cuda source code - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: cuda-source-linux path: ./cuda/ - name: Download Windows coverage data - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: coverage-data-windows path: ./ @@ -583,7 +583,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download combined coverage results - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + uses: $/.github/actions/download-artifact-retry with: name: coverage-combined path: docs/coverage/ diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index c5266302400..001f54288a8 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -185,7 +185,7 @@ jobs: - name: Download cuda-pathfinder build artifacts if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: cuda-pathfinder-wheel path: ./cuda_pathfinder @@ -194,7 +194,7 @@ jobs: - name: Download cuda-python build artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: cuda-python-wheel path: . @@ -204,21 +204,22 @@ jobs: - name: Download cuda.bindings build artifacts if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} run-id: ${{ inputs.run-id || github.run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Download cuda-python & cuda.bindings build artifacts from the prior branch + - name: Look up prior-branch artifacts run ID + id: prior if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt. - # gh is needed for artifact fetching. + # gh is needed for lookup-run-id, which calls gh api. mkdir -p -m 755 /etc/apt/keyrings \ && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ @@ -238,27 +239,39 @@ jobs: fi LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI") + echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}" + echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}" - gh run download \ - "${LATEST_PRIOR_RUN_ID}" \ - -p "${OLD_ARTIFACT_PATTERN}" \ - -R NVIDIA/cuda-python - OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") - test -d "${OLD_ARTIFACT_DIR}" - ls -al "${OLD_ARTIFACT_DIR}" - mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" - mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ - rmdir "${OLD_ARTIFACT_DIR}" + - name: Download prior-branch cuda.bindings build artifacts + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && + env.BINDINGS_SOURCE == 'backport' }} + uses: $/.github/actions/download-artifact-retry + with: + pattern: ${{ steps.prior.outputs.artifact-pattern }} + merge-multiple: "true" + path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + run-id: ${{ steps.prior.outputs.run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - if ${{ env.TEST_PYTHON == 'true' }}; then - gh run download \ - "${LATEST_PRIOR_RUN_ID}" \ - -p cuda-python-wheel \ - -R NVIDIA/cuda-python - ls -al cuda-python-wheel - mv cuda-python-wheel/*.whl . - rmdir cuda-python-wheel + - name: Download prior-branch cuda-python wheel + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'backport' }} + uses: $/.github/actions/download-artifact-retry + with: + name: cuda-python-wheel + path: . + run-id: ${{ steps.prior.outputs.run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify prior-branch artifact layout + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && + env.BINDINGS_SOURCE == 'backport' }} + run: | + bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl) + if [[ "${#bindings_wheels[@]}" -ne 1 ]] || [[ ! -f "${bindings_wheels[0]}" ]]; then + echo "::error::Expected exactly one cuda_bindings wheel in ${CUDA_BINDINGS_ARTIFACTS_DIR}; found: ${bindings_wheels[*]}" + exit 1 fi + ls -al "${CUDA_BINDINGS_ARTIFACTS_DIR}" - name: Display structure of downloaded cuda-python artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }} @@ -275,7 +288,7 @@ jobs: - name: Download cuda.bindings Cython tests if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} @@ -290,7 +303,7 @@ jobs: - name: Download cuda.core build artifacts if: ${{ env.TEST_CORE == 'true' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} @@ -305,7 +318,7 @@ jobs: - name: Download cuda.core Cython tests if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} @@ -320,7 +333,7 @@ jobs: - name: Download cuda.core test binaries if: ${{ env.TEST_CORE == 'true' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries path: ${{ env.CUDA_CORE_TEST_BINARIES_DIR }} diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 0b8f2802f3b..54f709226c5 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -176,7 +176,7 @@ jobs: - name: Download cuda-pathfinder build artifacts if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: cuda-pathfinder-wheel path: ./cuda_pathfinder @@ -185,7 +185,7 @@ jobs: - name: Download cuda-python build artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: cuda-python-wheel path: . @@ -195,14 +195,15 @@ jobs: - name: Download cuda.bindings build artifacts if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} run-id: ${{ inputs.run-id || github.run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Download cuda-python & cuda.bindings build artifacts from the prior branch + - name: Look up prior-branch artifacts run ID + id: prior if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: @@ -220,27 +221,40 @@ jobs: fi LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI") + echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}" + echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}" - gh run download \ - "${LATEST_PRIOR_RUN_ID}" \ - -p "${OLD_ARTIFACT_PATTERN}" \ - -R NVIDIA/cuda-python - OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") - test -d "${OLD_ARTIFACT_DIR}" - ls -al "${OLD_ARTIFACT_DIR}" - mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" - mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ - rmdir "${OLD_ARTIFACT_DIR}" + - name: Download prior-branch cuda.bindings build artifacts + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && + env.BINDINGS_SOURCE == 'backport' }} + uses: $/.github/actions/download-artifact-retry + with: + pattern: ${{ steps.prior.outputs.artifact-pattern }} + merge-multiple: "true" + path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + run-id: ${{ steps.prior.outputs.run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - if ${{ env.TEST_PYTHON == 'true' }}; then - gh run download \ - "${LATEST_PRIOR_RUN_ID}" \ - -p cuda-python-wheel \ - -R NVIDIA/cuda-python - ls -al cuda-python-wheel - mv cuda-python-wheel/*.whl . - rmdir cuda-python-wheel + - name: Download prior-branch cuda-python wheel + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'backport' }} + uses: $/.github/actions/download-artifact-retry + with: + name: cuda-python-wheel + path: . + run-id: ${{ steps.prior.outputs.run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify prior-branch artifact layout + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && + env.BINDINGS_SOURCE == 'backport' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl) + if [[ "${#bindings_wheels[@]}" -ne 1 ]] || [[ ! -f "${bindings_wheels[0]}" ]]; then + echo "::error::Expected exactly one cuda_bindings wheel in ${CUDA_BINDINGS_ARTIFACTS_DIR}; found: ${bindings_wheels[*]}" + exit 1 fi + ls -al "${CUDA_BINDINGS_ARTIFACTS_DIR}" - name: Display structure of downloaded cuda-python artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }} @@ -257,7 +271,7 @@ jobs: - name: Download cuda.bindings Cython tests if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} @@ -272,7 +286,7 @@ jobs: - name: Download cuda.core build artifacts if: ${{ env.TEST_CORE == 'true' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} @@ -287,7 +301,7 @@ jobs: - name: Download cuda.core Cython tests if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} @@ -302,7 +316,7 @@ jobs: - name: Download cuda.core test binaries if: ${{ env.TEST_CORE == 'true' }} - uses: NVIDIA/cuda-python/.github/actions/download-artifact-retry@f20d661666a5c92b757748ca237d3592ac3e942f # introduced in #3037 + uses: $/.github/actions/download-artifact-retry with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries path: ${{ env.CUDA_CORE_TEST_BINARIES_DIR }} diff --git a/ci/tools/download-wheels b/ci/tools/download-wheels index 58e8a213226..709c81d3d12 100755 --- a/ci/tools/download-wheels +++ b/ci/tools/download-wheels @@ -9,6 +9,43 @@ set -euo pipefail +# Retry gh run download with exponential backoff to absorb transient +# artifact-storage failures such as "HTTP 503: Egress is over the account limit". +# +# Downloads into a private staging directory and promotes the artifacts to CWD +# on success. The staging directory is wiped before each retry so that +# artifact directories left behind by a partial download do not cause gh to +# refuse to re-download them on subsequent attempts. Staging lives under +# RUNNER_TEMP, which the runner empties at the start and end of every job, so +# a killed or cancelled step cannot leak it. +_retry_gh_run_download() { + local max_attempts=5 + local delay=15 + local max_delay=120 + local attempt=1 + local staging + staging=$(mktemp -d -p "${RUNNER_TEMP:-/tmp}") + + until gh run download "$@" -D "${staging}"; do + rm -rf "${staging}" + if (( attempt >= max_attempts )); then + echo "::error::gh run download failed after ${attempt} attempt(s): $*" >&2 + return 1 + fi + echo "::warning::gh run download failed (attempt ${attempt}/${max_attempts}); retrying in ${delay}s" >&2 + sleep "${delay}" + attempt=$(( attempt + 1 )) + (( delay = delay * 2 > max_delay ? max_delay : delay * 2 )) + staging=$(mktemp -d -p "${RUNNER_TEMP:-/tmp}") + done + + # Promote each downloaded artifact directory into the caller's CWD. + for artifact_dir in "${staging}"/*/; do + mv "${artifact_dir%/}" . + done + rm -rf "${staging}" +} + # Check required arguments if [[ $# -lt 3 ]]; then echo "Usage: $0 [output-dir]" >&2 @@ -35,9 +72,9 @@ echo "Downloading wheels for component: $COMPONENT from run: $RUN_ID" # Download component wheels using the same logic as release.yml if [[ "$COMPONENT" == "all" ]]; then # Download all component patterns - gh run download "$RUN_ID" -p "cuda-*" -R "$REPOSITORY" + _retry_gh_run_download "$RUN_ID" -p "cuda-*" -R "$REPOSITORY" else - gh run download "$RUN_ID" -p "${COMPONENT}*" -R "$REPOSITORY" + _retry_gh_run_download "$RUN_ID" -p "${COMPONENT}*" -R "$REPOSITORY" fi # Create output directory