From 636fefac7dd9b9ecf52ca1f561aee95bacf5925f Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 26 Oct 2026 17:06:12 +0100 Subject: [PATCH 1/5] Distinguish release monitoring errors from workflow failures --- .../scripts/tests/test_release_automation.sh | 96 +++++++++++++++++++ utils/README.md | 13 ++- utils/release.sh | 56 ++++++++--- 3 files changed, 150 insertions(+), 15 deletions(-) diff --git a/.github/scripts/tests/test_release_automation.sh b/.github/scripts/tests/test_release_automation.sh index 49b8052a9e..ced20951a2 100755 --- a/.github/scripts/tests/test_release_automation.sh +++ b/.github/scripts/tests/test_release_automation.sh @@ -337,4 +337,100 @@ if git --git-dir="$REMOTE" show-ref --verify --quiet "refs/tags/v_1.49.0" 2>/dev fi pass +# --- Release CLI monitoring tests (GitHub CLI fixture, no network) ------------ + +REMOTE=$(setup_remote) +WORK=$(setup_work "$REMOTE" main) +CLI_SHA=$(git -C "$WORK" rev-parse HEAD) +CLI_BIN="$TEMP_DIR/cli-bin" +CLI_STATE="$TEMP_DIR/cli-state" +mkdir "$CLI_BIN" "$CLI_STATE" +cat > "$CLI_BIN/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "$CLI_STATE/calls" +case "$1 $2" in + 'auth status') echo 'Logged in to github.com account release-test' ;; + 'repo view') + case "$4" in + nameWithOwner) echo 'DataDog/java-profiler' ;; + url) echo 'https://github.com/DataDog/java-profiler' ;; + viewerPermission) echo WRITE ;; + *) exit 97 ;; + esac + ;; + 'api user') echo release-test ;; + 'api repos/DataDog/java-profiler/actions/runs?event=workflow_dispatch&per_page=50') + printf '%s\n' "$CLI_RUN_ID" + ;; + 'workflow run') exit "$CLI_DISPATCH_EXIT" ;; + 'run watch') + count=0 + [ ! -f "$CLI_STATE/watches" ] || count=$(cat "$CLI_STATE/watches") + printf '%s\n' "$((count + 1))" > "$CLI_STATE/watches" + if [ "$count" -eq 0 ]; then + exit "$CLI_WATCH_EXIT" + fi + exit "$CLI_NEXT_WATCH_EXIT" + ;; + 'run view') + [ "$CLI_QUERY_EXIT" -eq 0 ] || exit "$CLI_QUERY_EXIT" + printf '%s\n' "$CLI_REMOTE_JSON" | jq -r "${@: -1}" + ;; + *) echo "Unexpected fixture command: $*" >&2; exit 97 ;; +esac +EOF +printf '#!/usr/bin/env bash\nexit 0\n' > "$CLI_BIN/sleep" +chmod +x "$CLI_BIN/gh" "$CLI_BIN/sleep" + +expect_monitor() { + local name=$1 expected_exit=$2 expected_status=$3 watches=$4 views=$5 + shift 5 + local actual_exit=0 + rm -f "$CLI_STATE/watches" + : > "$CLI_STATE/calls" + CLI_OUTPUT=$( + cd "$WORK" + env PATH="$CLI_BIN:$PATH" CLI_STATE="$CLI_STATE" \ + CLI_RUN_ID=4242 CLI_DISPATCH_EXIT=0 CLI_WATCH_EXIT=0 CLI_NEXT_WATCH_EXIT=1 \ + CLI_QUERY_EXIT=0 CLI_REMOTE_JSON='{"status":"completed","conclusion":"success"}' \ + "$@" bash "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1 + ) || actual_exit=$? + [ "$actual_exit" -eq "$expected_exit" ] || + fail "$name: expected exit $expected_exit, got $actual_exit" + [[ "$CLI_OUTPUT" == *"Status: $expected_status"* ]] || + fail "$name: expected summary '$expected_status'" + [ "$(awk '/^workflow run / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq 1 ] || + fail "$name: workflow must be dispatched exactly once" + [ "$(awk '/^run watch / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq "$watches" ] || + fail "$name: unexpected watch count" + [ "$(awk '/^run view / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq "$views" ] || + fail "$name: unexpected status query count" + pass +} + +expect_monitor 'watch succeeds' 0 'WORKFLOW SUCCEEDED' 1 0 +expect_monitor 'connection lost after success' 0 'WORKFLOW SUCCEEDED' 1 1 CLI_WATCH_EXIT=1 +expect_monitor 'confirmed workflow failure' 1 'WORKFLOW FAILED' 1 1 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"failure"}' +expect_monitor 'confirmed cancellation' 1 'WORKFLOW FAILED' 1 1 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"cancelled"}' +expect_monitor 'confirmed timeout' 1 'WORKFLOW FAILED' 1 1 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"timed_out"}' +expect_monitor 'monitor reconnects to active run' 0 'WORKFLOW SUCCEEDED' 2 1 \ + CLI_WATCH_EXIT=1 CLI_NEXT_WATCH_EXIT=0 \ + CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +expect_monitor 'monitor cannot reconnect to active run' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +expect_monitor 'GitHub status unavailable' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \ + CLI_WATCH_EXIT=1 CLI_QUERY_EXIT=1 +[[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] || + fail 'unavailable GitHub status was reported as workflow failure' +expect_monitor 'completed run has no conclusion' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":null}' +expect_monitor 'run discovery exhausted' 1 'WORKFLOW STATUS UNKNOWN' 0 0 CLI_RUN_ID= +[[ "$CLI_OUTPUT" != *'Status: DRY-RUN COMPLETED'* ]] || + fail 'unknown dry-run status was reported as completed' +expect_monitor 'dispatch fails' 1 'FAILED TO TRIGGER WORKFLOW' 0 0 CLI_DISPATCH_EXIT=1 + echo "PASS: $TESTS hermetic release-automation test groups" diff --git a/utils/README.md b/utils/README.md index 6e1f6ad077..ce6fb48616 100644 --- a/utils/README.md +++ b/utils/README.md @@ -63,12 +63,19 @@ can be cancelled at any time with `q` or Ctrl-C. A dry run never creates a tag, pushes a branch, or triggers GitLab. +If workflow monitoring loses its API connection, the script checks the run's +status and reconnects to the same run, making up to three monitoring attempts +in total. If it cannot confirm the result, it reports `WORKFLOW STATUS UNKNOWN` +and exits nonzero. +Check the existing run before starting another release. + ### Testing release automation `.github/scripts/tests/test_release_automation.sh` is a single hermetic shell -test. It validates release tag/branch creation, version computation, and -trivial-PR authorization using temporary local git fixtures. It does not -load credentials or invoke `gh`, so it cannot publish, tag, push, or merge +test. It validates release tag/branch creation, version computation, +trivial-PR authorization, and recovery from workflow monitoring errors using +temporary local Git repositories and a GitHub CLI fixture. It does not load +credentials or contact GitHub, so it cannot publish, tag, push, or merge anything remotely. ```bash diff --git a/utils/release.sh b/utils/release.sh index ac5e1597a2..3bd64dd42c 100755 --- a/utils/release.sh +++ b/utils/release.sh @@ -698,13 +698,46 @@ if gh workflow run release-validated.yml \ print_info "Watching workflow run ${RUN_ID}..." echo " ${REPO_URL}/actions/runs/${RUN_ID}" echo "" - if gh run watch "$RUN_ID" --exit-status; then - WORKFLOW_CONCLUSION="success" + WORKFLOW_CONCLUSION="unknown" + for WATCH_ATTEMPT in 1 2 3; do + if gh run watch "$RUN_ID" --repo "$REPO" --exit-status; then + WORKFLOW_CONCLUSION="success" + break + fi + + # A failed watch can mean an API connection error. Check the run + # itself before deciding that the remote workflow failed. + print_warning "Workflow monitoring stopped; checking GitHub for the run's status..." + if REMOTE_CONCLUSION=$(gh run view "$RUN_ID" --repo "$REPO" \ + --json status,conclusion \ + --jq 'if .status == "completed" then (.conclusion // "unknown") else "unknown" end'); then + case "$REMOTE_CONCLUSION" in + success) + WORKFLOW_CONCLUSION="success" + break + ;; + failure|cancelled|timed_out|action_required|startup_failure|stale|neutral|skipped) + WORKFLOW_CONCLUSION="failure" + print_info "GitHub reports workflow conclusion: $REMOTE_CONCLUSION" + break + ;; + esac + fi + + if [ "$WATCH_ATTEMPT" -lt 3 ]; then + print_warning "Reconnecting to the same workflow run ($((WATCH_ATTEMPT + 1))/3)..." + sleep 2 + fi + done + + if [ "$WORKFLOW_CONCLUSION" = "success" ]; then print_success "✓ Workflow completed successfully!" - else - WORKFLOW_CONCLUSION="failure" + elif [ "$WORKFLOW_CONCLUSION" = "failure" ]; then print_error "✗ Workflow failed!" - echo " View logs: gh run view $RUN_ID --log-failed" + echo " View logs: gh run view $RUN_ID --repo $REPO --log-failed" + else + print_warning "Monitoring could not confirm the workflow result. The remote run may still be running." + echo " Inspect the existing run: ${REPO_URL}/actions/runs/${RUN_ID}" fi else WORKFLOW_CONCLUSION="unknown" @@ -717,7 +750,7 @@ if gh workflow run release-validated.yml \ print_info "Next Steps:" echo " 1. Verify Maven: https://repo1.maven.org/maven2/com/datadoghq/ddprof/" echo " 2. Check release: ${REPO_URL}/releases" - else + elif [ "${WORKFLOW_CONCLUSION:-unknown}" = "success" ]; then print_info "This was a dry-run. Review the output and run again with --no-dry-run" fi else @@ -767,14 +800,13 @@ if [ "$WORKFLOW_SUCCESS" = true ]; then elif [ "${WORKFLOW_CONCLUSION:-}" = "failure" ]; then print_error "Status: WORKFLOW FAILED" echo " View logs: gh run view $RUN_ID --log-failed" - elif [ "$DRY_RUN" == "true" ]; then - print_success "Status: DRY-RUN COMPLETED" - echo "" - echo " → No actual changes were made." - echo " → To perform the release, run: $0 $RELEASE_TYPE --no-dry-run --commit $SHORT_SHA" else print_warning "Status: WORKFLOW STATUS UNKNOWN" - echo " Check manually: gh run list --workflow=release-validated.yml --limit 1" + if [ -n "$RUN_ID" ]; then + echo " Check the existing run: gh run view $RUN_ID --repo $REPO --json status,conclusion,url" + else + echo " Check manually: gh run list --repo $REPO --workflow=release-validated.yml" + fi fi else print_error "Status: FAILED TO TRIGGER WORKFLOW" From 6c856054e0eb9d0329502fecf0c30d03f410c3e8 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Wed, 28 Oct 2026 12:18:07 +0100 Subject: [PATCH 2/5] Bound release monitoring and preserve GitHub conclusions --- .../scripts/tests/test_release_automation.sh | 101 ++++++++++++++++-- utils/README.md | 10 +- utils/release.sh | 86 ++++++++++++--- 3 files changed, 174 insertions(+), 23 deletions(-) diff --git a/.github/scripts/tests/test_release_automation.sh b/.github/scripts/tests/test_release_automation.sh index ced20951a2..7b58eda66a 100755 --- a/.github/scripts/tests/test_release_automation.sh +++ b/.github/scripts/tests/test_release_automation.sh @@ -365,33 +365,53 @@ case "$1 $2" in ;; 'workflow run') exit "$CLI_DISPATCH_EXIT" ;; 'run watch') + [ "$3" = 4242 ] && [ "$4 $5" = '--repo DataDog/java-profiler' ] || exit 97 count=0 [ ! -f "$CLI_STATE/watches" ] || count=$(cat "$CLI_STATE/watches") printf '%s\n' "$((count + 1))" > "$CLI_STATE/watches" + if { [ "$count" -eq 0 ] && [ "${CLI_WATCH_HANG:-0}" -eq 1 ]; } || + { [ "$count" -gt 0 ] && [ "${CLI_NEXT_WATCH_HANG:-0}" -eq 1 ]; }; then + echo "$$" > "$CLI_STATE/watch-pid" + exec /bin/sleep 30 + fi if [ "$count" -eq 0 ]; then + /bin/sleep "${CLI_WATCH_DELAY:-0}" exit "$CLI_WATCH_EXIT" fi exit "$CLI_NEXT_WATCH_EXIT" ;; 'run view') + [ "$3" = 4242 ] && [ "$4 $5" = '--repo DataDog/java-profiler' ] || exit 97 + if [ "${CLI_QUERY_HANG:-0}" -eq 1 ]; then + echo "$$" > "$CLI_STATE/query-pid" + exec /bin/sleep 30 + fi [ "$CLI_QUERY_EXIT" -eq 0 ] || exit "$CLI_QUERY_EXIT" printf '%s\n' "$CLI_REMOTE_JSON" | jq -r "${@: -1}" ;; *) echo "Unexpected fixture command: $*" >&2; exit 97 ;; esac EOF -printf '#!/usr/bin/env bash\nexit 0\n' > "$CLI_BIN/sleep" +cat > "$CLI_BIN/sleep" <<'EOF' +#!/usr/bin/env bash +# Keep deadline polling asynchronous while skipping discovery/reconnect delays. +if [ "$1" = 1 ]; then + exec /bin/sleep 0.02 +fi +exit 0 +EOF chmod +x "$CLI_BIN/gh" "$CLI_BIN/sleep" expect_monitor() { local name=$1 expected_exit=$2 expected_status=$3 watches=$4 views=$5 shift 5 local actual_exit=0 - rm -f "$CLI_STATE/watches" + rm -f "$CLI_STATE/watches" "$CLI_STATE/"*-pid : > "$CLI_STATE/calls" CLI_OUTPUT=$( cd "$WORK" env PATH="$CLI_BIN:$PATH" CLI_STATE="$CLI_STATE" \ + RELEASE_MONITOR_TIMEOUT_SECONDS=3600 \ CLI_RUN_ID=4242 CLI_DISPATCH_EXIT=0 CLI_WATCH_EXIT=0 CLI_NEXT_WATCH_EXIT=1 \ CLI_QUERY_EXIT=0 CLI_REMOTE_JSON='{"status":"completed","conclusion":"success"}' \ "$@" bash "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1 @@ -406,17 +426,39 @@ expect_monitor() { fail "$name: unexpected watch count" [ "$(awk '/^run view / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq "$views" ] || fail "$name: unexpected status query count" + for pid_file in "$CLI_STATE/"*-pid; do + [ -f "$pid_file" ] || continue + if kill -0 "$(cat "$pid_file")" 2>/dev/null; then + fail "$name: timed-out CLI process is still running" + fi + done pass } expect_monitor 'watch succeeds' 0 'WORKFLOW SUCCEEDED' 1 0 expect_monitor 'connection lost after success' 0 'WORKFLOW SUCCEEDED' 1 1 CLI_WATCH_EXIT=1 -expect_monitor 'confirmed workflow failure' 1 'WORKFLOW FAILED' 1 1 \ - CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"failure"}' -expect_monitor 'confirmed cancellation' 1 'WORKFLOW FAILED' 1 1 \ - CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"cancelled"}' -expect_monitor 'confirmed timeout' 1 'WORKFLOW FAILED' 1 1 \ - CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"timed_out"}' +for conclusion in failure startup_failure cancelled timed_out action_required stale neutral skipped; do + expected_status='WORKFLOW COMPLETED' + expected_message="Workflow completed with conclusion: $conclusion" + case "$conclusion" in + failure|startup_failure) + expected_status='WORKFLOW FAILED' + expected_message="Workflow failed! (conclusion: $conclusion)" + ;; + esac + expect_monitor "confirmed $conclusion" 1 "$expected_status" 1 1 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON="{\"status\":\"completed\",\"conclusion\":\"$conclusion\"}" + [[ "$CLI_OUTPUT" == *"$expected_message"* ]] || + fail "immediate result lost GitHub conclusion $conclusion" + [[ "$CLI_OUTPUT" == *"Workflow conclusion: $conclusion"* ]] || + fail "terminal summary lost GitHub conclusion $conclusion" + [[ "$CLI_OUTPUT" != *'Status: WORKFLOW STATUS UNKNOWN'* ]] || + fail "known conclusion $conclusion was reported as unknown" + if [ "$expected_status" = 'WORKFLOW COMPLETED' ]; then + [[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] || + fail "conclusion $conclusion was incorrectly reported as failure" + fi +done expect_monitor 'monitor reconnects to active run' 0 'WORKFLOW SUCCEEDED' 2 1 \ CLI_WATCH_EXIT=1 CLI_NEXT_WATCH_EXIT=0 \ CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' @@ -428,9 +470,52 @@ expect_monitor 'GitHub status unavailable' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \ fail 'unavailable GitHub status was reported as workflow failure' expect_monitor 'completed run has no conclusion' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \ CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":null}' +expect_monitor 'completed run has an unrecognized conclusion' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \ + CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"future_value"}' expect_monitor 'run discovery exhausted' 1 'WORKFLOW STATUS UNKNOWN' 0 0 CLI_RUN_ID= [[ "$CLI_OUTPUT" != *'Status: DRY-RUN COMPLETED'* ]] || fail 'unknown dry-run status was reported as completed' expect_monitor 'dispatch fails' 1 'FAILED TO TRIGGER WORKFLOW' 0 0 CLI_DISPATCH_EXIT=1 +expect_monitor 'watch deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 \ + CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +[[ "$CLI_OUTPUT" == *'Monitoring timed out after 1 seconds'* ]] || + fail 'watch deadline did not report a monitoring timeout' +[[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] || + fail 'local deadline was reported as remote workflow failure' + +expect_monitor 'final status confirms success after watch deadline' 0 'WORKFLOW SUCCEEDED' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 + +expect_monitor 'status query deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_EXIT=1 CLI_QUERY_HANG=1 + +expect_monitor 'final status query is also bounded' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_QUERY_HANG=1 + +monitor_started=$SECONDS +expect_monitor 'reconnections share the original deadline' 1 'WORKFLOW STATUS UNKNOWN' 2 2 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=5 CLI_WATCH_EXIT=1 CLI_WATCH_DELAY=3 \ + CLI_NEXT_WATCH_HANG=1 CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +[ "$((SECONDS - monitor_started))" -lt 7 ] || + fail 'reconnecting reset the monitoring deadline' + +for invalid_timeout in 0 -1 01 invalid 2147483648 999999999999999999999; do + : > "$CLI_STATE/calls" + if CLI_OUTPUT=$( + cd "$WORK" + env PATH="$CLI_BIN:$PATH" CLI_STATE="$CLI_STATE" \ + RELEASE_MONITOR_TIMEOUT_SECONDS="$invalid_timeout" \ + bash "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1 + ); then + fail "invalid monitoring timeout '$invalid_timeout' was accepted" + fi + [[ "$CLI_OUTPUT" == *'RELEASE_MONITOR_TIMEOUT_SECONDS must be a positive integer'* ]] || + fail "invalid monitoring timeout '$invalid_timeout' did not report a validation error" + [ "$(awk '/^workflow run / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq 0 ] || + fail 'invalid monitoring timeout dispatched a workflow' +done +pass + echo "PASS: $TESTS hermetic release-automation test groups" diff --git a/utils/README.md b/utils/README.md index ce6fb48616..2b12c555ea 100644 --- a/utils/README.md +++ b/utils/README.md @@ -65,9 +65,15 @@ A dry run never creates a tag, pushes a branch, or triggers GitLab. If workflow monitoring loses its API connection, the script checks the run's status and reconnects to the same run, making up to three monitoring attempts -in total. If it cannot confirm the result, it reports `WORKFLOW STATUS UNKNOWN` -and exits nonzero. +in total. All attempts share a one-hour monitoring deadline. After that deadline, +the script allows up to ten seconds for a final status query. If it cannot confirm +the result, it reports `WORKFLOW STATUS UNKNOWN` and exits nonzero. The remote +workflow keeps running when local monitoring stops. +`RELEASE_MONITOR_TIMEOUT_SECONDS` overrides the default 3600-second deadline. Check the existing run before starting another release. +Recognized completed outcomes retain GitHub's exact conclusion in the output. +Only a confirmed `success` exits zero; other recognized conclusions exit nonzero. +Only `failure` and `startup_failure` are labelled as workflow failures. ### Testing release automation diff --git a/utils/release.sh b/utils/release.sh index 3bd64dd42c..ae8f8ecd68 100755 --- a/utils/release.sh +++ b/utils/release.sh @@ -37,6 +37,7 @@ BRANCH="" RELEASE_TYPE="" COMMIT_SHA="" RETRY_RUN_ID="" +MONITOR_TIMEOUT_SECONDS=${RELEASE_MONITOR_TIMEOUT_SECONDS:-3600} # Function to print colored output print_error() { @@ -57,6 +58,41 @@ print_info() { trap 'echo ""; print_warning "Interrupted"; exit 130' INT +# Bound a local CLI command without requiring GNU timeout on macOS. The +# subshell keeps its child-cleanup traps separate from the caller's traps. +run_before_deadline() ( + local deadline=$1 command_pid="" result=0 + shift + + cleanup_deadline_command() { + if [ -n "$command_pid" ]; then + kill -KILL "$command_pid" 2>/dev/null || true + wait "$command_pid" 2>/dev/null || true + # A killed gh run watch cannot leave its alternate screen itself. + if [ -t 1 ]; then + printf '\033[?1049l' + fi + fi + } + trap cleanup_deadline_command EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + [ "$SECONDS" -lt "$deadline" ] || return 124 + "$@" & + command_pid=$! + while kill -0 "$command_pid" 2>/dev/null; do + # $$ still identifies the release script inside this subshell. + # Stop its CLI child if the release script itself was terminated. + kill -0 "$$" 2>/dev/null || return 143 + [ "$SECONDS" -lt "$deadline" ] || return 124 + sleep 1 + done + wait "$command_pid" || result=$? + command_pid="" + return "$result" +) + # Read a single keypress (arrow keys, enter, q) from /dev/tty read_key() { local key="" @@ -642,6 +678,13 @@ fi # end of normal-mode block (goto_trigger != true) fi # end of else from --retry intercept +if ! [[ "$MONITOR_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]] || + [ "${#MONITOR_TIMEOUT_SECONDS}" -gt 10 ] || + [ "$MONITOR_TIMEOUT_SECONDS" -gt 2147483647 ]; then + print_error "RELEASE_MONITOR_TIMEOUT_SECONDS must be a positive integer no greater than 2147483647" + exit 1 +fi + echo "" print_info "Triggering GitHub Actions workflow..." REQUEST_ID="release-$(date -u +%Y%m%dT%H%M%SZ)-$$-$RANDOM" @@ -699,8 +742,10 @@ if gh workflow run release-validated.yml \ echo " ${REPO_URL}/actions/runs/${RUN_ID}" echo "" WORKFLOW_CONCLUSION="unknown" + MONITOR_DEADLINE=$((SECONDS + MONITOR_TIMEOUT_SECONDS)) for WATCH_ATTEMPT in 1 2 3; do - if gh run watch "$RUN_ID" --repo "$REPO" --exit-status; then + if run_before_deadline "$MONITOR_DEADLINE" \ + gh run watch "$RUN_ID" --repo "$REPO" --exit-status; then WORKFLOW_CONCLUSION="success" break fi @@ -708,22 +753,29 @@ if gh workflow run release-validated.yml \ # A failed watch can mean an API connection error. Check the run # itself before deciding that the remote workflow failed. print_warning "Workflow monitoring stopped; checking GitHub for the run's status..." - if REMOTE_CONCLUSION=$(gh run view "$RUN_ID" --repo "$REPO" \ + STATUS_DEADLINE=$MONITOR_DEADLINE + if [ "$SECONDS" -ge "$MONITOR_DEADLINE" ]; then + # Allow one final status query after the monitoring deadline, + # but bound it too so an API outage cannot hang the script. + STATUS_DEADLINE=$((SECONDS + 10)) + fi + if REMOTE_CONCLUSION=$(run_before_deadline "$STATUS_DEADLINE" \ + gh run view "$RUN_ID" --repo "$REPO" \ --json status,conclusion \ --jq 'if .status == "completed" then (.conclusion // "unknown") else "unknown" end'); then case "$REMOTE_CONCLUSION" in - success) - WORKFLOW_CONCLUSION="success" - break - ;; - failure|cancelled|timed_out|action_required|startup_failure|stale|neutral|skipped) - WORKFLOW_CONCLUSION="failure" - print_info "GitHub reports workflow conclusion: $REMOTE_CONCLUSION" + success|failure|cancelled|timed_out|action_required|startup_failure|stale|neutral|skipped) + WORKFLOW_CONCLUSION="$REMOTE_CONCLUSION" break ;; esac fi + if [ "$SECONDS" -ge "$MONITOR_DEADLINE" ]; then + print_warning "Monitoring timed out after ${MONITOR_TIMEOUT_SECONDS} seconds. The remote workflow has not been cancelled." + break + fi + if [ "$WATCH_ATTEMPT" -lt 3 ]; then print_warning "Reconnecting to the same workflow run ($((WATCH_ATTEMPT + 1))/3)..." sleep 2 @@ -732,9 +784,12 @@ if gh workflow run release-validated.yml \ if [ "$WORKFLOW_CONCLUSION" = "success" ]; then print_success "✓ Workflow completed successfully!" - elif [ "$WORKFLOW_CONCLUSION" = "failure" ]; then - print_error "✗ Workflow failed!" + elif [ "$WORKFLOW_CONCLUSION" = "failure" ] || [ "$WORKFLOW_CONCLUSION" = "startup_failure" ]; then + print_error "✗ Workflow failed! (conclusion: $WORKFLOW_CONCLUSION)" echo " View logs: gh run view $RUN_ID --repo $REPO --log-failed" + elif [ "$WORKFLOW_CONCLUSION" != "unknown" ]; then + print_warning "Workflow completed with conclusion: $WORKFLOW_CONCLUSION" + echo " Inspect the run: gh run view $RUN_ID --repo $REPO --json status,conclusion,url" else print_warning "Monitoring could not confirm the workflow result. The remote run may still be running." echo " Inspect the existing run: ${REPO_URL}/actions/runs/${RUN_ID}" @@ -797,9 +852,14 @@ echo "" if [ "$WORKFLOW_SUCCESS" = true ]; then if [ "${WORKFLOW_CONCLUSION:-}" = "success" ]; then print_success "Status: WORKFLOW SUCCEEDED" - elif [ "${WORKFLOW_CONCLUSION:-}" = "failure" ]; then + elif [ "${WORKFLOW_CONCLUSION:-}" = "failure" ] || [ "${WORKFLOW_CONCLUSION:-}" = "startup_failure" ]; then print_error "Status: WORKFLOW FAILED" - echo " View logs: gh run view $RUN_ID --log-failed" + echo " Workflow conclusion: $WORKFLOW_CONCLUSION" + echo " View logs: gh run view $RUN_ID --repo $REPO --log-failed" + elif [ "${WORKFLOW_CONCLUSION:-unknown}" != "unknown" ]; then + print_warning "Status: WORKFLOW COMPLETED" + echo " Workflow conclusion: $WORKFLOW_CONCLUSION" + echo " Inspect the run: gh run view $RUN_ID --repo $REPO --json status,conclusion,url" else print_warning "Status: WORKFLOW STATUS UNKNOWN" if [ -n "$RUN_ID" ]; then From 0d40d16939f8c3128edff29d29748956583f0ff6 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Wed, 28 Oct 2026 13:55:05 +0100 Subject: [PATCH 3/5] Offer interactive release monitoring extensions after timeout --- .github/scripts/tests/run_release_cli.py | 68 +++++++++++++++++++ .../scripts/tests/test_release_automation.sh | 44 +++++++++++- utils/README.md | 10 ++- utils/release.sh | 12 +++- 4 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/tests/run_release_cli.py diff --git a/.github/scripts/tests/run_release_cli.py b/.github/scripts/tests/run_release_cli.py new file mode 100644 index 0000000000..e7f5f99899 --- /dev/null +++ b/.github/scripts/tests/run_release_cli.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright 2026, Datadog, Inc + +"""Run the release CLI, optionally answering prompts through a real terminal.""" + +import errno +import os +import pty +import select +import signal +import sys +import time + + +def main(): + """Forward the CLI exit status and script terminal replies when requested.""" + command = ["bash", *sys.argv[1:]] + if "CLI_PTY_REPLIES" not in os.environ: + os.execvp(command[0], command) + + replies = iter(os.environ["CLI_PTY_REPLIES"].split("\n")) + prompt = b"[y/N]: " + pending = b"" + child_pid, terminal = pty.fork() + if child_pid == 0: + os.execvp(command[0], command) + + try: + deadline = time.monotonic() + 30 + while True: + if time.monotonic() >= deadline: + raise RuntimeError("interactive release test timed out") + readable, _, _ = select.select([terminal], [], [], 0.1) + if not readable: + continue + try: + output = os.read(terminal, 4096) + except OSError as error: + if error.errno == errno.EIO: + break + raise + if not output: + break + sys.stdout.buffer.write(output) + sys.stdout.buffer.flush() + pending += output + while prompt in pending: + pending = pending.split(prompt, 1)[1] + reply = next(replies).encode() + # Ctrl-C and Ctrl-D exercise terminal interrupt and EOF handling. + os.write(terminal, reply if reply in (b"\x03", b"\x04") else reply + b"\n") + _, status = os.waitpid(child_pid, 0) + except BaseException: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.waitpid(child_pid, 0) + raise + finally: + os.close(terminal) + + result = os.waitstatus_to_exitcode(status) + return result if result >= 0 else 128 - result + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/tests/test_release_automation.sh b/.github/scripts/tests/test_release_automation.sh index 7b58eda66a..00c5f838c2 100755 --- a/.github/scripts/tests/test_release_automation.sh +++ b/.github/scripts/tests/test_release_automation.sh @@ -414,12 +414,15 @@ expect_monitor() { RELEASE_MONITOR_TIMEOUT_SECONDS=3600 \ CLI_RUN_ID=4242 CLI_DISPATCH_EXIT=0 CLI_WATCH_EXIT=0 CLI_NEXT_WATCH_EXIT=1 \ CLI_QUERY_EXIT=0 CLI_REMOTE_JSON='{"status":"completed","conclusion":"success"}' \ - "$@" bash "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1 + "$@" python3 "$ROOT/.github/scripts/tests/run_release_cli.py" \ + "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1 ) || actual_exit=$? [ "$actual_exit" -eq "$expected_exit" ] || fail "$name: expected exit $expected_exit, got $actual_exit" - [[ "$CLI_OUTPUT" == *"Status: $expected_status"* ]] || - fail "$name: expected summary '$expected_status'" + if [ -n "$expected_status" ]; then + [[ "$CLI_OUTPUT" == *"Status: $expected_status"* ]] || + fail "$name: expected summary '$expected_status'" + fi [ "$(awk '/^workflow run / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq 1 ] || fail "$name: workflow must be dispatched exactly once" [ "$(awk '/^run watch / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq "$watches" ] || @@ -484,6 +487,8 @@ expect_monitor 'watch deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ fail 'watch deadline did not report a monitoring timeout' [[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] || fail 'local deadline was reported as remote workflow failure' +[[ "$CLI_OUTPUT" != *'Continue monitoring the same workflow run'* ]] || + fail 'unattended monitoring prompted for input' expect_monitor 'final status confirms success after watch deadline' 0 'WORKFLOW SUCCEEDED' 1 1 \ RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 @@ -501,6 +506,39 @@ expect_monitor 'reconnections share the original deadline' 1 'WORKFLOW STATUS UN [ "$((SECONDS - monitor_started))" -lt 7 ] || fail 'reconnecting reset the monitoring deadline' +for reply in n '' $'\004'; do + expect_monitor 'decline monitoring extension, default, or EOF' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_PTY_REPLIES="$reply" \ + CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' + [[ "$CLI_OUTPUT" == *'Continue monitoring the same workflow run'* ]] || + fail 'interactive timeout did not ask whether to continue' + [[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] || + fail 'stopping local monitoring was reported as a remote failure' +done + +expect_monitor 'continue watching after timeout' 0 'WORKFLOW SUCCEEDED' 2 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_NEXT_WATCH_EXIT=0 \ + CLI_PTY_REPLIES=y CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +[[ "$CLI_OUTPUT" == *'Continuing to monitor the same workflow run'* ]] || + fail 'accepted monitoring extension did not resume watching' + +expect_monitor 'renewed monitoring window prompts again' 1 'WORKFLOW STATUS UNKNOWN' 2 2 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_NEXT_WATCH_HANG=1 \ + CLI_PTY_REPLIES=$'y\nn' CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +[ "$(printf '%s\n' "$CLI_OUTPUT" | awk '/Continue monitoring the same workflow run/ { n++ } END { print n+0 }')" -eq 2 ] || + fail 'renewed monitoring window did not ask again at its deadline' + +expect_monitor 'confirmed result does not prompt for extension' 0 'WORKFLOW SUCCEEDED' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_PTY_REPLIES=n +[[ "$CLI_OUTPUT" != *'Continue monitoring the same workflow run'* ]] || + fail 'confirmed workflow completion prompted for an extension' + +expect_monitor 'interrupt at timeout prompt' 130 '' 1 1 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_PTY_REPLIES=$'\003' \ + CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}' +[[ "$CLI_OUTPUT" == *'Interrupted'* ]] || + fail 'interrupt at timeout prompt was not handled' + for invalid_timeout in 0 -1 01 invalid 2147483648 999999999999999999999; do : > "$CLI_STATE/calls" if CLI_OUTPUT=$( diff --git a/utils/README.md b/utils/README.md index 2b12c555ea..34fbb242a9 100644 --- a/utils/README.md +++ b/utils/README.md @@ -65,10 +65,13 @@ A dry run never creates a tag, pushes a branch, or triggers GitLab. If workflow monitoring loses its API connection, the script checks the run's status and reconnects to the same run, making up to three monitoring attempts -in total. All attempts share a one-hour monitoring deadline. After that deadline, +per monitoring window. All attempts share a one-hour deadline. After that deadline, the script allows up to ten seconds for a final status query. If it cannot confirm -the result, it reports `WORKFLOW STATUS UNKNOWN` and exits nonzero. The remote -workflow keeps running when local monitoring stops. +the result, an interactive session asks whether to watch the same run for another +hour. Answering `y` or `yes` renews the deadline and monitoring attempts. Answering +`n`, pressing Enter, or reaching EOF stops monitoring with +`WORKFLOW STATUS UNKNOWN` and a nonzero exit. Unattended sessions stop without +prompting. The remote workflow keeps running when local monitoring stops. `RELEASE_MONITOR_TIMEOUT_SECONDS` overrides the default 3600-second deadline. Check the existing run before starting another release. Recognized completed outcomes retain GitHub's exact conclusion in the output. @@ -83,6 +86,7 @@ trivial-PR authorization, and recovery from workflow monitoring errors using temporary local Git repositories and a GitHub CLI fixture. It does not load credentials or contact GitHub, so it cannot publish, tag, push, or merge anything remotely. +The terminal interaction tests require Python 3. ```bash .github/scripts/tests/test_release_automation.sh diff --git a/utils/release.sh b/utils/release.sh index ae8f8ecd68..97a10f76bc 100755 --- a/utils/release.sh +++ b/utils/release.sh @@ -743,7 +743,8 @@ if gh workflow run release-validated.yml \ echo "" WORKFLOW_CONCLUSION="unknown" MONITOR_DEADLINE=$((SECONDS + MONITOR_TIMEOUT_SECONDS)) - for WATCH_ATTEMPT in 1 2 3; do + WATCH_ATTEMPT=1 + while [ "$WATCH_ATTEMPT" -le 3 ]; do if run_before_deadline "$MONITOR_DEADLINE" \ gh run watch "$RUN_ID" --repo "$REPO" --exit-status; then WORKFLOW_CONCLUSION="success" @@ -773,6 +774,14 @@ if gh workflow run release-validated.yml \ if [ "$SECONDS" -ge "$MONITOR_DEADLINE" ]; then print_warning "Monitoring timed out after ${MONITOR_TIMEOUT_SECONDS} seconds. The remote workflow has not been cancelled." + if [ -t 0 ] && + read -r -p "Continue monitoring the same workflow run for another ${MONITOR_TIMEOUT_SECONDS} seconds? [y/N]: " REPLY Date: Wed, 28 Oct 2026 13:59:36 +0100 Subject: [PATCH 4/5] Explain expected process cleanup race in release CLI tests --- .github/scripts/tests/run_release_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/tests/run_release_cli.py b/.github/scripts/tests/run_release_cli.py index e7f5f99899..cafa50470d 100644 --- a/.github/scripts/tests/run_release_cli.py +++ b/.github/scripts/tests/run_release_cli.py @@ -54,6 +54,7 @@ def main(): try: os.killpg(child_pid, signal.SIGKILL) except ProcessLookupError: + # The child may have exited and its process group disappeared before cleanup. pass os.waitpid(child_pid, 0) raise From 07b51ecab33c923fa8f02b942a423ca66c9f200d Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Fri, 30 Oct 2026 15:03:08 +0100 Subject: [PATCH 5/5] Preserve final status-query grace across the monitoring deadline --- .../scripts/tests/test_release_automation.sh | 18 ++++++++- utils/README.md | 7 ++-- utils/release.sh | 40 ++++++++++++------- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/.github/scripts/tests/test_release_automation.sh b/.github/scripts/tests/test_release_automation.sh index 00c5f838c2..cab0e25ae4 100755 --- a/.github/scripts/tests/test_release_automation.sh +++ b/.github/scripts/tests/test_release_automation.sh @@ -387,6 +387,14 @@ case "$1 $2" in exec /bin/sleep 30 fi [ "$CLI_QUERY_EXIT" -eq 0 ] || exit "$CLI_QUERY_EXIT" + if [ "${CLI_QUERY_DELAY:-0}" -gt 0 ]; then + echo "$$" > "$CLI_STATE/query-pid" + exec python3 -c ' +import os, sys, time +time.sleep(float(os.environ["CLI_QUERY_DELAY"])) +os.execvp("jq", ["jq", "-nr", "--argjson", "run", os.environ["CLI_REMOTE_JSON"], "$run | " + sys.argv[1]]) +' "${@: -1}" + fi printf '%s\n' "$CLI_REMOTE_JSON" | jq -r "${@: -1}" ;; *) echo "Unexpected fixture command: $*" >&2; exit 97 ;; @@ -493,8 +501,16 @@ expect_monitor 'watch deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ expect_monitor 'final status confirms success after watch deadline' 0 'WORKFLOW SUCCEEDED' 1 1 \ RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 -expect_monitor 'status query deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ +expect_monitor 'status query crossing the deadline still confirms success' 0 'WORKFLOW SUCCEEDED' 1 2 \ + RELEASE_MONITOR_TIMEOUT_SECONDS=2 CLI_WATCH_EXIT=1 CLI_QUERY_DELAY=3 CLI_PTY_REPLIES=n +[[ "$CLI_OUTPUT" != *'Continue monitoring the same workflow run'* ]] || + fail 'status-query grace was skipped before prompting for an extension' + +status_started=$SECONDS +expect_monitor 'status query deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 2 \ RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_EXIT=1 CLI_QUERY_HANG=1 +[ "$((SECONDS - status_started))" -lt 14 ] || + fail 'status-query retries exceeded the bounded final grace period' expect_monitor 'final status query is also bounded' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \ RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_QUERY_HANG=1 diff --git a/utils/README.md b/utils/README.md index 34fbb242a9..36e5077183 100644 --- a/utils/README.md +++ b/utils/README.md @@ -65,9 +65,10 @@ A dry run never creates a tag, pushes a branch, or triggers GitLab. If workflow monitoring loses its API connection, the script checks the run's status and reconnects to the same run, making up to three monitoring attempts -per monitoring window. All attempts share a one-hour deadline. After that deadline, -the script allows up to ten seconds for a final status query. If it cannot confirm -the result, an interactive session asks whether to watch the same run for another +per monitoring window. All attempts share a one-hour deadline. When the deadline +is reached, the script allows up to ten seconds for a final status query, including +when it must retry a query interrupted by the shared deadline. If the result remains +unconfirmed, an interactive session asks whether to watch the same run for another hour. Answering `y` or `yes` renews the deadline and monitoring attempts. Answering `n`, pressing Enter, or reaching EOF stops monitoring with `WORKFLOW STATUS UNKNOWN` and a nonzero exit. Unattended sessions stop without diff --git a/utils/release.sh b/utils/release.sh index 97a10f76bc..7baf57cca8 100755 --- a/utils/release.sh +++ b/utils/release.sh @@ -755,21 +755,31 @@ if gh workflow run release-validated.yml \ # itself before deciding that the remote workflow failed. print_warning "Workflow monitoring stopped; checking GitHub for the run's status..." STATUS_DEADLINE=$MONITOR_DEADLINE - if [ "$SECONDS" -ge "$MONITOR_DEADLINE" ]; then - # Allow one final status query after the monitoring deadline, - # but bound it too so an API outage cannot hang the script. - STATUS_DEADLINE=$((SECONDS + 10)) - fi - if REMOTE_CONCLUSION=$(run_before_deadline "$STATUS_DEADLINE" \ - gh run view "$RUN_ID" --repo "$REPO" \ - --json status,conclusion \ - --jq 'if .status == "completed" then (.conclusion // "unknown") else "unknown" end'); then - case "$REMOTE_CONCLUSION" in - success|failure|cancelled|timed_out|action_required|startup_failure|stale|neutral|skipped) - WORKFLOW_CONCLUSION="$REMOTE_CONCLUSION" - break - ;; - esac + while true; do + if [ "$SECONDS" -ge "$MONITOR_DEADLINE" ]; then + # Bound the final query too so an API outage cannot hang the script. + STATUS_DEADLINE=$((SECONDS + 10)) + fi + if REMOTE_CONCLUSION=$(run_before_deadline "$STATUS_DEADLINE" \ + gh run view "$RUN_ID" --repo "$REPO" \ + --json status,conclusion \ + --jq 'if .status == "completed" then (.conclusion // "unknown") else "unknown" end'); then + case "$REMOTE_CONCLUSION" in + success|failure|cancelled|timed_out|action_required|startup_failure|stale|neutral|skipped) + WORKFLOW_CONCLUSION="$REMOTE_CONCLUSION" + ;; + esac + break + fi + # If the shared deadline expired during this query, retry once + # with the full final-query allowance before prompting or stopping. + if [ "$STATUS_DEADLINE" -ne "$MONITOR_DEADLINE" ] || + [ "$SECONDS" -lt "$MONITOR_DEADLINE" ]; then + break + fi + done + if [ "$WORKFLOW_CONCLUSION" != "unknown" ]; then + break fi if [ "$SECONDS" -ge "$MONITOR_DEADLINE" ]; then