Fix false release failures after GitHub API connection errors

.github/scripts/tests/run_release_cli.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# Copyright 2026, Datadog, Inc
+
+"""Run the release CLI, optionally answering prompts through a real terminal."""
+
+import errno
+import os
+import pty
+import select
+import signal
+import sys
+import time
+
+
+def main():
+    """Forward the CLI exit status and script terminal replies when requested."""
+    command = ["bash", *sys.argv[1:]]
+    if "CLI_PTY_REPLIES" not in os.environ:
+        os.execvp(command[0], command)
+
+    replies = iter(os.environ["CLI_PTY_REPLIES"].split("\n"))
+    prompt = b"[y/N]: "
+    pending = b""
+    child_pid, terminal = pty.fork()
+    if child_pid == 0:
+        os.execvp(command[0], command)
+
+    try:
+        deadline = time.monotonic() + 30
+        while True:
+            if time.monotonic() >= deadline:
+                raise RuntimeError("interactive release test timed out")
+            readable, _, _ = select.select([terminal], [], [], 0.1)
+            if not readable:
+                continue
+            try:
+                output = os.read(terminal, 4096)
+            except OSError as error:
+                if error.errno == errno.EIO:
+                    break
+                raise
+            if not output:
+                break
+            sys.stdout.buffer.write(output)
+            sys.stdout.buffer.flush()
+            pending += output
+            while prompt in pending:
+                pending = pending.split(prompt, 1)[1]
+                reply = next(replies).encode()
+                # Ctrl-C and Ctrl-D exercise terminal interrupt and EOF handling.
+                os.write(terminal, reply if reply in (b"\x03", b"\x04") else reply + b"\n")
+        _, status = os.waitpid(child_pid, 0)
+    except BaseException:
+        try:
+            os.killpg(child_pid, signal.SIGKILL)
+        except ProcessLookupError:
+            # The child may have exited and its process group disappeared before cleanup.
+            pass
+        os.waitpid(child_pid, 0)
+        raise
+    finally:
+        os.close(terminal)
+
+    result = os.waitstatus_to_exitcode(status)
+    return result if result >= 0 else 128 - result
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/tests/test_release_automation.sh

@@ -337,4 +337,239 @@ if git --git-dir="$REMOTE" show-ref --verify --quiet "refs/tags/v_1.49.0" 2>/dev
 fi
 pass
 
+# --- Release CLI monitoring tests (GitHub CLI fixture, no network) ------------
+
+REMOTE=$(setup_remote)
+WORK=$(setup_work "$REMOTE" main)
+CLI_SHA=$(git -C "$WORK" rev-parse HEAD)
+CLI_BIN="$TEMP_DIR/cli-bin"
+CLI_STATE="$TEMP_DIR/cli-state"
+mkdir "$CLI_BIN" "$CLI_STATE"
+cat > "$CLI_BIN/gh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "$*" >> "$CLI_STATE/calls"
+case "$1 $2" in
+  'auth status') echo 'Logged in to github.com account release-test' ;;
+  'repo view')
+    case "$4" in
+      nameWithOwner) echo 'DataDog/java-profiler' ;;
+      url) echo 'https://github.com/DataDog/java-profiler' ;;
+      viewerPermission) echo WRITE ;;
+      *) exit 97 ;;
+    esac
+    ;;
+  'api user') echo release-test ;;
+  'api repos/DataDog/java-profiler/actions/runs?event=workflow_dispatch&per_page=50')
+    printf '%s\n' "$CLI_RUN_ID"
+    ;;
+  'workflow run') exit "$CLI_DISPATCH_EXIT" ;;
+  'run watch')
+    [ "$3" = 4242 ] && [ "$4 $5" = '--repo DataDog/java-profiler' ] || exit 97
+    count=0
+    [ ! -f "$CLI_STATE/watches" ] || count=$(cat "$CLI_STATE/watches")
+    printf '%s\n' "$((count + 1))" > "$CLI_STATE/watches"
+    if { [ "$count" -eq 0 ] && [ "${CLI_WATCH_HANG:-0}" -eq 1 ]; } ||
+       { [ "$count" -gt 0 ] && [ "${CLI_NEXT_WATCH_HANG:-0}" -eq 1 ]; }; then
+      echo "$$" > "$CLI_STATE/watch-pid"
+      exec /bin/sleep 30
+    fi
+    if [ "$count" -eq 0 ]; then
+      /bin/sleep "${CLI_WATCH_DELAY:-0}"
+      exit "$CLI_WATCH_EXIT"
+    fi
+    exit "$CLI_NEXT_WATCH_EXIT"
+    ;;
+  'run view')
+    [ "$3" = 4242 ] && [ "$4 $5" = '--repo DataDog/java-profiler' ] || exit 97
+    if [ "${CLI_QUERY_HANG:-0}" -eq 1 ]; then
+      echo "$$" > "$CLI_STATE/query-pid"
+      exec /bin/sleep 30
+    fi
+    [ "$CLI_QUERY_EXIT" -eq 0 ] || exit "$CLI_QUERY_EXIT"
+    if [ "${CLI_QUERY_DELAY:-0}" -gt 0 ]; then
+      echo "$$" > "$CLI_STATE/query-pid"
+      exec python3 -c '
+import os, sys, time
+time.sleep(float(os.environ["CLI_QUERY_DELAY"]))
+os.execvp("jq", ["jq", "-nr", "--argjson", "run", os.environ["CLI_REMOTE_JSON"], "$run | " + sys.argv[1]])
+' "${@: -1}"
+    fi
+    printf '%s\n' "$CLI_REMOTE_JSON" | jq -r "${@: -1}"
+    ;;
+  *) echo "Unexpected fixture command: $*" >&2; exit 97 ;;
+esac
+EOF
+cat > "$CLI_BIN/sleep" <<'EOF'
+#!/usr/bin/env bash
+# Keep deadline polling asynchronous while skipping discovery/reconnect delays.
+if [ "$1" = 1 ]; then
+  exec /bin/sleep 0.02
+fi
+exit 0
+EOF
+chmod +x "$CLI_BIN/gh" "$CLI_BIN/sleep"
+
+expect_monitor() {
+  local name=$1 expected_exit=$2 expected_status=$3 watches=$4 views=$5
+  shift 5
+  local actual_exit=0
+  rm -f "$CLI_STATE/watches" "$CLI_STATE/"*-pid
+  : > "$CLI_STATE/calls"
+  CLI_OUTPUT=$(
+    cd "$WORK"
+    env PATH="$CLI_BIN:$PATH" CLI_STATE="$CLI_STATE" \
+      RELEASE_MONITOR_TIMEOUT_SECONDS=3600 \
+      CLI_RUN_ID=4242 CLI_DISPATCH_EXIT=0 CLI_WATCH_EXIT=0 CLI_NEXT_WATCH_EXIT=1 \
+      CLI_QUERY_EXIT=0 CLI_REMOTE_JSON='{"status":"completed","conclusion":"success"}' \
+      "$@" python3 "$ROOT/.github/scripts/tests/run_release_cli.py" \
+      "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1
+  ) || actual_exit=$?
+  [ "$actual_exit" -eq "$expected_exit" ] ||
+    fail "$name: expected exit $expected_exit, got $actual_exit"
+  if [ -n "$expected_status" ]; then
+    [[ "$CLI_OUTPUT" == *"Status: $expected_status"* ]] ||
+      fail "$name: expected summary '$expected_status'"
+  fi
+  [ "$(awk '/^workflow run / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq 1 ] ||
+    fail "$name: workflow must be dispatched exactly once"
+  [ "$(awk '/^run watch / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq "$watches" ] ||
+    fail "$name: unexpected watch count"
+  [ "$(awk '/^run view / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq "$views" ] ||
+    fail "$name: unexpected status query count"
+  for pid_file in "$CLI_STATE/"*-pid; do
+    [ -f "$pid_file" ] || continue
+    if kill -0 "$(cat "$pid_file")" 2>/dev/null; then
+      fail "$name: timed-out CLI process is still running"
+    fi
+  done
+  pass
+}
+
+expect_monitor 'watch succeeds' 0 'WORKFLOW SUCCEEDED' 1 0
+expect_monitor 'connection lost after success' 0 'WORKFLOW SUCCEEDED' 1 1 CLI_WATCH_EXIT=1
+for conclusion in failure startup_failure cancelled timed_out action_required stale neutral skipped; do
+  expected_status='WORKFLOW COMPLETED'
+  expected_message="Workflow completed with conclusion: $conclusion"
+  case "$conclusion" in
+    failure|startup_failure)
+      expected_status='WORKFLOW FAILED'
+      expected_message="Workflow failed! (conclusion: $conclusion)"
+      ;;
+  esac
+  expect_monitor "confirmed $conclusion" 1 "$expected_status" 1 1 \
+    CLI_WATCH_EXIT=1 CLI_REMOTE_JSON="{\"status\":\"completed\",\"conclusion\":\"$conclusion\"}"
+  [[ "$CLI_OUTPUT" == *"$expected_message"* ]] ||
+    fail "immediate result lost GitHub conclusion $conclusion"
+  [[ "$CLI_OUTPUT" == *"Workflow conclusion: $conclusion"* ]] ||
+    fail "terminal summary lost GitHub conclusion $conclusion"
+  [[ "$CLI_OUTPUT" != *'Status: WORKFLOW STATUS UNKNOWN'* ]] ||
+    fail "known conclusion $conclusion was reported as unknown"
+  if [ "$expected_status" = 'WORKFLOW COMPLETED' ]; then
+    [[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] ||
+      fail "conclusion $conclusion was incorrectly reported as failure"
+  fi
+done
+expect_monitor 'monitor reconnects to active run' 0 'WORKFLOW SUCCEEDED' 2 1 \
+  CLI_WATCH_EXIT=1 CLI_NEXT_WATCH_EXIT=0 \
+  CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+expect_monitor 'monitor cannot reconnect to active run' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \
+  CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+expect_monitor 'GitHub status unavailable' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \
+  CLI_WATCH_EXIT=1 CLI_QUERY_EXIT=1
+[[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] ||
+  fail 'unavailable GitHub status was reported as workflow failure'
+expect_monitor 'completed run has no conclusion' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \
+  CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":null}'
+expect_monitor 'completed run has an unrecognized conclusion' 1 'WORKFLOW STATUS UNKNOWN' 3 3 \
+  CLI_WATCH_EXIT=1 CLI_REMOTE_JSON='{"status":"completed","conclusion":"future_value"}'
+expect_monitor 'run discovery exhausted' 1 'WORKFLOW STATUS UNKNOWN' 0 0 CLI_RUN_ID=
+[[ "$CLI_OUTPUT" != *'Status: DRY-RUN COMPLETED'* ]] ||
+  fail 'unknown dry-run status was reported as completed'
+expect_monitor 'dispatch fails' 1 'FAILED TO TRIGGER WORKFLOW' 0 0 CLI_DISPATCH_EXIT=1
+
+expect_monitor 'watch deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 \
+  CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+[[ "$CLI_OUTPUT" == *'Monitoring timed out after 1 seconds'* ]] ||
+  fail 'watch deadline did not report a monitoring timeout'
+[[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] ||
+  fail 'local deadline was reported as remote workflow failure'
+[[ "$CLI_OUTPUT" != *'Continue monitoring the same workflow run'* ]] ||
+  fail 'unattended monitoring prompted for input'
+
+expect_monitor 'final status confirms success after watch deadline' 0 'WORKFLOW SUCCEEDED' 1 1 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1
+
+expect_monitor 'status query crossing the deadline still confirms success' 0 'WORKFLOW SUCCEEDED' 1 2 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=2 CLI_WATCH_EXIT=1 CLI_QUERY_DELAY=3 CLI_PTY_REPLIES=n
+[[ "$CLI_OUTPUT" != *'Continue monitoring the same workflow run'* ]] ||
+  fail 'status-query grace was skipped before prompting for an extension'
+
+status_started=$SECONDS
+expect_monitor 'status query deadline expires' 1 'WORKFLOW STATUS UNKNOWN' 1 2 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_EXIT=1 CLI_QUERY_HANG=1
+[ "$((SECONDS - status_started))" -lt 14 ] ||
+  fail 'status-query retries exceeded the bounded final grace period'
+
+expect_monitor 'final status query is also bounded' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_QUERY_HANG=1
+
+monitor_started=$SECONDS
+expect_monitor 'reconnections share the original deadline' 1 'WORKFLOW STATUS UNKNOWN' 2 2 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=5 CLI_WATCH_EXIT=1 CLI_WATCH_DELAY=3 \
+  CLI_NEXT_WATCH_HANG=1 CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+[ "$((SECONDS - monitor_started))" -lt 7 ] ||
+  fail 'reconnecting reset the monitoring deadline'
+
+for reply in n '' $'\004'; do
+  expect_monitor 'decline monitoring extension, default, or EOF' 1 'WORKFLOW STATUS UNKNOWN' 1 1 \
+    RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_PTY_REPLIES="$reply" \
+    CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+  [[ "$CLI_OUTPUT" == *'Continue monitoring the same workflow run'* ]] ||
+    fail 'interactive timeout did not ask whether to continue'
+  [[ "$CLI_OUTPUT" != *'Status: WORKFLOW FAILED'* ]] ||
+    fail 'stopping local monitoring was reported as a remote failure'
+done
+
+expect_monitor 'continue watching after timeout' 0 'WORKFLOW SUCCEEDED' 2 1 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_NEXT_WATCH_EXIT=0 \
+  CLI_PTY_REPLIES=y CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+[[ "$CLI_OUTPUT" == *'Continuing to monitor the same workflow run'* ]] ||
+  fail 'accepted monitoring extension did not resume watching'
+
+expect_monitor 'renewed monitoring window prompts again' 1 'WORKFLOW STATUS UNKNOWN' 2 2 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_NEXT_WATCH_HANG=1 \
+  CLI_PTY_REPLIES=$'y\nn' CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+[ "$(printf '%s\n' "$CLI_OUTPUT" | awk '/Continue monitoring the same workflow run/ { n++ } END { print n+0 }')" -eq 2 ] ||
+  fail 'renewed monitoring window did not ask again at its deadline'
+
+expect_monitor 'confirmed result does not prompt for extension' 0 'WORKFLOW SUCCEEDED' 1 1 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_PTY_REPLIES=n
+[[ "$CLI_OUTPUT" != *'Continue monitoring the same workflow run'* ]] ||
+  fail 'confirmed workflow completion prompted for an extension'
+
+expect_monitor 'interrupt at timeout prompt' 130 '' 1 1 \
+  RELEASE_MONITOR_TIMEOUT_SECONDS=1 CLI_WATCH_HANG=1 CLI_PTY_REPLIES=$'\003' \
+  CLI_REMOTE_JSON='{"status":"in_progress","conclusion":null}'
+[[ "$CLI_OUTPUT" == *'Interrupted'* ]] ||
+  fail 'interrupt at timeout prompt was not handled'
+
+for invalid_timeout in 0 -1 01 invalid 2147483648 999999999999999999999; do
+  : > "$CLI_STATE/calls"
+  if CLI_OUTPUT=$(
+    cd "$WORK"
+    env PATH="$CLI_BIN:$PATH" CLI_STATE="$CLI_STATE" \
+      RELEASE_MONITOR_TIMEOUT_SECONDS="$invalid_timeout" \
+      bash "$ROOT/utils/release.sh" minor --branch main --commit "$CLI_SHA" 2>&1
+  ); then
+    fail "invalid monitoring timeout '$invalid_timeout' was accepted"
+  fi
+  [[ "$CLI_OUTPUT" == *'RELEASE_MONITOR_TIMEOUT_SECONDS must be a positive integer'* ]] ||
+    fail "invalid monitoring timeout '$invalid_timeout' did not report a validation error"
+  [ "$(awk '/^workflow run / { n++ } END { print n+0 }' "$CLI_STATE/calls")" -eq 0 ] ||
+    fail 'invalid monitoring timeout dispatched a workflow'
+done
+pass
+
 echo "PASS: $TESTS hermetic release-automation test groups"

utils/README.md

@@ -63,13 +63,31 @@ can be cancelled at any time with `q` or Ctrl-C.
 
 A dry run never creates a tag, pushes a branch, or triggers GitLab.
 
+If workflow monitoring loses its API connection, the script checks the run's
+status and reconnects to the same run, making up to three monitoring attempts
+per monitoring window. All attempts share a one-hour deadline. When the deadline
+is reached, the script allows up to ten seconds for a final status query, including
+when it must retry a query interrupted by the shared deadline. If the result remains
+unconfirmed, an interactive session asks whether to watch the same run for another
+hour. Answering `y` or `yes` renews the deadline and monitoring attempts. Answering
+`n`, pressing Enter, or reaching EOF stops monitoring with
+`WORKFLOW STATUS UNKNOWN` and a nonzero exit. Unattended sessions stop without
+prompting. The remote workflow keeps running when local monitoring stops.
+`RELEASE_MONITOR_TIMEOUT_SECONDS` overrides the default 3600-second deadline.
+Check the existing run before starting another release.
+Recognized completed outcomes retain GitHub's exact conclusion in the output.
+Only a confirmed `success` exits zero; other recognized conclusions exit nonzero.
+Only `failure` and `startup_failure` are labelled as workflow failures.
+
 ### Testing release automation
 
 `.github/scripts/tests/test_release_automation.sh` is a single hermetic shell
-test. It validates release tag/branch creation, version computation, and
-trivial-PR authorization using temporary local git fixtures. It does not
-load credentials or invoke `gh`, so it cannot publish, tag, push, or merge
+test. It validates release tag/branch creation, version computation,
+trivial-PR authorization, and recovery from workflow monitoring errors using
+temporary local Git repositories and a GitHub CLI fixture. It does not load
+credentials or contact GitHub, so it cannot publish, tag, push, or merge
 anything remotely.
+The terminal interaction tests require Python 3.
 
 ```bash
 .github/scripts/tests/test_release_automation.sh