Release script refuses a release that can't be cut

README.md

@@ -346,6 +346,8 @@ scripts/release.sh 0.4.0
 
 It works from `origin/main` in a temporary worktree, so your checked-out branch and uncommitted changes don't matter and aren't touched. It opens a pull request from `release-<version>` that bumps the version in `Cargo.toml` and `Cargo.lock` and nothing else. Its body is a summary of the release, then the version diff. `claude -p`, with no tools, writes the summary from the version diff and the title, number and body of each pull request merged since the last tag, following the prompt in [`scripts/release-summary.md`](scripts/release-summary.md). If `claude` fails or prints nothing, the script warns and uses GitHub's generated notes as the summary instead, with a note saying so. It waits for the pull request's checks, merges it with a merge commit, then tags that merge commit `v<version>` and pushes the tag. It prints a line on stderr for each step. If the checks fail, it stops before merging and leaves the pull request open.
 
+Before it pushes anything, it refuses a release that can't be cut and says why: a version that isn't plain `X.Y.Z` (no leading `v`), a version that isn't higher than the one in `Cargo.toml` on `origin/main`, a `v<version>` tag that already exists locally or on origin, or a latest CI run on `origin/main` that failed or hasn't finished.
+
 The tag starts the release workflow, generated by [`dist`](https://github.com/axodotdev/cargo-dist) from `dist-workspace.toml`. It builds the Linux and macOS binaries and the installer, checks that the Linux binary starts on Rocky Linux 9 and in WSL2 with Ubuntu 24.04, and only then publishes the GitHub Release. It then publishes the same version to crates.io, skipping it if it is already there, and finally replaces the Release's body with notes generated from the merged pull requests.
 
 ## Credits and license

scripts/release.sh

@@ -14,8 +14,14 @@
 # release-summary.md beside this script. If it fails, the summary is GitHub's
 # generated notes instead, and the script warns and carries on.
 #
+# Before it pushes anything, it refuses a release that can't be cut: a version
+# that isn't plain X.Y.Z, or isn't higher than the one on main, a v<version>
+# tag that already exists locally or on origin, or a latest CI run on main that
+# didn't succeed.
+#
 # Prints a progress line on stderr for each step. Exits 0 once the tag is
-# pushed; if the PR's checks fail, it leaves the PR open and exits 1.
+# pushed. It exits 1 if it refuses, or if the PR's checks fail, in which case it
+# leaves the PR open.
 set -euo pipefail
 # So a failure inside $(…) stops the script too.
 shopt -s inherit_errexit
@@ -37,6 +43,7 @@ main() {
 
 	fetch_main
 	base=$(git rev-parse --verify 'origin/main^{commit}')
+	check_can_release "$version" "$tag" "$base"
 	repo=$PWD
 	work=$(mktemp -d)
 	trap 'git -C "$repo" worktree remove --force "$work" 2>/dev/null || rm -rf "$work"' EXIT
@@ -77,24 +84,89 @@ progress() {
 	echo "release: $*" >&2
 }
 
+# refuse <reason>
+refuse() {
+	progress "refusing to release $version: $*"
+	exit 1
+}
+
+# check_can_release <version> <tag> <base>
+# Refuses unless <version> is plain X.Y.Z and higher than the version in the
+# manifest at <base>, main's tip, no <tag> exists on origin or locally, and
+# the latest CI run on main is on <base> and succeeded.
+check_can_release() {
+	local current short=${3:0:7} latest sha status conclusion
+	if ! [[ $1 =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
+		refuse "it is not a plain X.Y.Z version, such as 0.4.0"
+	fi
+	current=$(git show "$3:Cargo.toml" | package_field version)
+	if ! is_higher "$1" "$current"; then
+		refuse "it is not higher than $current on main"
+	fi
+
+	# Origin first: fetching main brings in the tags on it, so a tag found
+	# locally may have come from origin.
+	if [ -n "$(git ls-remote --tags origin "refs/tags/$2")" ]; then
+		refuse "$2 already exists on origin"
+	fi
+	if git rev-parse --quiet --verify "refs/tags/$2" >/dev/null; then
+		refuse "$2 already exists locally"
+	fi
+
+	latest=$(gh run list --branch main --workflow ci.yml --event push --limit 1 \
+		--json headSha,status,conclusion --jq '.[] | "\(.headSha) \(.status) \(.conclusion)"')
+	read -r sha status conclusion <<<"$latest"
+	if [ "$sha" != "$3" ]; then
+		refuse "CI has not run on main at $short yet"
+	elif [ "$status" != completed ]; then
+		refuse "CI on main at $short is $status, not finished"
+	elif [ "$conclusion" != success ]; then
+		refuse "CI on main at $short ended in $conclusion"
+	fi
+}
+
+# is_higher <version> <other>
+# Whether plain semver <version> is higher than <other>, part by part.
+is_higher() {
+	local -a ours theirs
+	local i
+	IFS=. read -ra ours <<<"$1"
+	IFS=. read -ra theirs <<<"$2"
+	for i in 0 1 2; do
+		if ((ours[i] != theirs[i])); then
+			((ours[i] > theirs[i]))
+			return
+		fi
+	done
+	return 1
+}
+
 fetch_main() {
 	git fetch --quiet origin +refs/heads/main:refs/remotes/origin/main
 }
 
+# package_field <key>
+# The value of <key> in the [package] section of the manifest on stdin.
+package_field() {
+	awk -F '"' -v key="$1" '
+		/^\[/ { section = $0 }
+		section == "[package]" && $1 == key " = " { print $2; exit }'
+}
+
 # bump_version <version>
 # Sets the package's version in Cargo.toml, and its entry's in Cargo.lock, to
 # <version>, leaving every other line alone.
 # shellcheck disable=SC2016 # the $0 in the awk programs is awk's
 bump_version() {
 	local name
-	name=$(awk '/^\[/ { section = $0 } section == "[package]" && /^name = / { print; exit }' Cargo.toml)
+	name=$(package_field name <Cargo.toml)
 	rewrite Cargo.toml -v version="$1" '
 		/^\[/ { section = $0 }
 		section == "[package]" && /^version = / && !done { $0 = "version = \"" version "\""; done = 1 }
 		{ print }'
 	rewrite Cargo.lock -v version="$1" -v name="$name" '
 		/^\[\[package\]\]/ { ours = 0 }
-		$0 == name { ours = 1 }
+		$0 == "name = \"" name "\"" { ours = 1 }
 		ours && /^version = / { $0 = "version = \"" version "\""; ours = 0 }
 		{ print }'
 }

tests/fakes/gh

@@ -11,6 +11,7 @@ State lives in the JSON file named by $FAKE_GH_STATE:
               "mergeable"?, "unknown_polls"?}],
      "checks": {"<sha>": [{"name", "conclusion", "url", "pending_polls"?}]},
      "statuses": {"<sha>": [{"context", "state", "url"}]},
+     "runs"?: [{"branch", "workflow", "event", "headSha", "status", "conclusion"}],
      "on_ci_read"?: {"times", "script", "seen"},
      "on_merge"?: {"times", "script"},
      "refuse_merges"?: {"times", "error"},
@@ -43,6 +44,10 @@ generated notes: a line for each merged PR whose merge commit is reachable from
 `gh api repos/<repo>/actions/runs?head_sha=<sha>` answers with the workflow
 runs on <sha>: its check runs, read as the check-runs endpoint reads them.
 
+`gh run list` answers with the workflow runs in `runs`, newest first, that
+match its `--branch`, `--workflow` and `--event`. Runs there are only ones a
+test records with `gh fake run`: they are separate from the check runs.
+
 `gh api graphql` answers the one query thirdshift reads a Spec's Tickets
 with: the sub-issues of issue `$number` (its `sub_issues`, in order), each
 with its state, its `labels`, how many sub-issues it has of its own, and the
@@ -60,6 +65,8 @@ command):
 
     gh fake checks <sha> '<JSON list>'      set the check runs on <sha>
     gh fake statuses <sha> '<JSON list>'    set the commit statuses on <sha>
+    gh fake run '<JSON>'                    record a workflow run, newer than
+                                            any recorded before it
     gh fake pr <head> <field> '<JSON>'      set a field of <head>'s newest PR
     gh fake issue <number> <state>          set issue <number> OPEN or CLOSED
     gh fake on-ci-read <times> '<script>'   run <script> in bash the first time
@@ -445,6 +452,26 @@ def generate_notes(state, args):
     print(f"\n**Full Changelog**: https://github.com/{state['repo']}/compare/{compare}")
 
 
+LATEST_RUN_LINE = '.[] | "\\(.headSha) \\(.status) \\(.conclusion)"'
+
+
+def run_list(state, flags):
+    """Only the flags the release script reads the latest CI run on a branch
+    with are supported, and only its one `--jq`."""
+    unsupported = set(flags) - {"branch", "workflow", "event", "limit", "json", "jq"}
+    if unsupported:
+        die(f"fake gh: unsupported run list flags {sorted(unsupported)}", 2)
+    if flags.get("jq") != LATEST_RUN_LINE:
+        die(f"fake gh: unsupported --jq {flags.get('jq')}", 2)
+    runs = [
+        run
+        for run in reversed(state.get("runs", []))
+        if all(run[field] == flags[field] for field in ("branch", "workflow", "event") if field in flags)
+    ][: int(flags.get("limit", 20))]
+    for run in runs:
+        print(f"{run['headSha']} {run['status']} {run['conclusion']}")
+
+
 def graphql(state, args):
     """The Tickets query: `-f query=...` plus `-f`/`-F` variables."""
     variables = api_fields(args)
@@ -502,6 +529,8 @@ def run_hook(state, name, **env):
 def fake(state, args):
     if args[0] in ("checks", "statuses"):
         state.setdefault(args[0], {})[args[1]] = json.loads(args[2])
+    elif args[0] == "run":
+        state.setdefault("runs", []).append(json.loads(args[1]))
     elif args[0] == "on-ci-read":
         state["on_ci_read"] = {"times": int(args[1]), "script": args[2], "seen": []}
     elif args[0] == "on-merge":
@@ -562,6 +591,10 @@ def main():
         positional, flags = parse(args[2:])
         check_repo(state, flags)
         issue_close(state, positional, flags)
+    elif args[:2] == ["run", "list"]:
+        positional, flags = parse(args[2:])
+        check_repo(state, flags)
+        run_list(state, flags)
     elif args[:2] == ["api", "graphql"]:
         graphql(state, args[2:])
     elif args[0] == "api" and GENERATE_NOTES.fullmatch(args[1]):

tests/release.rs

@@ -59,9 +59,10 @@ fn with_version(file: &str, version: &str) -> String {
 }
 
 /// A bare origin whose main is at version 0.1.0, tagged `v0.1.0`, with one PR
-/// merged before the tag and two after; a contributor's clone that makes
-/// them; the maintainer's clone the script runs in; the fake `gh`'s state;
-/// and a fake `claude` that prints `AGENT_SUMMARY`.
+/// merged before the tag and two after, and a green CI run on main's tip; a
+/// contributor's clone that makes them; the maintainer's clone the script
+/// runs in; the fake `gh`'s state; and a fake `claude` that prints
+/// `AGENT_SUMMARY`.
 struct Release {
     temp: TempDir,
 }
@@ -115,6 +116,7 @@ impl Release {
         release.agent_runs(&format!("printf '%s\\n' '{AGENT_SUMMARY}'"));
 
         git(root, &["clone", "-q", "origin.git", "maintainer"]);
+        release.main_ci("completed", "success");
         release
     }
 
@@ -181,6 +183,25 @@ impl Release {
         ]);
     }
 
+    /// Records a CI run on main's tip on origin, newer than any before it.
+    fn main_ci(&self, status: &str, conclusion: &str) {
+        let run = json!({
+            "branch": "main",
+            "workflow": "ci.yml",
+            "event": "push",
+            "headSha": self.origin(&["rev-parse", "main"]),
+            "status": status,
+            "conclusion": conclusion,
+        });
+        self.gh(&["fake", "run", &run.to_string()]);
+    }
+
+    /// Every ref on origin with the commit it points at, to show a refusal
+    /// pushed nothing.
+    fn origin_refs(&self) -> String {
+        self.origin(&["for-each-ref", "--format=%(refname) %(objectname)"])
+    }
+
     /// Makes the fake `claude` run the bash `script`.
     fn agent_runs(&self, script: &str) {
         fs::write(self.root().join("claude-script.sh"), script).unwrap();
@@ -546,3 +567,129 @@ fn a_failure_to_generate_the_notes_stops_before_the_pr_is_opened() {
     );
     assert_eq!(release.origin_tag("v0.2.0"), None);
 }
+
+/// Runs the script with `version` and asserts it refused before pushing
+/// anything, with a message on stderr containing each of `reasons`.
+fn assert_refused(release: &Release, version: &str, reasons: &[&str]) {
+    let refs_before = release.origin_refs();
+    let prs_before = release.gh_state()["prs"].clone();
+
+    let output = release.run_script(version);
+
+    assert!(!output.status.success(), "{version} was not refused");
+    let err = stderr(&output);
+    assert!(err.contains("refusing to release"), "{version}: {err}");
+    for reason in reasons {
+        assert!(err.contains(reason), "{version}: no {reason:?} in {err}");
+    }
+    assert_eq!(release.origin_refs(), refs_before, "{version}");
+    assert_eq!(release.gh_state()["prs"], prs_before, "{version}");
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_version_that_is_not_plain_semver_is_refused() {
+    let release = Release::new();
+
+    for version in [
+        "v0.4.0",
+        "0.4",
+        "abc",
+        "0.4.0-rc.1",
+        "01.4.0",
+        "0.4.0.1",
+        "",
+    ] {
+        assert_refused(&release, version, &["not a plain X.Y.Z version"]);
+    }
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_version_not_higher_than_the_one_on_main_is_refused() {
+    let release = Release::new();
+
+    for version in ["0.1.0", "0.0.9", "0.0.10"] {
+        assert_refused(&release, version, &["not higher than 0.1.0 on main"]);
+    }
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn versions_are_compared_as_numbers_not_text() {
+    let release = Release::new();
+    release.ci_reports(GREEN);
+
+    let output = release.run_script("0.10.0");
+
+    assert!(output.status.success(), "{}", stderr(&output));
+    assert!(release.origin_tag("v0.10.0").is_some());
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_tag_that_exists_locally_is_refused() {
+    let release = Release::new();
+    git(&release.maintainer(), &["tag", "v0.2.0"]);
+
+    assert_refused(&release, "0.2.0", &["v0.2.0 already exists locally"]);
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_tag_that_exists_on_origin_is_refused() {
+    let release = Release::new();
+    // On a commit main never reaches, so fetching main doesn't bring it in.
+    let contributor = release.contributor();
+    git(
+        &contributor,
+        &["checkout", "-q", "-b", "stray", "origin/main"],
+    );
+    git(
+        &contributor,
+        &["commit", "-q", "--allow-empty", "-m", "Stray"],
+    );
+    git(&contributor, &["tag", "v0.2.0"]);
+    git(&contributor, &["push", "-q", "origin", "v0.2.0"]);
+
+    assert_refused(&release, "0.2.0", &["v0.2.0 already exists on origin"]);
+    assert!(git(&release.maintainer(), &["tag", "--list", "v0.2.0"]).is_empty());
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_tag_on_main_that_exists_on_origin_is_refused_as_on_origin() {
+    let release = Release::new();
+    let contributor = release.contributor();
+    git(&contributor, &["tag", "v0.2.0", "origin/main"]);
+    git(&contributor, &["push", "-q", "origin", "v0.2.0"]);
+
+    assert_refused(&release, "0.2.0", &["v0.2.0 already exists on origin"]);
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_red_ci_run_on_main_is_refused() {
+    let release = Release::new();
+    release.main_ci("completed", "failure");
+
+    assert_refused(&release, "0.2.0", &["CI on main", "failure"]);
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn an_unfinished_ci_run_on_main_is_refused() {
+    let release = Release::new();
+    release.main_ci("in_progress", "");
+
+    assert_refused(&release, "0.2.0", &["CI on main", "in_progress"]);
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn a_main_whose_tip_ci_has_not_run_on_is_refused() {
+    let release = Release::new();
+    release.merge_pr("untested", "Not yet built", "");
+
+    assert_refused(&release, "0.2.0", &["CI has not run on main at"]);
+}