diff --git a/scripts/ci-test-shard.py b/scripts/ci-test-shard.py index 079861c6b..0142bce5a 100644 --- a/scripts/ci-test-shard.py +++ b/scripts/ci-test-shard.py @@ -12,12 +12,17 @@ Windows debug CI sets CI_TEST_TIMINGS to the checked-in runtime sample. Those shards balance measured test time plus linking work, with shard 1's -unit tests reserved up front. Other runners retain the count-based split. +unit tests reserved up front. The Linux `coverage` shards use the same +sample by default (see CI_TEST_COVERAGE below). Other runners retain the +count-based split. CI_TEST_COVERAGE=1 runs the shard under cargo-llvm-cov for ci.yml's `coverage` job: `cargo llvm-cov --no-report` in place of `cargo test`, and no doctests, which cargo-llvm-cov leaves out without `--doctests`. A later -`cargo llvm-cov report` exports the shard's profiles. +`cargo llvm-cov report` exports the shard's profiles. Coverage shards +balance on scripts/ci-test-durations.json unless CI_TEST_TIMINGS is set (an +empty value selects the count split): under the count split shard 2 ran well +past shard 1 and set the end of every CI run. Extra arguments after `SHARD COUNT` go to every `cargo` invocation. Each invocation runs with `--no-fail-fast`; the exit status is non-zero if @@ -33,6 +38,8 @@ import subprocess import sys +DEFAULT_COVERAGE_TIMINGS = Path(__file__).resolve().parent / "ci-test-durations.json" + # Shard 1's share of the integration targets, relative to every other # shard's 1.0 (it also carries the unit tests and doctests). FIRST_SHARD_WEIGHT = 0.5 @@ -99,9 +106,9 @@ def main(argv): ["cargo", "metadata", "--no-deps", "--format-version", "1"], check=True, capture_output=True, text=True).stdout) names = integration_targets(metadata) - timing_path = os.environ.get("CI_TEST_TIMINGS") - timings = json.loads(Path(timing_path).read_text(encoding="utf-8")) if timing_path else None coverage = os.environ.get("CI_TEST_COVERAGE") == "1" + timing_path = os.environ.get("CI_TEST_TIMINGS", DEFAULT_COVERAGE_TIMINGS if coverage else None) + timings = json.loads(Path(timing_path).read_text(encoding="utf-8")) if timing_path else None print(f"ci-test-shard: shard {shard}/{count}: " f"{len(partition(names, count, timings)[shard - 1])} of {len(names)} integration targets", flush=True) status = 0 diff --git a/scripts/tests/test_ci_test_shard.py b/scripts/tests/test_ci_test_shard.py index 6ffd95ffa..830d1f04f 100644 --- a/scripts/tests/test_ci_test_shard.py +++ b/scripts/tests/test_ci_test_shard.py @@ -96,6 +96,28 @@ def test_coverage_mode_comes_from_the_environment(self): self.assertEqual(run.call_count, 2) self.assertEqual(run.call_args_list[-1].args[0][:3], ["cargo", "llvm-cov", "--no-report"]) + def test_coverage_shards_default_to_the_timing_sample(self): + timings = json.loads(shard.DEFAULT_COVERAGE_TIMINGS.read_text()) + names = sorted(timings["targets"]) + timed, counted = shard.partition(names, 2, timings), shard.partition(names, 2) + self.assertNotEqual(timed, counted) + metadata = {"workspace_members": ["a"], "packages": [ + {"id": "a", "targets": [{"name": n, "kind": ["test"]} for n in names]}]} + env = {k: v for k, v in shard.os.environ.items() if k != "CI_TEST_TIMINGS"} + cases = (({"CI_TEST_COVERAGE": "1"}, timed), + ({"CI_TEST_COVERAGE": "1", "CI_TEST_TIMINGS": ""}, counted), + ({}, counted)) + for extra, expected in cases: + for k in (1, 2): + with self.subTest(env=extra, shard=k), \ + patch.dict(shard.os.environ, {**env, **extra}, clear=True), \ + patch.object(shard.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0, stdout=json.dumps(metadata)), + ] + [subprocess.CompletedProcess([], 0)] * 2) as run: + self.assertEqual(shard.main([str(k), "2"]), 0) + self.assertCountEqual(selected([c.args[0] for c in run.call_args_list[1:]]), + expected[k - 1]) + def test_negative_bad_shard(self): with self.assertRaises(ValueError): shard.invocations(3, 2, self.NAMES)