Balance coverage shards on measured test time

scripts/ci-test-shard.py

@@ -12,12 +12,17 @@
 
 Windows debug CI sets CI_TEST_TIMINGS to the checked-in runtime sample.
 Those shards balance measured test time plus linking work, with shard 1's
-unit tests reserved up front. Other runners retain the count-based split.
+unit tests reserved up front. The Linux `coverage` shards use the same
+sample by default (see CI_TEST_COVERAGE below). Other runners retain the
+count-based split.
 
 CI_TEST_COVERAGE=1 runs the shard under cargo-llvm-cov for ci.yml's
 `coverage` job: `cargo llvm-cov --no-report` in place of `cargo test`, and
 no doctests, which cargo-llvm-cov leaves out without `--doctests`. A later
-`cargo llvm-cov report` exports the shard's profiles.
+`cargo llvm-cov report` exports the shard's profiles. Coverage shards
+balance on scripts/ci-test-durations.json unless CI_TEST_TIMINGS is set (an
+empty value selects the count split): under the count split shard 2 ran well
+past shard 1 and set the end of every CI run.
 
 Extra arguments after `SHARD COUNT` go to every `cargo` invocation.
 Each invocation runs with `--no-fail-fast`; the exit status is non-zero if
@@ -33,6 +38,8 @@
 import subprocess
 import sys
 
+DEFAULT_COVERAGE_TIMINGS = Path(__file__).resolve().parent / "ci-test-durations.json"
+
 # Shard 1's share of the integration targets, relative to every other
 # shard's 1.0 (it also carries the unit tests and doctests).
 FIRST_SHARD_WEIGHT = 0.5
@@ -99,9 +106,9 @@ def main(argv):
         ["cargo", "metadata", "--no-deps", "--format-version", "1"],
         check=True, capture_output=True, text=True).stdout)
     names = integration_targets(metadata)
-    timing_path = os.environ.get("CI_TEST_TIMINGS")
-    timings = json.loads(Path(timing_path).read_text(encoding="utf-8")) if timing_path else None
     coverage = os.environ.get("CI_TEST_COVERAGE") == "1"
+    timing_path = os.environ.get("CI_TEST_TIMINGS", DEFAULT_COVERAGE_TIMINGS if coverage else None)
+    timings = json.loads(Path(timing_path).read_text(encoding="utf-8")) if timing_path else None
     print(f"ci-test-shard: shard {shard}/{count}: "
           f"{len(partition(names, count, timings)[shard - 1])} of {len(names)} integration targets", flush=True)
     status = 0

scripts/tests/test_ci_test_shard.py

@@ -96,6 +96,28 @@ def test_coverage_mode_comes_from_the_environment(self):
         self.assertEqual(run.call_count, 2)
         self.assertEqual(run.call_args_list[-1].args[0][:3], ["cargo", "llvm-cov", "--no-report"])
 
+    def test_coverage_shards_default_to_the_timing_sample(self):
+        timings = json.loads(shard.DEFAULT_COVERAGE_TIMINGS.read_text())
+        names = sorted(timings["targets"])
+        timed, counted = shard.partition(names, 2, timings), shard.partition(names, 2)
+        self.assertNotEqual(timed, counted)
+        metadata = {"workspace_members": ["a"], "packages": [
+            {"id": "a", "targets": [{"name": n, "kind": ["test"]} for n in names]}]}
+        env = {k: v for k, v in shard.os.environ.items() if k != "CI_TEST_TIMINGS"}
+        cases = (({"CI_TEST_COVERAGE": "1"}, timed),
+                 ({"CI_TEST_COVERAGE": "1", "CI_TEST_TIMINGS": ""}, counted),
+                 ({}, counted))
+        for extra, expected in cases:
+            for k in (1, 2):
+                with self.subTest(env=extra, shard=k), \
+                        patch.dict(shard.os.environ, {**env, **extra}, clear=True), \
+                        patch.object(shard.subprocess, "run", side_effect=[
+                            subprocess.CompletedProcess([], 0, stdout=json.dumps(metadata)),
+                        ] + [subprocess.CompletedProcess([], 0)] * 2) as run:
+                    self.assertEqual(shard.main([str(k), "2"]), 0)
+                    self.assertCountEqual(selected([c.args[0] for c in run.call_args_list[1:]]),
+                                          expected[k - 1])
+
     def test_negative_bad_shard(self):
         with self.assertRaises(ValueError):
             shard.invocations(3, 2, self.NAMES)