diff --git a/.github/workflows/pr-bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml index 3cb60c619bf..98038eaa2bf 100644 --- a/.github/workflows/pr-bench-dispatch.yml +++ b/.github/workflows/pr-bench-dispatch.yml @@ -151,3 +151,21 @@ jobs: secrets: inherit with: matrix_preset: "pr-compact" + + remove-sql-extended-label: + runs-on: ubuntu-latest + timeout-minutes: 10 + if: github.event.label.name == 'action/bench-sql-extended' + steps: + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' + with: + labels: action/bench-sql-extended + fail_on_error: true + + sql-extended-bench: + needs: remove-sql-extended-label + uses: ./.github/workflows/pr-bench-sql.yml + secrets: inherit + with: + matrix_preset: "pr-full" diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py index 06ad8ea4d1b..8457f30e06f 100644 --- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py @@ -13,10 +13,10 @@ PRESETS = { "develop": "Every regular SQL benchmark at full target coverage.", - "pr": "The quicker pull-request SQL benchmark matrix.", + "pr": "The quicker pull-request SQL benchmark matrix (action/bench-sql).", "pr-compact": "Pull-request SQL benchmarks for Vortex Compact plus Parquet controls.", - "pr-all": "The union of the focused PR and PR Compact benchmark matrices.", - "pr-full": "Every regular SQL benchmark at full PR target coverage.", + "pr-all": "The PR and PR Compact benchmark matrices plus extended-only default targets (action/bench-all).", + "pr-full": "Every regular SQL benchmark at full PR target coverage (action/bench-sql-extended).", "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", } @@ -92,7 +92,6 @@ benchmark=Benchmark.CLICKBENCH_SORTED, name="Clickbench Sorted on NVME", runs={ - "pr": DEFAULT, "pr-compact": COMPACT, "pr-all": STANDARD, "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, @@ -197,7 +196,6 @@ scale_factor=100, local_dir="vortex-bench/data/statpopgen", runs={ - "pr": DEFAULT, "pr-compact": COMPACT, "pr-all": STANDARD, "pr-full": STANDARD, @@ -226,7 +224,6 @@ local_dir="vortex-bench/data/fineweb", remote_key="fineweb", runs={ - "pr": DEFAULT, "pr-compact": COMPACT, "pr-all": STANDARD, "pr-full": STANDARD, diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index 38d3e162611..c4a405f529a 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -42,7 +42,15 @@ "pr": tuple( benchmark_id for benchmark_id in REGULAR_IDS - if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"} + if benchmark_id + not in { + "clickbench-sorted-nvme", + "tpch-s3-10", + "statpopgen", + "fineweb-s3", + "appian-nvme", + "vortex-queries", + } ), "pr-compact": COMPACT_IDS, "pr-all": PR_ALL_IDS, @@ -84,7 +92,7 @@ def test_pr_target_selection() -> None: ("duckdb", "vortex"), } assert ("datafusion", "lance") in _targets(develop["tpch-nvme"]) - for preset in (pr, pr_full, develop): + for preset in (pr_full, develop): assert {engine for engine, _format in _targets(preset["statpopgen"])} == {"datafusion", "duckdb"} assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values()) assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"]) @@ -94,22 +102,27 @@ def test_pr_target_selection() -> None: assert set(cast("list[str]", entry["data_formats"])) == {"parquet", "vortex-compact"} -def test_pr_all_is_union_of_focused_presets() -> None: +def test_pr_all_covers_focused_presets() -> None: pr = {entry["id"]: entry for entry in _entries("pr")} pr_compact = {entry["id"]: entry for entry in _entries("pr-compact")} pr_all = {entry["id"]: entry for entry in _entries("pr-all")} assert set(pr_all) == set(pr) | set(pr_compact) for benchmark_id, entry in pr_all.items(): - expected_targets: set[tuple[str, str]] = set() - expected_formats: set[str] = set() for preset in (pr, pr_compact): if source := preset.get(benchmark_id): - expected_targets |= _targets(source) - expected_formats |= set(cast("list[str]", source["data_formats"])) + assert _targets(source) <= _targets(entry) + assert set(cast("list[str]", source["data_formats"])) <= set(cast("list[str]", entry["data_formats"])) - assert _targets(entry) == expected_targets - assert set(cast("list[str]", entry["data_formats"])) == expected_formats + +def test_pr_full_covers_pr() -> None: + pr = {entry["id"]: entry for entry in _entries("pr")} + pr_full = {entry["id"]: entry for entry in _entries("pr-full")} + + for benchmark_id, entry in pr.items(): + full_entry = pr_full[benchmark_id] + assert _targets(entry) <= _targets(full_entry) + assert set(cast("list[str]", entry["data_formats"])) <= set(cast("list[str]", full_entry["data_formats"])) def test_resolver_rejects_empty_targets() -> None: diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index ef6f8127ca1..1755a1c6cf9 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -261,13 +261,18 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman - **String encoding** -- `action/bench-string` runs only the string encoding benchmark. - **GPU compression** -- `action/bench-gpu-compress` runs the allow-listed Vortex decompression cases on a GPU runner. -- **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`. +- **SQL** -- `action/bench-sql` runs the `pr` preset, a quick core subset of the SQL benchmarks + that excludes `vortex-compact`, Clickbench Sorted, Appian, statpopgen, FineWeb on S3, TPC-H SF=10 + on S3, and Vortex queries. +- **SQL Extended** -- `action/bench-sql-extended` runs the `pr-full` preset: every regular SQL + benchmark, including `vortex-compact` and DuckDB-format targets. - **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which benchmarks `vortex-compact` plus Parquet control rows used to distinguish code changes from runner drift. - **All CPU benchmarks** -- `action/bench-all` runs random access, compression, string encoding, - and the `pr-all` SQL preset, which combines the `pr` and `pr-compact` coverage without - repeating shared jobs. Do not combine it with other benchmark labels; GPU compression is the - only exception. + and the `pr-all` SQL preset, which combines the `pr` and `pr-compact` coverage, plus the + default targets of Clickbench Sorted, statpopgen, and FineWeb on S3, without repeating shared + jobs. Do not combine it with other benchmark labels; + GPU compression is the only exception. All CI benchmarks run on dedicated instances with the `release_debug` profile and `-C target-cpu=native` to produce representative numbers.