From 2a84bce47f49a2bbce49b82e9b0c7e681027e75b Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 16:39:28 +0000 Subject: [PATCH 1/2] Choose the scan version per benchmark in the bench orchestrator The duckdb-bench and datafusion-bench binaries no longer pick the V2 scan from the benchmark name or VORTEX_SCAN_V2. `--scan-v2` now defaults to false, and the bench orchestrator passes `--scan-v2 true` for TPC-H, TPC-DS and ClickBench and `--scan-v2 false` for every other benchmark. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBB3qKAchqvKuep7iL2 Signed-off-by: Joe Isaacs --- .../bench_orchestrator/config.py | 4 ++++ .../bench_orchestrator/runner/executor.py | 3 ++- bench-orchestrator/tests/test_executor.py | 20 +++++++++++++++++++ benchmarks/datafusion-bench/src/main.rs | 13 ++++-------- benchmarks/duckdb-bench/src/main.rs | 14 ++++--------- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 5d51fb8ecc3..6d57cefa20b 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -92,6 +92,10 @@ class Benchmark(Enum): } +# Benchmarks that run their Vortex scans on the planner-driven V2 scan. The others use V1. +SCAN_V2_BENCHMARKS: frozenset[Benchmark] = frozenset({Benchmark.TPCH, Benchmark.TPCDS, Benchmark.CLICKBENCH}) + + def engines_for_benchmark(benchmark: Benchmark) -> frozenset[Engine]: """Return the engines `benchmark` supports, defaulting to every engine when unrestricted.""" return BENCHMARK_ENGINES.get(benchmark, frozenset(Engine)) diff --git a/bench-orchestrator/bench_orchestrator/runner/executor.py b/bench-orchestrator/bench_orchestrator/runner/executor.py index b7afee7e948..74cd648b63d 100644 --- a/bench-orchestrator/bench_orchestrator/runner/executor.py +++ b/bench-orchestrator/bench_orchestrator/runner/executor.py @@ -14,7 +14,7 @@ from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from ..config import Benchmark, Engine, Format +from ..config import SCAN_V2_BENCHMARKS, Benchmark, Engine, Format console = Console() @@ -55,6 +55,7 @@ def build_command( if self.backend in {Engine.DATAFUSION, Engine.DUCKDB}: cmd.extend(["--formats", ",".join(fmt.value for fmt in formats)]) + cmd.extend(["--scan-v2", str(benchmark in SCAN_V2_BENCHMARKS).lower()]) if self.backend == Engine.DUCKDB: cmd.append("--delete-duckdb-database") diff --git a/bench-orchestrator/tests/test_executor.py b/bench-orchestrator/tests/test_executor.py index b8afb066e75..8cdf198c1a5 100644 --- a/bench-orchestrator/tests/test_executor.py +++ b/bench-orchestrator/tests/test_executor.py @@ -5,6 +5,7 @@ import textwrap from pathlib import Path +import pytest from bench_orchestrator.config import Benchmark, Engine, Format from bench_orchestrator.runner.executor import BenchmarkExecutor @@ -46,6 +47,24 @@ def test_build_command_serializes_vortex_spatial_native_format() -> None: assert cmd[cmd.index("--formats") + 1] == "parquet,vortex,vortex-spatial-native" +@pytest.mark.parametrize("engine", [Engine.DATAFUSION, Engine.DUCKDB]) +@pytest.mark.parametrize( + ("benchmark", "scan_v2"), + [ + (Benchmark.TPCH, "true"), + (Benchmark.TPCDS, "true"), + (Benchmark.CLICKBENCH, "true"), + (Benchmark.STATPOPGEN, "false"), + ], +) +def test_build_command_selects_scan_version(engine: Engine, benchmark: Benchmark, scan_v2: str) -> None: + executor = BenchmarkExecutor(Path("/tmp/bench"), engine) + + cmd = executor.build_command(benchmark=benchmark, formats=[Format.VORTEX]) + + assert cmd[cmd.index("--scan-v2") + 1] == scan_v2 + + def test_build_command_omits_formats_for_lance_backend() -> None: executor = BenchmarkExecutor(Path("/tmp/lance-bench"), Engine.LANCE) @@ -57,6 +76,7 @@ def test_build_command_omits_formats_for_lance_backend() -> None: assert cmd[0] == "/tmp/lance-bench" assert "--formats" not in cmd + assert "--scan-v2" not in cmd assert cmd[cmd.index("--queries") + 1] == "3" diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index 3d3983f43ee..765b34fafcf 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -120,9 +120,9 @@ struct Args { #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))] formats: Vec, - /// Use the planner-driven V2 Vortex scan (defaults to true for TPC-H, TPC-DS and ClickBench). - #[arg(long, action = clap::ArgAction::Set)] - scan_v2: Option, + /// Use the planner-driven V2 Vortex scan. + #[arg(long, default_value_t = false, action = clap::ArgAction::Set)] + scan_v2: bool, #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))] options: Vec, @@ -136,12 +136,7 @@ async fn main() -> anyhow::Result<()> { vortex_bench::configure_allocator(); let args = Args::parse(); let opts = Opts::from(args.options); - let scan_v2 = args.scan_v2.unwrap_or_else(|| { - matches!( - args.benchmark, - BenchmarkArg::TpcH | BenchmarkArg::TpcDS | BenchmarkArg::ClickBench - ) && !std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0") - }); + let scan_v2 = args.scan_v2; set_join_set_tracer(get_static_tracer())?; setup_logging_and_tracing(args.verbose, args.tracing)?; diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index 903cd958b54..5a9a0f2f4c9 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -103,9 +103,9 @@ struct Args { )] reuse: bool, - /// Use the planner-driven V2 Vortex scan (defaults to true for TPC-H, TPC-DS and ClickBench). - #[arg(long, action = clap::ArgAction::Set)] - scan_v2: Option, + /// Use the planner-driven V2 Vortex scan. + #[arg(long, default_value_t = false, action = clap::ArgAction::Set)] + scan_v2: bool, } #[global_allocator] @@ -116,13 +116,7 @@ fn main() -> anyhow::Result<()> { let args = Args::parse(); let opts = Opts::from(args.options); - let scan_v2 = args.scan_v2.unwrap_or_else(|| { - matches!( - args.benchmark, - BenchmarkArg::TpcH | BenchmarkArg::TpcDS | BenchmarkArg::ClickBench - ) && !std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0") - }); - vortex_duckdb::set_scan_v2(scan_v2); + vortex_duckdb::set_scan_v2(args.scan_v2); setup_logging_and_tracing(args.verbose, args.tracing)?; From af654ec92bc2b61323f4a6ce01feb19b0b7c3279 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 17:14:19 +0000 Subject: [PATCH 2/2] List the Vortex scan version of every benchmark Replace the set of V2 benchmarks with a map that gives every benchmark its scan version, checked by a test to cover every benchmark, and print the scan version when `vx-bench run` starts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBB3qKAchqvKuep7iL2 Signed-off-by: Joe Isaacs --- bench-orchestrator/bench_orchestrator/cli.py | 3 +++ .../bench_orchestrator/config.py | 19 ++++++++++++++++-- .../bench_orchestrator/runner/executor.py | 4 ++-- bench-orchestrator/tests/test_executor.py | 20 ++++++++----------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index 53952b02661..cd8654b9fe2 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -22,6 +22,7 @@ from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( + BENCHMARK_SCAN_V2, Benchmark, BenchmarkTarget, BuildConfig, @@ -399,6 +400,8 @@ def run( console.print(f"\n[bold]Running {benchmark.value} benchmark[/bold]") console.print(f" Targets: {', '.join(map(str, config.targets))}") console.print(f" Iterations: {iterations}") + if any(target.engine in {Engine.DATAFUSION, Engine.DUCKDB} for target in config.targets): + console.print(f" Vortex scan: {'V2' if BENCHMARK_SCAN_V2[benchmark] else 'V1'}") if label: console.print(f" Label: {label}") console.print() diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 6d57cefa20b..920efd4813b 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -92,8 +92,23 @@ class Benchmark(Enum): } -# Benchmarks that run their Vortex scans on the planner-driven V2 scan. The others use V1. -SCAN_V2_BENCHMARKS: frozenset[Benchmark] = frozenset({Benchmark.TPCH, Benchmark.TPCDS, Benchmark.CLICKBENCH}) +# Whether each benchmark runs its Vortex scans on the planner-driven V2 scan (`True`) or on V1. +# Every benchmark is listed so its scan version is explicit. +BENCHMARK_SCAN_V2: dict[Benchmark, bool] = { + Benchmark.APPIAN: False, + Benchmark.TPCH: True, + Benchmark.TPCDS: True, + Benchmark.CLICKBENCH: True, + Benchmark.CLICKBENCH_SORTED: False, + Benchmark.FINEWEB: False, + Benchmark.GHARCHIVE: False, + Benchmark.JOB: False, + Benchmark.POLARSIGNALS: False, + Benchmark.PUBLIC_BI: False, + Benchmark.STATPOPGEN: False, + Benchmark.SPATIALBENCH: False, + Benchmark.VORTEX_QUERIES: False, +} def engines_for_benchmark(benchmark: Benchmark) -> frozenset[Engine]: diff --git a/bench-orchestrator/bench_orchestrator/runner/executor.py b/bench-orchestrator/bench_orchestrator/runner/executor.py index 74cd648b63d..171a7c7479f 100644 --- a/bench-orchestrator/bench_orchestrator/runner/executor.py +++ b/bench-orchestrator/bench_orchestrator/runner/executor.py @@ -14,7 +14,7 @@ from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from ..config import SCAN_V2_BENCHMARKS, Benchmark, Engine, Format +from ..config import BENCHMARK_SCAN_V2, Benchmark, Engine, Format console = Console() @@ -55,7 +55,7 @@ def build_command( if self.backend in {Engine.DATAFUSION, Engine.DUCKDB}: cmd.extend(["--formats", ",".join(fmt.value for fmt in formats)]) - cmd.extend(["--scan-v2", str(benchmark in SCAN_V2_BENCHMARKS).lower()]) + cmd.extend(["--scan-v2", str(BENCHMARK_SCAN_V2[benchmark]).lower()]) if self.backend == Engine.DUCKDB: cmd.append("--delete-duckdb-database") diff --git a/bench-orchestrator/tests/test_executor.py b/bench-orchestrator/tests/test_executor.py index 8cdf198c1a5..0d5cb38a0f0 100644 --- a/bench-orchestrator/tests/test_executor.py +++ b/bench-orchestrator/tests/test_executor.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from bench_orchestrator.config import Benchmark, Engine, Format +from bench_orchestrator.config import BENCHMARK_SCAN_V2, Benchmark, Engine, Format from bench_orchestrator.runner.executor import BenchmarkExecutor @@ -47,22 +47,18 @@ def test_build_command_serializes_vortex_spatial_native_format() -> None: assert cmd[cmd.index("--formats") + 1] == "parquet,vortex,vortex-spatial-native" +def test_every_benchmark_has_a_scan_version() -> None: + assert set(BENCHMARK_SCAN_V2) == set(Benchmark) + + @pytest.mark.parametrize("engine", [Engine.DATAFUSION, Engine.DUCKDB]) -@pytest.mark.parametrize( - ("benchmark", "scan_v2"), - [ - (Benchmark.TPCH, "true"), - (Benchmark.TPCDS, "true"), - (Benchmark.CLICKBENCH, "true"), - (Benchmark.STATPOPGEN, "false"), - ], -) -def test_build_command_selects_scan_version(engine: Engine, benchmark: Benchmark, scan_v2: str) -> None: +@pytest.mark.parametrize("benchmark", list(Benchmark)) +def test_build_command_selects_scan_version(engine: Engine, benchmark: Benchmark) -> None: executor = BenchmarkExecutor(Path("/tmp/bench"), engine) cmd = executor.build_command(benchmark=benchmark, formats=[Format.VORTEX]) - assert cmd[cmd.index("--scan-v2") + 1] == scan_v2 + assert cmd[cmd.index("--scan-v2") + 1] == str(BENCHMARK_SCAN_V2[benchmark]).lower() def test_build_command_omits_formats_for_lance_backend() -> None: