From e561eacef20b348632f6ffea66c9f6716a63cb02 Mon Sep 17 00:00:00 2001 From: Joery van den Hoff <8142891+JoeryH@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:57:48 -0500 Subject: [PATCH 1/3] perf: compute using columns once per SELECT, not per item SqlToRel::sql_select_to_rex called LogicalPlan::using_columns() for every SELECT item, walking the whole input plan each time. Compute it once in prepare_select_exprs (and once for a wildcard's REPLACE items) and pass it down. Adds a nested wide SELECT case to the sql_planner benchmark. --- datafusion/core/benches/sql_planner.rs | 20 ++++++++++++++++++++ datafusion/sql/src/select.rs | 16 +++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rs index 0a9015b68fbba..6dd4e64c1f164 100644 --- a/datafusion/core/benches/sql_planner.rs +++ b/datafusion/core/benches/sql_planner.rs @@ -861,6 +861,26 @@ fn criterion_benchmark(c: &mut Criterion) { }); } + // Nested SELECTs that each name every column of the one below + { + let columns = |qualifier: &str| { + (0..700) + .map(|i| format!("{qualifier}.c{i} AS c{i}")) + .collect::>() + .join(", ") + }; + let mut query = format!("SELECT {} FROM t700", columns("t700")); + for level in 0..8 { + query = format!( + "SELECT {} FROM ({query}) l{level}", + columns(&format!("l{level}")) + ); + } + c.bench_function("logical_nested_select_700_columns_8_levels", |b| { + b.iter(|| logical_plan(&ctx, &rt, &query)) + }); + } + // --- Mixed: deep plan + wide expressions --- // This is the worst case for optimizer: many nodes AND many expressions. diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 59cef5df0e975..ae72b77cf99d6 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -1114,9 +1114,16 @@ impl SqlToRel<'_, S> { ) -> Result> { let mut prepared_select_exprs = Vec::with_capacity(projection.len()); let mut error_builder = DataFusionErrorBuilder::new(); + let using_columns = plan.using_columns()?; for expr in projection { - match self.sql_select_to_rex(expr, plan, empty_from, planner_context) { + match self.sql_select_to_rex( + expr, + plan, + &using_columns, + empty_from, + planner_context, + ) { Ok(expr) => prepared_select_exprs.push(expr), Err(err) => error_builder.add_error(err), } @@ -1129,6 +1136,7 @@ impl SqlToRel<'_, S> { &self, sql: SelectItem, plan: &LogicalPlan, + using_columns: &[HashSet], empty_from: bool, planner_context: &mut PlannerContext, ) -> Result { @@ -1138,7 +1146,7 @@ impl SqlToRel<'_, S> { let col = normalize_col_with_schemas_and_ambiguity_check( expr, &[&[plan.schema()]], - &plan.using_columns()?, + using_columns, )?; Ok(SelectExpr::Expression(col)) @@ -1149,7 +1157,7 @@ impl SqlToRel<'_, S> { let col = normalize_col_with_schemas_and_ambiguity_check( select_expr, &[&[plan.schema()]], - &plan.using_columns()?, + using_columns, )?; let name = self.ident_normalizer.normalize(alias); // avoiding adding an alias if the column name is the same. @@ -1241,6 +1249,7 @@ impl SqlToRel<'_, S> { rename: options.opt_rename, }; if let Some(replace) = options.opt_replace { + let using_columns = plan.using_columns()?; let replace_expr = replace .items .iter() @@ -1248,6 +1257,7 @@ impl SqlToRel<'_, S> { self.sql_select_to_rex( SelectItem::UnnamedExpr(item.expr.clone()), plan, + &using_columns, empty_from, planner_context, ) From 3e532c4da279844d6aac2b9adbadd1f8d064ecd7 Mon Sep 17 00:00:00 2001 From: Joery van den Hoff Date: Fri, 6 Nov 2026 09:47:02 -0600 Subject: [PATCH 2/3] Collect using columns lazily, on the first item that needs them Wildcard-only projections never normalize an expression, so they no longer walk the input plan for USING columns. REPLACE items share the same lazily collected set. Adds a width/depth matrix and a qualified-wildcard control to the nested SELECT benchmark, and an SLT for REPLACE over a USING join. Co-Authored-By: Claude Opus 5.5 --- datafusion/core/benches/sql_planner.rs | 37 +++++++++++++------ datafusion/sql/src/select.rs | 31 ++++++++++++---- .../sqllogictest/test_files/wildcard.slt | 20 ++++++++++ 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rs index 6dd4e64c1f164..5cf7801adf198 100644 --- a/datafusion/core/benches/sql_planner.rs +++ b/datafusion/core/benches/sql_planner.rs @@ -861,24 +861,39 @@ fn criterion_benchmark(c: &mut Criterion) { }); } - // Nested SELECTs that each name every column of the one below + // Nested SELECTs that each name every column of the one below, against + // the same nesting through qualified wildcards { - let columns = |qualifier: &str| { - (0..700) + let named_columns = |qualifier: &str, width: usize| { + (0..width) .map(|i| format!("{qualifier}.c{i} AS c{i}")) .collect::>() .join(", ") }; - let mut query = format!("SELECT {} FROM t700", columns("t700")); - for level in 0..8 { - query = format!( - "SELECT {} FROM ({query}) l{level}", - columns(&format!("l{level}")) + let nested_query = |depth: usize, projection: &dyn Fn(&str) -> String| { + let mut query = format!("SELECT {} FROM t700", projection("t700")); + for level in 0..depth { + query = format!( + "SELECT {} FROM ({query}) l{level}", + projection(&format!("l{level}")) + ); + } + query + }; + for (width, depth) in [(100, 1), (100, 8), (700, 1), (700, 8)] { + let query = nested_query(depth, &|qualifier| named_columns(qualifier, width)); + c.bench_function( + &format!("logical_nested_select_{width}_columns_{depth}_levels"), + |b| b.iter(|| logical_plan(&ctx, &rt, &query)), + ); + } + for depth in [1, 8] { + let query = nested_query(depth, &|qualifier| format!("{qualifier}.*")); + c.bench_function( + &format!("logical_nested_qualified_wildcard_700_columns_{depth}_levels"), + |b| b.iter(|| logical_plan(&ctx, &rt, &query)), ); } - c.bench_function("logical_nested_select_700_columns_8_levels", |b| { - b.iter(|| logical_plan(&ctx, &rt, &query)) - }); } // --- Mixed: deep plan + wide expressions --- diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index ae72b77cf99d6..118cef10b52ab 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -80,6 +80,21 @@ struct RewrittenUnnestExprGroups { expr_groups: Vec>, } +/// The USING-join column sets of a SELECT's input plan, collected on first +/// use: collecting walks the whole input plan, and wildcard-only projections +/// never need them. +#[derive(Default)] +struct LazyUsingColumns(Option>>); + +impl LazyUsingColumns { + fn get(&mut self, plan: &LogicalPlan) -> Result<&[HashSet]> { + if self.0.is_none() { + self.0 = Some(plan.using_columns()?); + } + Ok(self.0.as_deref().unwrap_or_default()) + } +} + fn flatten_expr_groups(expr_groups: Vec>) -> Vec { expr_groups.into_iter().flatten().collect() } @@ -1114,13 +1129,13 @@ impl SqlToRel<'_, S> { ) -> Result> { let mut prepared_select_exprs = Vec::with_capacity(projection.len()); let mut error_builder = DataFusionErrorBuilder::new(); - let using_columns = plan.using_columns()?; + let mut using_columns = LazyUsingColumns::default(); for expr in projection { match self.sql_select_to_rex( expr, plan, - &using_columns, + &mut using_columns, empty_from, planner_context, ) { @@ -1136,7 +1151,7 @@ impl SqlToRel<'_, S> { &self, sql: SelectItem, plan: &LogicalPlan, - using_columns: &[HashSet], + using_columns: &mut LazyUsingColumns, empty_from: bool, planner_context: &mut PlannerContext, ) -> Result { @@ -1146,7 +1161,7 @@ impl SqlToRel<'_, S> { let col = normalize_col_with_schemas_and_ambiguity_check( expr, &[&[plan.schema()]], - using_columns, + using_columns.get(plan)?, )?; Ok(SelectExpr::Expression(col)) @@ -1157,7 +1172,7 @@ impl SqlToRel<'_, S> { let col = normalize_col_with_schemas_and_ambiguity_check( select_expr, &[&[plan.schema()]], - using_columns, + using_columns.get(plan)?, )?; let name = self.ident_normalizer.normalize(alias); // avoiding adding an alias if the column name is the same. @@ -1178,6 +1193,7 @@ impl SqlToRel<'_, S> { } let planned_options = self.plan_wildcard_options( plan, + using_columns, empty_from, planner_context, options, @@ -1200,6 +1216,7 @@ impl SqlToRel<'_, S> { let qualifier = self.object_name_to_table_reference(object_name)?; let planned_options = self.plan_wildcard_options( plan, + using_columns, empty_from, planner_context, options, @@ -1237,6 +1254,7 @@ impl SqlToRel<'_, S> { fn plan_wildcard_options( &self, plan: &LogicalPlan, + using_columns: &mut LazyUsingColumns, empty_from: bool, planner_context: &mut PlannerContext, options: WildcardAdditionalOptions, @@ -1249,7 +1267,6 @@ impl SqlToRel<'_, S> { rename: options.opt_rename, }; if let Some(replace) = options.opt_replace { - let using_columns = plan.using_columns()?; let replace_expr = replace .items .iter() @@ -1257,7 +1274,7 @@ impl SqlToRel<'_, S> { self.sql_select_to_rex( SelectItem::UnnamedExpr(item.expr.clone()), plan, - &using_columns, + using_columns, empty_from, planner_context, ) diff --git a/datafusion/sqllogictest/test_files/wildcard.slt b/datafusion/sqllogictest/test_files/wildcard.slt index 7a84429b90a21..1bbcaf946bb19 100644 --- a/datafusion/sqllogictest/test_files/wildcard.slt +++ b/datafusion/sqllogictest/test_files/wildcard.slt @@ -146,3 +146,23 @@ select *, a as aka from t; statement count 0 drop table t; + +# REPLACE over a USING join: the replacements resolve the unqualified join key +# through the USING columns, and the key appears once +statement ok +create table using_l(k int, v int) as values (1, 10), (2, 20); + +statement ok +create table using_r(k int, w int) as values (1, 100), (2, 200); + +query III rowsort +select * replace (k + v as v, k * w as w) from using_l join using_r using (k); +---- +1 11 100 +2 22 400 + +statement count 0 +drop table using_l; + +statement count 0 +drop table using_r; From 5b406be62f0a480945d7fbd53ec31b8e5e4485e8 Mon Sep 17 00:00:00 2001 From: Joery van den Hoff Date: Wed, 18 Nov 2026 16:15:50 -0600 Subject: [PATCH 3/3] Move the nested SELECT benchmark to its own PR The cases now live in #27572, so the benchmark bot can run them against main before this change lands. Co-Authored-By: Claude Opus 5.5 --- datafusion/core/benches/sql_planner.rs | 35 -------------------------- 1 file changed, 35 deletions(-) diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rs index 5cf7801adf198..0a9015b68fbba 100644 --- a/datafusion/core/benches/sql_planner.rs +++ b/datafusion/core/benches/sql_planner.rs @@ -861,41 +861,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); } - // Nested SELECTs that each name every column of the one below, against - // the same nesting through qualified wildcards - { - let named_columns = |qualifier: &str, width: usize| { - (0..width) - .map(|i| format!("{qualifier}.c{i} AS c{i}")) - .collect::>() - .join(", ") - }; - let nested_query = |depth: usize, projection: &dyn Fn(&str) -> String| { - let mut query = format!("SELECT {} FROM t700", projection("t700")); - for level in 0..depth { - query = format!( - "SELECT {} FROM ({query}) l{level}", - projection(&format!("l{level}")) - ); - } - query - }; - for (width, depth) in [(100, 1), (100, 8), (700, 1), (700, 8)] { - let query = nested_query(depth, &|qualifier| named_columns(qualifier, width)); - c.bench_function( - &format!("logical_nested_select_{width}_columns_{depth}_levels"), - |b| b.iter(|| logical_plan(&ctx, &rt, &query)), - ); - } - for depth in [1, 8] { - let query = nested_query(depth, &|qualifier| format!("{qualifier}.*")); - c.bench_function( - &format!("logical_nested_qualified_wildcard_700_columns_{depth}_levels"), - |b| b.iter(|| logical_plan(&ctx, &rt, &query)), - ); - } - } - // --- Mixed: deep plan + wide expressions --- // This is the worst case for optimizer: many nodes AND many expressions.