From 3546d384c93530f5251326bc9b3fb2c2ced0ff44 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Thu, 5 Nov 2026 01:04:48 +0100 Subject: [PATCH 1/4] fix: wrong aggregate functional dependencies when a GROUP BY column is not determined by the key `aggregate_functional_dependencies` marked a key as `Single` on the aggregate output when another GROUP BY column was determined by no input dependency, and gave every re-expressed dependency all output columns as targets. Rules that trust these dependencies, such as `EliminateJoin`, returned wrong results. Closes #27112. --- .../common/src/functional_dependencies.rs | 83 ++++++++------ .../test_files/functional_dependencies.slt | 107 ++++++++++++++++++ 2 files changed, 158 insertions(+), 32 deletions(-) diff --git a/datafusion/common/src/functional_dependencies.rs b/datafusion/common/src/functional_dependencies.rs index e6b978fa078f2..812ffaa69d45c 100644 --- a/datafusion/common/src/functional_dependencies.rs +++ b/datafusion/common/src/functional_dependencies.rs @@ -453,31 +453,23 @@ pub fn aggregate_functional_dependencies( let mut aggregate_func_dependencies = vec![]; let aggr_fields = aggr_schema.fields(); // Association covers the whole table: - let target_indices = (0..aggr_schema.fields().len()).collect::>(); + let all_output_indices = (0..aggr_schema.fields().len()).collect::>(); // Get functional dependencies of the schema: let func_dependencies = aggr_input_schema.functional_dependencies(); // The loop below only re-expresses input dependencies. Skip it when the // input has none. The GROUP BY-key dependency below always runs. if !func_dependencies.is_empty() { - // Compare the columns covered by every dependency, whatever its NULL - // semantics: leaving out the ones that do not hold across NULLs would - // hide GROUP BY expressions this determinant does not cover and - // upgrade its mode to `Single` wrongly. - let existing_target_indices = get_target_indices_from_dependencies( - func_dependencies.iter(), - group_by_input_indices, - ); for FunctionalDependence { source_indices, nullable, null_equality, - mode, + mode: input_mode, .. } in &func_dependencies.deps { // Indices into the GROUP BY list for this determinant: - let mut new_source_indices = vec![]; - let mut new_source_input_indices = vec![]; + let mut source_output_indices = vec![]; + let mut grouped_source_indices = vec![]; for (idx, input_idx) in group_by_input_indices.iter().enumerate() { // When one of the input determinant columns is a GROUP BY // expression, add the index of the GROUP BY expression as a new @@ -486,42 +478,55 @@ pub fn aggregate_functional_dependencies( && source_indices.contains(input_idx) // A key column can be in the GROUP BY list more than one // time (`x, x AS k`). Count it one time. - && !new_source_input_indices.contains(&Some(*input_idx)) + && !grouped_source_indices.contains(&Some(*input_idx)) { - new_source_indices.push(idx); - new_source_input_indices.push(Some(*input_idx)); + source_output_indices.push(idx); + grouped_source_indices.push(Some(*input_idx)); } } - let new_target_indices = get_target_indices_from_dependencies( + let determined_input_indices = get_target_indices_from_dependencies( func_dependencies.iter(), - &new_source_input_indices, - ); - let mode = if existing_target_indices == new_target_indices - && new_target_indices.is_some() - { - // If dependency covers all GROUP BY expressions, mode will be `Single`: - Dependency::Single + &grouped_source_indices, + ) + .unwrap_or_default(); + // Dependencies are tracked for columns only, so a computed + // expression is never counted as determined, even when its inputs + // are. + let mut determined_output_indices = vec![]; + for (output_idx, input_idx) in group_by_input_indices.iter().enumerate() { + if let Some(input_idx) = input_idx + && determined_input_indices.contains(input_idx) + { + determined_output_indices.push(output_idx); + } + } + // A determinant is a key of the output, and so determines the + // aggregate values too, if it is a key of the input or determines + // every GROUP BY expression. + let is_output_key = *input_mode == Dependency::Single + || determined_output_indices.len() == group_by_input_indices.len(); + let (output_mode, output_target_indices) = if is_output_key { + (Dependency::Single, all_output_indices.clone()) } else { - // Otherwise, existing mode is preserved: - *mode + (*input_mode, determined_output_indices) }; // All of the composite indices occur in the GROUP BY expression: - if new_source_indices.len() == source_indices.len() { + if source_output_indices.len() == source_indices.len() { // GROUP BY treats NULLs as equal: a determinant covering the // complete grouping key gets at most one output row per NULL too. let output_null_equality = - if new_source_indices.len() == group_by_input_indices.len() { + if source_output_indices.len() == group_by_input_indices.len() { NullEquality::NullEqualsNull } else { *null_equality }; aggregate_func_dependencies.push( FunctionalDependence::new( - new_source_indices, - target_indices.clone(), + source_output_indices, + output_target_indices, *nullable, ) - .with_mode(mode) + .with_mode(output_mode) .with_null_equality(output_null_equality), ); } @@ -532,7 +537,19 @@ pub fn aggregate_functional_dependencies( // aggregation: if !group_by_input_indices.is_empty() { let count = group_by_input_indices.len(); - let source_indices = (0..count).collect::>(); + // Remove extraneous columns (the GROUP BY expressions that the others + // determine) from the key, so that the key still exists after a + // projection removes them. + let source_indices = match get_required_group_by_exprs_indices( + aggr_input_schema, + group_by_input_indices, + ) { + Some(mut indices) => { + indices.sort_unstable(); + indices + } + None => (0..count).collect(), + }; let nullable = source_indices .iter() .any(|idx| aggr_fields[*idx].is_nullable()); @@ -544,6 +561,8 @@ pub fn aggregate_functional_dependencies( // The following simple comparison is working well because // GROUP BY expressions come here as a prefix. item.source_indices.iter().all(|idx| idx < &count) + // Only a key determines the aggregate values: + && item.mode == Dependency::Single // A dependency that is not valid across NULLs cannot replace // the whole GROUP BY key dependency: && item.is_valid_across_nulls(aggr_schema) @@ -551,7 +570,7 @@ pub fn aggregate_functional_dependencies( // Add a new functional dependency associated with the whole table: // Use nullable property of the GROUP BY expression: aggregate_func_dependencies.push( - FunctionalDependence::new(source_indices, target_indices, nullable) + FunctionalDependence::new(source_indices, all_output_indices, nullable) .with_mode(Dependency::Single) // Grouping collapses NULL keys into a single group: .with_null_equality(NullEquality::NullEqualsNull), diff --git a/datafusion/sqllogictest/test_files/functional_dependencies.slt b/datafusion/sqllogictest/test_files/functional_dependencies.slt index f24527129428e..1e65d4eb2fb12 100644 --- a/datafusion/sqllogictest/test_files/functional_dependencies.slt +++ b/datafusion/sqllogictest/test_files/functional_dependencies.slt @@ -706,6 +706,113 @@ logical_plan 04)------Aggregate: groupBy=[[t_uniq.x]], aggr=[[count(Int64(1))]] 05)--------TableScan: t_uniq projection=[x] +########## +## GROUP BY a key and a column the key does not determine +## +## `item` has PRIMARY KEY `i_item_sk`. Joined with `sales`, item 1 has two +## dates, so `GROUP BY i_item_sk, s_date` forms the groups (1, 100), (1, 101) +## and (2, 100): the pair is a key of the output, `i_item_sk` alone is not. +########## + +statement ok +CREATE TABLE item (i_item_sk INT, i_desc VARCHAR, PRIMARY KEY (i_item_sk)) +AS VALUES (1, 'a'), (2, 'b'); + +statement ok +CREATE TABLE sales (s_item_sk INT, s_date INT) AS VALUES (1, 100), (1, 101), (2, 100); + +statement ok +CREATE TABLE orders (o_item_sk INT, o_amount INT) AS VALUES (1, 10), (2, 20); + +statement ok +CREATE VIEW item_dates AS +SELECT i_item_sk, s_date FROM item JOIN sales ON i_item_sk = s_item_sk +GROUP BY i_item_sk, s_date; + +# 8.1 Each order row joins once per matching group, so the order of item 1 +# counts twice. The join must stay an inner join, not become a semi join. +query I +SELECT sum(o_amount) FROM orders JOIN item_dates ON o_item_sk = i_item_sk; +---- +40 + +query TT +EXPLAIN SELECT sum(o_amount) FROM orders JOIN item_dates ON o_item_sk = i_item_sk; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[sum(CAST(orders.o_amount AS Int64))]] +02)--Projection: orders.o_amount +03)----Inner Join: orders.o_item_sk = item_dates.i_item_sk +04)------TableScan: orders projection=[o_item_sk, o_amount] +05)------SubqueryAlias: item_dates +06)--------Projection: item.i_item_sk +07)----------Aggregate: groupBy=[[item.i_item_sk, sales.s_date]], aggr=[[]] +08)------------Projection: item.i_item_sk, sales.s_date +09)--------------Inner Join: item.i_item_sk = sales.s_item_sk +10)----------------TableScan: item projection=[i_item_sk] +11)----------------TableScan: sales projection=[s_item_sk, s_date] + +# 8.2 GROUP BY the key alone: `i_item_sk` is a key of the output, so the +# join is still converted to a semi join. +query I +SELECT sum(o_amount) FROM orders +JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk GROUP BY i_item_sk) f + ON o_item_sk = f.i_item_sk; +---- +30 + +query TT +EXPLAIN SELECT sum(o_amount) FROM orders +JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk GROUP BY i_item_sk) f + ON o_item_sk = f.i_item_sk; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[sum(CAST(orders.o_amount AS Int64))]] +02)--Projection: orders.o_amount +03)----LeftSemi Join: orders.o_item_sk = f.i_item_sk +04)------TableScan: orders projection=[o_item_sk, o_amount] +05)------SubqueryAlias: f +06)--------Aggregate: groupBy=[[item.i_item_sk]], aggr=[[]] +07)----------LeftSemi Join: item.i_item_sk = sales.s_item_sk +08)------------TableScan: item projection=[i_item_sk] +09)------------TableScan: sales projection=[s_item_sk] + +# 8.3 `i_item_sk` does not determine `s_date`, so an outer GROUP BY must keep +# `s_date` and its three groups. +query II +SELECT i_item_sk, count(*) FROM item_dates GROUP BY i_item_sk, s_date +ORDER BY i_item_sk; +---- +1 1 +1 1 +2 1 + +# 8.4 The planner adds `i_desc`, which `i_item_sk` determines, to the inner +# GROUP BY. `i_desc` is an extraneous column of the inner aggregate's key and +# is removed from it, so the key still exists after the projection removes +# `i_desc`, and `c` can be selected. +query III +SELECT i_item_sk, s_date, c +FROM (SELECT i_item_sk, s_date, count(*) AS c FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, s_date) q +GROUP BY i_item_sk, s_date ORDER BY i_item_sk, s_date; +---- +1 100 1 +1 101 1 +2 100 1 + +statement ok +DROP VIEW item_dates; + +statement ok +DROP TABLE item; + +statement ok +DROP TABLE sales; + +statement ok +DROP TABLE orders; + ########## ## Cleanup ########## From 730a9fe97ef46f0ed1d9a61242c384f7d84ebf9e Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Sun, 8 Nov 2026 22:44:15 +0100 Subject: [PATCH 2/4] fix: treat a deterministic computed GROUP BY expression as determined by a key A computed GROUP BY expression was never counted as determined, so a key next to one, such as `GROUP BY i_item_sk, upper(i_desc)`, was no longer a key of the aggregate output, and queries that plan on 55.2.0 failed. The new `aggregate_functional_dependencies_with_computed_inputs` takes a `computed_input_indices` parameter with the input fields each deterministic computed GROUP BY expression reads. Such an expression is determined when every field it reads is. Volatile expressions and expressions with subqueries or outer references are never determined. `aggregate_functional_dependencies` keeps its signature and is deprecated since 57.0.0; it treats every computed expression as not determined. Also adds tests for the DISTINCT and window frame symptoms of #27112. --- .../common/src/functional_dependencies.rs | 49 +++++++++++++-- datafusion/common/src/lib.rs | 7 ++- datafusion/expr/src/logical_plan/plan.rs | 54 +++++++++++++--- .../test_files/functional_dependencies.slt | 61 ++++++++++++++++++- .../library-user-guide/upgrading/56.0.0.md | 28 +++++++++ 5 files changed, 181 insertions(+), 18 deletions(-) diff --git a/datafusion/common/src/functional_dependencies.rs b/datafusion/common/src/functional_dependencies.rs index 812ffaa69d45c..7618be7776f51 100644 --- a/datafusion/common/src/functional_dependencies.rs +++ b/datafusion/common/src/functional_dependencies.rs @@ -445,10 +445,43 @@ impl Deref for FunctionalDependencies { /// the expression computes a new value. Only a column reference can carry an /// input dependency; a computed expression such as `CAST(pk AS INT)` cannot, /// even though its output name may equal the column's name. +/// +/// Every computed GROUP BY expression is treated as not determined by any +/// determinant. +#[deprecated( + since = "57.0.0", + note = "Use `aggregate_functional_dependencies_with_computed_inputs` instead" +)] pub fn aggregate_functional_dependencies( aggr_input_schema: &DFSchema, group_by_input_indices: &[Option], aggr_schema: &DFSchema, +) -> FunctionalDependencies { + aggregate_functional_dependencies_with_computed_inputs( + aggr_input_schema, + group_by_input_indices, + &vec![None; group_by_input_indices.len()], + aggr_schema, + ) +} + +/// Calculates functional dependencies for aggregate output, when there is a GROUP BY expression. +/// +/// `group_by_input_indices[i]` is the index of the input field that GROUP BY +/// expression `i` passes through unchanged (a column reference), or `None` if +/// the expression computes a new value. Only a column reference can carry an +/// input dependency; a computed expression such as `CAST(pk AS INT)` cannot, +/// even though its output name may equal the column's name. +/// +/// `computed_input_indices[i]` lists the input fields that a deterministic +/// computed GROUP BY expression `i` reads, or is `None` for a column +/// reference or a volatile expression. A computed expression whose fields +/// are all determined by a determinant is determined by it too. +pub fn aggregate_functional_dependencies_with_computed_inputs( + aggr_input_schema: &DFSchema, + group_by_input_indices: &[Option], + computed_input_indices: &[Option>], + aggr_schema: &DFSchema, ) -> FunctionalDependencies { let mut aggregate_func_dependencies = vec![]; let aggr_fields = aggr_schema.fields(); @@ -489,14 +522,18 @@ pub fn aggregate_functional_dependencies( &grouped_source_indices, ) .unwrap_or_default(); - // Dependencies are tracked for columns only, so a computed - // expression is never counted as determined, even when its inputs - // are. + // A deterministic computed expression is determined when every + // field it reads is. let mut determined_output_indices = vec![]; for (output_idx, input_idx) in group_by_input_indices.iter().enumerate() { - if let Some(input_idx) = input_idx - && determined_input_indices.contains(input_idx) - { + let determined = match (input_idx, &computed_input_indices[output_idx]) { + (Some(input_idx), _) => determined_input_indices.contains(input_idx), + (None, Some(read)) => read + .iter() + .all(|idx| determined_input_indices.contains(idx)), + (None, None) => false, + }; + if determined { determined_output_indices.push(output_idx); } } diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 985528afe685a..b37ee729c9e63 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -71,10 +71,13 @@ pub use file_options::file_type::{ DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION, DEFAULT_JSON_EXTENSION, DEFAULT_PARQUET_EXTENSION, GetExt, }; +#[expect(deprecated)] +pub use functional_dependencies::aggregate_functional_dependencies; pub use functional_dependencies::{ Constraint, Constraints, Dependency, FunctionalDependence, FunctionalDependencies, - aggregate_functional_dependencies, get_required_group_by_exprs_indices, - get_required_sort_exprs_indices, get_target_functional_dependencies, + aggregate_functional_dependencies_with_computed_inputs, + get_required_group_by_exprs_indices, get_required_sort_exprs_indices, + get_target_functional_dependencies, }; use hashbrown::DefaultHashBuilder; pub use join_type::{JoinConstraint, JoinSide, JoinType}; diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index c5b5b23ba8cc9..d1297bc9bee3c 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -58,8 +58,9 @@ use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency, FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result, ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions, - aggregate_functional_dependencies, assert_eq_or_internal_err, assert_or_internal_err, - internal_err, plan_datafusion_err, plan_err, validate_range_split_points, + aggregate_functional_dependencies_with_computed_inputs, assert_eq_or_internal_err, + assert_or_internal_err, internal_err, plan_datafusion_err, plan_err, + validate_range_split_points, }; use itertools::Itertools as _; @@ -4529,21 +4530,56 @@ fn calc_func_dependencies_for_aggregate( // - Otherwise, it may be possible to propagate functional dependencies. if !contains_grouping_set(group_expr) { // One entry per GROUP BY output field, in the order of `aggr_schema` - let group_by_input_indices = grouping_set_to_exprlist(group_expr)? - .into_iter() + let group_exprs = grouping_set_to_exprlist(group_expr)?; + let group_by_input_indices = group_exprs + .iter() .map(|item| passthrough_field_index(item, input.schema())) .collect::>(); - let aggregate_func_dependencies = aggregate_functional_dependencies( - input.schema(), - &group_by_input_indices, - aggr_schema, - ); + let computed_input_indices = group_exprs + .iter() + .zip(&group_by_input_indices) + .map(|(item, input_idx)| match input_idx { + Some(_) => None, + None => deterministic_input_indices(item, input.schema()), + }) + .collect::>(); + let aggregate_func_dependencies = + aggregate_functional_dependencies_with_computed_inputs( + input.schema(), + &group_by_input_indices, + &computed_input_indices, + aggr_schema, + ); Ok(aggregate_func_dependencies) } else { Ok(FunctionalDependencies::empty()) } } +/// Returns the input fields that `expr` reads, or `None` if `expr` is volatile, +/// contains a subquery or reads a column that is not an input field. +fn deterministic_input_indices(expr: &Expr, schema: &DFSchema) -> Option> { + let has_subquery = expr + .exists(|e| { + Ok(matches!( + e, + Expr::ScalarSubquery(_) + | Expr::Exists(_) + | Expr::InSubquery(_) + | Expr::SetComparison(_) + | Expr::OuterReferenceColumn(..) + )) + }) + .unwrap_or(true); + if expr.is_volatile() || has_subquery { + return None; + } + expr.column_refs() + .into_iter() + .map(|column| schema.index_of_column(column).ok()) + .collect() +} + /// This function projects functional dependencies of the `input` plan according /// to projection expressions `exprs`. fn calc_func_dependencies_for_project( diff --git a/datafusion/sqllogictest/test_files/functional_dependencies.slt b/datafusion/sqllogictest/test_files/functional_dependencies.slt index 1e65d4eb2fb12..0adc05238b941 100644 --- a/datafusion/sqllogictest/test_files/functional_dependencies.slt +++ b/datafusion/sqllogictest/test_files/functional_dependencies.slt @@ -787,7 +787,23 @@ ORDER BY i_item_sk; 1 1 2 1 -# 8.4 The planner adds `i_desc`, which `i_item_sk` determines, to the inner +# 8.4 `i_item_sk` is not a key of `item_dates`: the DISTINCT must stay, and the +# default window frame must stay RANGE (peers of item 1 count together). +query I +SELECT DISTINCT i_item_sk FROM item_dates ORDER BY i_item_sk; +---- +1 +2 + +query II +SELECT i_item_sk, count(*) OVER (ORDER BY i_item_sk) FROM item_dates +ORDER BY i_item_sk; +---- +1 2 +1 2 +2 3 + +# 8.5 The planner adds `i_desc`, which `i_item_sk` determines, to the inner # GROUP BY. `i_desc` is an extraneous column of the inner aggregate's key and # is removed from it, so the key still exists after the projection removes # `i_desc`, and `c` can be selected. @@ -801,6 +817,49 @@ GROUP BY i_item_sk, s_date ORDER BY i_item_sk, s_date; 1 101 1 2 100 1 +# 8.6 `upper(i_desc)` reads only `i_desc`, which `i_item_sk` determines, so +# `i_item_sk` is a key of the inner aggregate and `c` can be selected. +query II +SELECT i_item_sk, c +FROM (SELECT i_item_sk, upper(i_desc) AS u, count(*) AS c + FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, upper(i_desc)) q +GROUP BY i_item_sk ORDER BY i_item_sk; +---- +1 2 +2 1 + +# 8.7 `s_date + 1` reads `s_date`, which `i_item_sk` does not determine. +statement error +SELECT i_item_sk, c +FROM (SELECT i_item_sk, s_date + 1 AS d, count(*) AS c + FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, s_date + 1) q +GROUP BY i_item_sk; +---- +DataFusion error: Error during planning: Column in SELECT must be in GROUP BY or an aggregate function: While expanding wildcard, column "q.c" must appear in the GROUP BY clause or must be part of an aggregate function, currently only "q.i_item_sk" appears in the SELECT clause satisfies this requirement + + +# 8.8 A volatile expression is never determined: the join stays an inner join. +query TT +EXPLAIN SELECT sum(o_amount) FROM orders +JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, random() > 0.5) g + ON o_item_sk = g.i_item_sk; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[sum(CAST(orders.o_amount AS Int64))]] +02)--Projection: orders.o_amount +03)----Inner Join: orders.o_item_sk = g.i_item_sk +04)------TableScan: orders projection=[o_item_sk, o_amount] +05)------SubqueryAlias: g +06)--------Projection: item.i_item_sk +07)----------Aggregate: groupBy=[[item.i_item_sk, random() > Float64(0.5), item.i_desc]], aggr=[[]] +08)------------Projection: item.i_item_sk, item.i_desc +09)--------------Inner Join: item.i_item_sk = sales.s_item_sk +10)----------------TableScan: item projection=[i_item_sk, i_desc] +11)----------------TableScan: sales projection=[s_item_sk] + statement ok DROP VIEW item_dates; diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md index 769d3fdbeb1db..6d60c70ca5df7 100644 --- a/docs/source/library-user-guide/upgrading/56.0.0.md +++ b/docs/source/library-user-guide/upgrading/56.0.0.md @@ -1123,6 +1123,34 @@ let indices: Vec> = exprs let target = get_target_functional_dependencies(schema, &indices); ``` +### `aggregate_functional_dependencies` is deprecated + +`aggregate_functional_dependencies` is deprecated; use +`aggregate_functional_dependencies_with_computed_inputs` instead. Its new +`computed_input_indices: &[Option>]` parameter lists, for each GROUP +BY expression, the input fields that a deterministic computed expression reads, +or is `None` for a column reference or a volatile expression. A computed +expression is determined by a key when every field it reads is, so the key +stays a key of the aggregate output. The deprecated function treats every +computed expression as not determined. + +**Migration guide:** + +```rust,ignore +// Before +let deps = aggregate_functional_dependencies(input_schema, &group_by_input_indices, aggr_schema); + +// After +let deps = aggregate_functional_dependencies_with_computed_inputs( + input_schema, + &group_by_input_indices, + &computed_input_indices, + aggr_schema, +); +``` + +See [issue #27112](https://github.com/apache/datafusion/issues/27112) for details. + ### `FileSource::exact_filter` controls scan equivalence properties `FileScanConfig` no longer derives equivalence properties (constant columns From aef831217ed721d4a6b1c958a11799bd4c4e6d1e Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Mon, 9 Nov 2026 15:33:30 +0100 Subject: [PATCH 3/4] fix: remove determined computed GROUP BY expressions from the GROUP BY key `get_required_group_by_exprs_indices` gives up on any computed GROUP BY expression, so the GROUP BY key kept every expression. A projection that removed one of them dropped the key, and a query such as `GROUP BY i_item_sk, s_date, upper(i_desc)` below an outer `GROUP BY i_item_sk, s_date` failed to plan, although it plans on 55.2.0. The new `group_by_key_indices` removes extraneous columns from the key as before, and also a deterministic computed expression whose fields the remaining key columns determine. A key left with no columns falls back to the whole GROUP BY list. --- .../common/src/functional_dependencies.rs | 63 ++++++++++++++++--- .../test_files/functional_dependencies.slt | 26 +++++++- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/datafusion/common/src/functional_dependencies.rs b/datafusion/common/src/functional_dependencies.rs index 7618be7776f51..38712b87610db 100644 --- a/datafusion/common/src/functional_dependencies.rs +++ b/datafusion/common/src/functional_dependencies.rs @@ -577,16 +577,11 @@ pub fn aggregate_functional_dependencies_with_computed_inputs( // Remove extraneous columns (the GROUP BY expressions that the others // determine) from the key, so that the key still exists after a // projection removes them. - let source_indices = match get_required_group_by_exprs_indices( + let source_indices = group_by_key_indices( aggr_input_schema, group_by_input_indices, - ) { - Some(mut indices) => { - indices.sort_unstable(); - indices - } - None => (0..count).collect(), - }; + computed_input_indices, + ); let nullable = source_indices .iter() .any(|idx| aggr_fields[*idx].is_nullable()); @@ -617,6 +612,58 @@ pub fn aggregate_functional_dependencies_with_computed_inputs( FunctionalDependencies::new(aggregate_func_dependencies) } +/// Returns the GROUP BY expressions that the whole GROUP BY key needs. A column +/// that the other columns determine is left out, and so is a deterministic +/// computed expression whose fields the remaining columns determine. +fn group_by_key_indices( + schema: &DFSchema, + group_by_input_indices: &[Option], + computed_input_indices: &[Option>], +) -> Vec { + let all = || (0..group_by_input_indices.len()).collect(); + let (column_positions, column_inputs): (Vec, Vec>) = + group_by_input_indices + .iter() + .enumerate() + .filter(|(_, input_idx)| input_idx.is_some()) + .map(|(idx, input_idx)| (idx, *input_idx)) + .unzip(); + let Some(required) = get_required_group_by_exprs_indices(schema, &column_inputs) + else { + return all(); + }; + let key_inputs = required + .iter() + .map(|idx| column_inputs[*idx]) + .collect::>(); + let determined = + get_target_functional_dependencies(schema, &key_inputs).unwrap_or_default(); + let mut key = required + .iter() + .map(|idx| column_positions[*idx]) + .collect::>(); + for (idx, (input_idx, read)) in group_by_input_indices + .iter() + .zip(computed_input_indices) + .enumerate() + { + let is_determined = read.as_ref().is_some_and(|read| { + read.iter().all(|field| { + determined.contains(field) || key_inputs.contains(&Some(*field)) + }) + }); + if input_idx.is_none() && !is_determined { + key.push(idx); + } + } + // A key with no columns is left as the whole GROUP BY list. + if key.is_empty() { + return all(); + } + key.sort_unstable(); + key +} + /// Returns target indices, for the determinant keys that are inside /// group by expressions. /// diff --git a/datafusion/sqllogictest/test_files/functional_dependencies.slt b/datafusion/sqllogictest/test_files/functional_dependencies.slt index 0adc05238b941..ff9a0deb9debc 100644 --- a/datafusion/sqllogictest/test_files/functional_dependencies.slt +++ b/datafusion/sqllogictest/test_files/functional_dependencies.slt @@ -817,7 +817,27 @@ GROUP BY i_item_sk, s_date ORDER BY i_item_sk, s_date; 1 101 1 2 100 1 -# 8.6 `upper(i_desc)` reads only `i_desc`, which `i_item_sk` determines, so +# 8.6 `upper(i_desc)` reads `i_desc`, which `i_item_sk` determines, and +# `s_date + 1` reads `s_date`: both are removed from the key of the inner +# aggregate. +query III +SELECT i_item_sk, s_date, c +FROM (SELECT i_item_sk, s_date, count(*) AS c FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, s_date, upper(i_desc), s_date + 1) q +GROUP BY i_item_sk, s_date ORDER BY i_item_sk, s_date; +---- +1 100 1 +1 101 1 +2 100 1 + +# 8.7 A GROUP BY that reads no column keeps the whole GROUP BY list as its key. +query I +SELECT count(*) FROM orders +JOIN (SELECT 1 AS k FROM sales WHERE false GROUP BY 'x') g ON o_item_sk = g.k; +---- +0 + +# 8.8 `upper(i_desc)` reads only `i_desc`, which `i_item_sk` determines, so # `i_item_sk` is a key of the inner aggregate and `c` can be selected. query II SELECT i_item_sk, c @@ -829,7 +849,7 @@ GROUP BY i_item_sk ORDER BY i_item_sk; 1 2 2 1 -# 8.7 `s_date + 1` reads `s_date`, which `i_item_sk` does not determine. +# 8.9 `s_date + 1` reads `s_date`, which `i_item_sk` does not determine. statement error SELECT i_item_sk, c FROM (SELECT i_item_sk, s_date + 1 AS d, count(*) AS c @@ -840,7 +860,7 @@ GROUP BY i_item_sk; DataFusion error: Error during planning: Column in SELECT must be in GROUP BY or an aggregate function: While expanding wildcard, column "q.c" must appear in the GROUP BY clause or must be part of an aggregate function, currently only "q.i_item_sk" appears in the SELECT clause satisfies this requirement -# 8.8 A volatile expression is never determined: the join stays an inner join. +# 8.10 A volatile expression is never determined: the join stays an inner join. query TT EXPLAIN SELECT sum(o_amount) FROM orders JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk From 360c072384d404bce5667c26cd7771f2297a964f Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Tue, 10 Nov 2026 00:04:56 +0100 Subject: [PATCH 4/4] refactor: decide which GROUP BY expressions are determined in one place The dependency loop and the GROUP BY key reduction each checked whether a GROUP BY expression is determined, in two different ways. Both now use `group_by_positions_determined_by`, which returns the GROUP BY positions that a set of input fields determines. `group_by_key_indices` builds that set from the GROUP BY columns and what the remaining key columns determine, and keeps the key columns and every expression outside it. --- .../common/src/functional_dependencies.rs | 122 +++++++++++------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/datafusion/common/src/functional_dependencies.rs b/datafusion/common/src/functional_dependencies.rs index 38712b87610db..aa45723a6f182 100644 --- a/datafusion/common/src/functional_dependencies.rs +++ b/datafusion/common/src/functional_dependencies.rs @@ -522,21 +522,11 @@ pub fn aggregate_functional_dependencies_with_computed_inputs( &grouped_source_indices, ) .unwrap_or_default(); - // A deterministic computed expression is determined when every - // field it reads is. - let mut determined_output_indices = vec![]; - for (output_idx, input_idx) in group_by_input_indices.iter().enumerate() { - let determined = match (input_idx, &computed_input_indices[output_idx]) { - (Some(input_idx), _) => determined_input_indices.contains(input_idx), - (None, Some(read)) => read - .iter() - .all(|idx| determined_input_indices.contains(idx)), - (None, None) => false, - }; - if determined { - determined_output_indices.push(output_idx); - } - } + let determined_output_indices = group_by_positions_determined_by( + &determined_input_indices, + group_by_input_indices, + computed_input_indices, + ); // A determinant is a key of the output, and so determines the // aggregate values too, if it is a key of the input or determines // every GROUP BY expression. @@ -612,22 +602,63 @@ pub fn aggregate_functional_dependencies_with_computed_inputs( FunctionalDependencies::new(aggregate_func_dependencies) } -/// Returns the GROUP BY expressions that the whole GROUP BY key needs. A column -/// that the other columns determine is left out, and so is a deterministic -/// computed expression whose fields the remaining columns determine. +/// Returns the positions of the GROUP BY expressions that the input fields +/// `fields` determine: when two input rows have the same value in every field +/// of `fields`, they also have the same value for each of these expressions. +/// NULL counts as the same value as NULL, as in GROUP BY. +/// +/// A column reference is determined when its field is in `fields`. A computed +/// expression is determined when it is deterministic (its +/// `computed_input_indices` entry is `Some`) and every field it reads is in +/// `fields`. +/// +/// For example, for `GROUP BY i_item_sk, s_date, upper(i_desc)` over input +/// fields `[i_item_sk, i_desc, s_item_sk, s_date]`, the fields `[0, 1]` +/// (`i_item_sk` and `i_desc`) determine positions `[0, 2]`, but not `s_date` +/// at position 1. +fn group_by_positions_determined_by( + fields: &[usize], + group_by_input_indices: &[Option], + computed_input_indices: &[Option>], +) -> Vec { + group_by_input_indices + .iter() + .zip(computed_input_indices) + .enumerate() + .filter(|(_, item)| match item { + (Some(field), _) => fields.contains(field), + (None, Some(read)) => read.iter().all(|field| fields.contains(field)), + (None, None) => false, + }) + .map(|(position, _)| position) + .collect() +} + +/// Returns the positions of the GROUP BY expressions that the GROUP BY key +/// needs: the GROUP BY list without its extraneous expressions. An expression +/// is extraneous when the other expressions of the key determine it. +/// +/// Only column references can be key columns, because a computed expression +/// does not expose the fields it reads. The key columns are the column +/// references without the extraneous ones. They determine every column +/// reference of the GROUP BY list, and through the input dependencies some +/// more fields; every expression that these fields determine is left out of +/// the key. +/// +/// For example, `GROUP BY i_item_sk, s_date, upper(i_desc), i_desc` over a +/// join where `i_item_sk` determines `i_desc` needs only `i_item_sk` and +/// `s_date`. fn group_by_key_indices( schema: &DFSchema, group_by_input_indices: &[Option], computed_input_indices: &[Option>], ) -> Vec { let all = || (0..group_by_input_indices.len()).collect(); - let (column_positions, column_inputs): (Vec, Vec>) = - group_by_input_indices - .iter() - .enumerate() - .filter(|(_, input_idx)| input_idx.is_some()) - .map(|(idx, input_idx)| (idx, *input_idx)) - .unzip(); + let column_inputs = group_by_input_indices + .iter() + .filter(|input_idx| input_idx.is_some()) + .copied() + .collect::>(); let Some(required) = get_required_group_by_exprs_indices(schema, &column_inputs) else { return all(); @@ -636,31 +667,34 @@ fn group_by_key_indices( .iter() .map(|idx| column_inputs[*idx]) .collect::>(); - let determined = - get_target_functional_dependencies(schema, &key_inputs).unwrap_or_default(); - let mut key = required + let mut fields = column_inputs.iter().flatten().copied().collect::>(); + fields.extend( + get_target_functional_dependencies(schema, &key_inputs).unwrap_or_default(), + ); + let determined = group_by_positions_determined_by( + &fields, + group_by_input_indices, + computed_input_indices, + ); + // A column can be in the GROUP BY list more than one time (`x, x AS k`): + // the key keeps its first position. + let key_positions = key_inputs .iter() - .map(|idx| column_positions[*idx]) + .filter_map(|key_input| { + group_by_input_indices + .iter() + .position(|input| input == key_input) + }) + .collect::>(); + let key = (0..group_by_input_indices.len()) + .filter(|position| { + key_positions.contains(position) || !determined.contains(position) + }) .collect::>(); - for (idx, (input_idx, read)) in group_by_input_indices - .iter() - .zip(computed_input_indices) - .enumerate() - { - let is_determined = read.as_ref().is_some_and(|read| { - read.iter().all(|field| { - determined.contains(field) || key_inputs.contains(&Some(*field)) - }) - }); - if input_idx.is_none() && !is_determined { - key.push(idx); - } - } // A key with no columns is left as the whole GROUP BY list. if key.is_empty() { return all(); } - key.sort_unstable(); key }