diff --git a/datafusion/common/src/functional_dependencies.rs b/datafusion/common/src/functional_dependencies.rs index e6b978fa078f2..aa45723a6f182 100644 --- a/datafusion/common/src/functional_dependencies.rs +++ b/datafusion/common/src/functional_dependencies.rs @@ -445,39 +445,64 @@ impl Deref for FunctionalDependencies { /// the expression computes a new value. Only a column reference can carry an /// input dependency; a computed expression such as `CAST(pk AS INT)` cannot, /// even though its output name may equal the column's name. +/// +/// Every computed GROUP BY expression is treated as not determined by any +/// determinant. +#[deprecated( + since = "57.0.0", + note = "Use `aggregate_functional_dependencies_with_computed_inputs` instead" +)] pub fn aggregate_functional_dependencies( aggr_input_schema: &DFSchema, group_by_input_indices: &[Option], aggr_schema: &DFSchema, +) -> FunctionalDependencies { + aggregate_functional_dependencies_with_computed_inputs( + aggr_input_schema, + group_by_input_indices, + &vec![None; group_by_input_indices.len()], + aggr_schema, + ) +} + +/// Calculates functional dependencies for aggregate output, when there is a GROUP BY expression. +/// +/// `group_by_input_indices[i]` is the index of the input field that GROUP BY +/// expression `i` passes through unchanged (a column reference), or `None` if +/// the expression computes a new value. Only a column reference can carry an +/// input dependency; a computed expression such as `CAST(pk AS INT)` cannot, +/// even though its output name may equal the column's name. +/// +/// `computed_input_indices[i]` lists the input fields that a deterministic +/// computed GROUP BY expression `i` reads, or is `None` for a column +/// reference or a volatile expression. A computed expression whose fields +/// are all determined by a determinant is determined by it too. +pub fn aggregate_functional_dependencies_with_computed_inputs( + aggr_input_schema: &DFSchema, + group_by_input_indices: &[Option], + computed_input_indices: &[Option>], + aggr_schema: &DFSchema, ) -> FunctionalDependencies { let mut aggregate_func_dependencies = vec![]; let aggr_fields = aggr_schema.fields(); // Association covers the whole table: - let target_indices = (0..aggr_schema.fields().len()).collect::>(); + let all_output_indices = (0..aggr_schema.fields().len()).collect::>(); // Get functional dependencies of the schema: let func_dependencies = aggr_input_schema.functional_dependencies(); // The loop below only re-expresses input dependencies. Skip it when the // input has none. The GROUP BY-key dependency below always runs. if !func_dependencies.is_empty() { - // Compare the columns covered by every dependency, whatever its NULL - // semantics: leaving out the ones that do not hold across NULLs would - // hide GROUP BY expressions this determinant does not cover and - // upgrade its mode to `Single` wrongly. - let existing_target_indices = get_target_indices_from_dependencies( - func_dependencies.iter(), - group_by_input_indices, - ); for FunctionalDependence { source_indices, nullable, null_equality, - mode, + mode: input_mode, .. } in &func_dependencies.deps { // Indices into the GROUP BY list for this determinant: - let mut new_source_indices = vec![]; - let mut new_source_input_indices = vec![]; + let mut source_output_indices = vec![]; + let mut grouped_source_indices = vec![]; for (idx, input_idx) in group_by_input_indices.iter().enumerate() { // When one of the input determinant columns is a GROUP BY // expression, add the index of the GROUP BY expression as a new @@ -486,42 +511,49 @@ pub fn aggregate_functional_dependencies( && source_indices.contains(input_idx) // A key column can be in the GROUP BY list more than one // time (`x, x AS k`). Count it one time. - && !new_source_input_indices.contains(&Some(*input_idx)) + && !grouped_source_indices.contains(&Some(*input_idx)) { - new_source_indices.push(idx); - new_source_input_indices.push(Some(*input_idx)); + source_output_indices.push(idx); + grouped_source_indices.push(Some(*input_idx)); } } - let new_target_indices = get_target_indices_from_dependencies( + let determined_input_indices = get_target_indices_from_dependencies( func_dependencies.iter(), - &new_source_input_indices, + &grouped_source_indices, + ) + .unwrap_or_default(); + let determined_output_indices = group_by_positions_determined_by( + &determined_input_indices, + group_by_input_indices, + computed_input_indices, ); - let mode = if existing_target_indices == new_target_indices - && new_target_indices.is_some() - { - // If dependency covers all GROUP BY expressions, mode will be `Single`: - Dependency::Single + // A determinant is a key of the output, and so determines the + // aggregate values too, if it is a key of the input or determines + // every GROUP BY expression. + let is_output_key = *input_mode == Dependency::Single + || determined_output_indices.len() == group_by_input_indices.len(); + let (output_mode, output_target_indices) = if is_output_key { + (Dependency::Single, all_output_indices.clone()) } else { - // Otherwise, existing mode is preserved: - *mode + (*input_mode, determined_output_indices) }; // All of the composite indices occur in the GROUP BY expression: - if new_source_indices.len() == source_indices.len() { + if source_output_indices.len() == source_indices.len() { // GROUP BY treats NULLs as equal: a determinant covering the // complete grouping key gets at most one output row per NULL too. let output_null_equality = - if new_source_indices.len() == group_by_input_indices.len() { + if source_output_indices.len() == group_by_input_indices.len() { NullEquality::NullEqualsNull } else { *null_equality }; aggregate_func_dependencies.push( FunctionalDependence::new( - new_source_indices, - target_indices.clone(), + source_output_indices, + output_target_indices, *nullable, ) - .with_mode(mode) + .with_mode(output_mode) .with_null_equality(output_null_equality), ); } @@ -532,7 +564,14 @@ pub fn aggregate_functional_dependencies( // aggregation: if !group_by_input_indices.is_empty() { let count = group_by_input_indices.len(); - let source_indices = (0..count).collect::>(); + // Remove extraneous columns (the GROUP BY expressions that the others + // determine) from the key, so that the key still exists after a + // projection removes them. + let source_indices = group_by_key_indices( + aggr_input_schema, + group_by_input_indices, + computed_input_indices, + ); let nullable = source_indices .iter() .any(|idx| aggr_fields[*idx].is_nullable()); @@ -544,6 +583,8 @@ pub fn aggregate_functional_dependencies( // The following simple comparison is working well because // GROUP BY expressions come here as a prefix. item.source_indices.iter().all(|idx| idx < &count) + // Only a key determines the aggregate values: + && item.mode == Dependency::Single // A dependency that is not valid across NULLs cannot replace // the whole GROUP BY key dependency: && item.is_valid_across_nulls(aggr_schema) @@ -551,7 +592,7 @@ pub fn aggregate_functional_dependencies( // Add a new functional dependency associated with the whole table: // Use nullable property of the GROUP BY expression: aggregate_func_dependencies.push( - FunctionalDependence::new(source_indices, target_indices, nullable) + FunctionalDependence::new(source_indices, all_output_indices, nullable) .with_mode(Dependency::Single) // Grouping collapses NULL keys into a single group: .with_null_equality(NullEquality::NullEqualsNull), @@ -561,6 +602,102 @@ pub fn aggregate_functional_dependencies( FunctionalDependencies::new(aggregate_func_dependencies) } +/// Returns the positions of the GROUP BY expressions that the input fields +/// `fields` determine: when two input rows have the same value in every field +/// of `fields`, they also have the same value for each of these expressions. +/// NULL counts as the same value as NULL, as in GROUP BY. +/// +/// A column reference is determined when its field is in `fields`. A computed +/// expression is determined when it is deterministic (its +/// `computed_input_indices` entry is `Some`) and every field it reads is in +/// `fields`. +/// +/// For example, for `GROUP BY i_item_sk, s_date, upper(i_desc)` over input +/// fields `[i_item_sk, i_desc, s_item_sk, s_date]`, the fields `[0, 1]` +/// (`i_item_sk` and `i_desc`) determine positions `[0, 2]`, but not `s_date` +/// at position 1. +fn group_by_positions_determined_by( + fields: &[usize], + group_by_input_indices: &[Option], + computed_input_indices: &[Option>], +) -> Vec { + group_by_input_indices + .iter() + .zip(computed_input_indices) + .enumerate() + .filter(|(_, item)| match item { + (Some(field), _) => fields.contains(field), + (None, Some(read)) => read.iter().all(|field| fields.contains(field)), + (None, None) => false, + }) + .map(|(position, _)| position) + .collect() +} + +/// Returns the positions of the GROUP BY expressions that the GROUP BY key +/// needs: the GROUP BY list without its extraneous expressions. An expression +/// is extraneous when the other expressions of the key determine it. +/// +/// Only column references can be key columns, because a computed expression +/// does not expose the fields it reads. The key columns are the column +/// references without the extraneous ones. They determine every column +/// reference of the GROUP BY list, and through the input dependencies some +/// more fields; every expression that these fields determine is left out of +/// the key. +/// +/// For example, `GROUP BY i_item_sk, s_date, upper(i_desc), i_desc` over a +/// join where `i_item_sk` determines `i_desc` needs only `i_item_sk` and +/// `s_date`. +fn group_by_key_indices( + schema: &DFSchema, + group_by_input_indices: &[Option], + computed_input_indices: &[Option>], +) -> Vec { + let all = || (0..group_by_input_indices.len()).collect(); + let column_inputs = group_by_input_indices + .iter() + .filter(|input_idx| input_idx.is_some()) + .copied() + .collect::>(); + let Some(required) = get_required_group_by_exprs_indices(schema, &column_inputs) + else { + return all(); + }; + let key_inputs = required + .iter() + .map(|idx| column_inputs[*idx]) + .collect::>(); + let mut fields = column_inputs.iter().flatten().copied().collect::>(); + fields.extend( + get_target_functional_dependencies(schema, &key_inputs).unwrap_or_default(), + ); + let determined = group_by_positions_determined_by( + &fields, + group_by_input_indices, + computed_input_indices, + ); + // A column can be in the GROUP BY list more than one time (`x, x AS k`): + // the key keeps its first position. + let key_positions = key_inputs + .iter() + .filter_map(|key_input| { + group_by_input_indices + .iter() + .position(|input| input == key_input) + }) + .collect::>(); + let key = (0..group_by_input_indices.len()) + .filter(|position| { + key_positions.contains(position) || !determined.contains(position) + }) + .collect::>(); + // A key with no columns is left as the whole GROUP BY list. + if key.is_empty() { + return all(); + } + key +} + /// Returns target indices, for the determinant keys that are inside /// group by expressions. /// diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 985528afe685a..b37ee729c9e63 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -71,10 +71,13 @@ pub use file_options::file_type::{ DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION, DEFAULT_JSON_EXTENSION, DEFAULT_PARQUET_EXTENSION, GetExt, }; +#[expect(deprecated)] +pub use functional_dependencies::aggregate_functional_dependencies; pub use functional_dependencies::{ Constraint, Constraints, Dependency, FunctionalDependence, FunctionalDependencies, - aggregate_functional_dependencies, get_required_group_by_exprs_indices, - get_required_sort_exprs_indices, get_target_functional_dependencies, + aggregate_functional_dependencies_with_computed_inputs, + get_required_group_by_exprs_indices, get_required_sort_exprs_indices, + get_target_functional_dependencies, }; use hashbrown::DefaultHashBuilder; pub use join_type::{JoinConstraint, JoinSide, JoinType}; diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index c5b5b23ba8cc9..d1297bc9bee3c 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -58,8 +58,9 @@ use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency, FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result, ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions, - aggregate_functional_dependencies, assert_eq_or_internal_err, assert_or_internal_err, - internal_err, plan_datafusion_err, plan_err, validate_range_split_points, + aggregate_functional_dependencies_with_computed_inputs, assert_eq_or_internal_err, + assert_or_internal_err, internal_err, plan_datafusion_err, plan_err, + validate_range_split_points, }; use itertools::Itertools as _; @@ -4529,21 +4530,56 @@ fn calc_func_dependencies_for_aggregate( // - Otherwise, it may be possible to propagate functional dependencies. if !contains_grouping_set(group_expr) { // One entry per GROUP BY output field, in the order of `aggr_schema` - let group_by_input_indices = grouping_set_to_exprlist(group_expr)? - .into_iter() + let group_exprs = grouping_set_to_exprlist(group_expr)?; + let group_by_input_indices = group_exprs + .iter() .map(|item| passthrough_field_index(item, input.schema())) .collect::>(); - let aggregate_func_dependencies = aggregate_functional_dependencies( - input.schema(), - &group_by_input_indices, - aggr_schema, - ); + let computed_input_indices = group_exprs + .iter() + .zip(&group_by_input_indices) + .map(|(item, input_idx)| match input_idx { + Some(_) => None, + None => deterministic_input_indices(item, input.schema()), + }) + .collect::>(); + let aggregate_func_dependencies = + aggregate_functional_dependencies_with_computed_inputs( + input.schema(), + &group_by_input_indices, + &computed_input_indices, + aggr_schema, + ); Ok(aggregate_func_dependencies) } else { Ok(FunctionalDependencies::empty()) } } +/// Returns the input fields that `expr` reads, or `None` if `expr` is volatile, +/// contains a subquery or reads a column that is not an input field. +fn deterministic_input_indices(expr: &Expr, schema: &DFSchema) -> Option> { + let has_subquery = expr + .exists(|e| { + Ok(matches!( + e, + Expr::ScalarSubquery(_) + | Expr::Exists(_) + | Expr::InSubquery(_) + | Expr::SetComparison(_) + | Expr::OuterReferenceColumn(..) + )) + }) + .unwrap_or(true); + if expr.is_volatile() || has_subquery { + return None; + } + expr.column_refs() + .into_iter() + .map(|column| schema.index_of_column(column).ok()) + .collect() +} + /// This function projects functional dependencies of the `input` plan according /// to projection expressions `exprs`. fn calc_func_dependencies_for_project( diff --git a/datafusion/sqllogictest/test_files/functional_dependencies.slt b/datafusion/sqllogictest/test_files/functional_dependencies.slt index f24527129428e..ff9a0deb9debc 100644 --- a/datafusion/sqllogictest/test_files/functional_dependencies.slt +++ b/datafusion/sqllogictest/test_files/functional_dependencies.slt @@ -706,6 +706,192 @@ logical_plan 04)------Aggregate: groupBy=[[t_uniq.x]], aggr=[[count(Int64(1))]] 05)--------TableScan: t_uniq projection=[x] +########## +## GROUP BY a key and a column the key does not determine +## +## `item` has PRIMARY KEY `i_item_sk`. Joined with `sales`, item 1 has two +## dates, so `GROUP BY i_item_sk, s_date` forms the groups (1, 100), (1, 101) +## and (2, 100): the pair is a key of the output, `i_item_sk` alone is not. +########## + +statement ok +CREATE TABLE item (i_item_sk INT, i_desc VARCHAR, PRIMARY KEY (i_item_sk)) +AS VALUES (1, 'a'), (2, 'b'); + +statement ok +CREATE TABLE sales (s_item_sk INT, s_date INT) AS VALUES (1, 100), (1, 101), (2, 100); + +statement ok +CREATE TABLE orders (o_item_sk INT, o_amount INT) AS VALUES (1, 10), (2, 20); + +statement ok +CREATE VIEW item_dates AS +SELECT i_item_sk, s_date FROM item JOIN sales ON i_item_sk = s_item_sk +GROUP BY i_item_sk, s_date; + +# 8.1 Each order row joins once per matching group, so the order of item 1 +# counts twice. The join must stay an inner join, not become a semi join. +query I +SELECT sum(o_amount) FROM orders JOIN item_dates ON o_item_sk = i_item_sk; +---- +40 + +query TT +EXPLAIN SELECT sum(o_amount) FROM orders JOIN item_dates ON o_item_sk = i_item_sk; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[sum(CAST(orders.o_amount AS Int64))]] +02)--Projection: orders.o_amount +03)----Inner Join: orders.o_item_sk = item_dates.i_item_sk +04)------TableScan: orders projection=[o_item_sk, o_amount] +05)------SubqueryAlias: item_dates +06)--------Projection: item.i_item_sk +07)----------Aggregate: groupBy=[[item.i_item_sk, sales.s_date]], aggr=[[]] +08)------------Projection: item.i_item_sk, sales.s_date +09)--------------Inner Join: item.i_item_sk = sales.s_item_sk +10)----------------TableScan: item projection=[i_item_sk] +11)----------------TableScan: sales projection=[s_item_sk, s_date] + +# 8.2 GROUP BY the key alone: `i_item_sk` is a key of the output, so the +# join is still converted to a semi join. +query I +SELECT sum(o_amount) FROM orders +JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk GROUP BY i_item_sk) f + ON o_item_sk = f.i_item_sk; +---- +30 + +query TT +EXPLAIN SELECT sum(o_amount) FROM orders +JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk GROUP BY i_item_sk) f + ON o_item_sk = f.i_item_sk; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[sum(CAST(orders.o_amount AS Int64))]] +02)--Projection: orders.o_amount +03)----LeftSemi Join: orders.o_item_sk = f.i_item_sk +04)------TableScan: orders projection=[o_item_sk, o_amount] +05)------SubqueryAlias: f +06)--------Aggregate: groupBy=[[item.i_item_sk]], aggr=[[]] +07)----------LeftSemi Join: item.i_item_sk = sales.s_item_sk +08)------------TableScan: item projection=[i_item_sk] +09)------------TableScan: sales projection=[s_item_sk] + +# 8.3 `i_item_sk` does not determine `s_date`, so an outer GROUP BY must keep +# `s_date` and its three groups. +query II +SELECT i_item_sk, count(*) FROM item_dates GROUP BY i_item_sk, s_date +ORDER BY i_item_sk; +---- +1 1 +1 1 +2 1 + +# 8.4 `i_item_sk` is not a key of `item_dates`: the DISTINCT must stay, and the +# default window frame must stay RANGE (peers of item 1 count together). +query I +SELECT DISTINCT i_item_sk FROM item_dates ORDER BY i_item_sk; +---- +1 +2 + +query II +SELECT i_item_sk, count(*) OVER (ORDER BY i_item_sk) FROM item_dates +ORDER BY i_item_sk; +---- +1 2 +1 2 +2 3 + +# 8.5 The planner adds `i_desc`, which `i_item_sk` determines, to the inner +# GROUP BY. `i_desc` is an extraneous column of the inner aggregate's key and +# is removed from it, so the key still exists after the projection removes +# `i_desc`, and `c` can be selected. +query III +SELECT i_item_sk, s_date, c +FROM (SELECT i_item_sk, s_date, count(*) AS c FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, s_date) q +GROUP BY i_item_sk, s_date ORDER BY i_item_sk, s_date; +---- +1 100 1 +1 101 1 +2 100 1 + +# 8.6 `upper(i_desc)` reads `i_desc`, which `i_item_sk` determines, and +# `s_date + 1` reads `s_date`: both are removed from the key of the inner +# aggregate. +query III +SELECT i_item_sk, s_date, c +FROM (SELECT i_item_sk, s_date, count(*) AS c FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, s_date, upper(i_desc), s_date + 1) q +GROUP BY i_item_sk, s_date ORDER BY i_item_sk, s_date; +---- +1 100 1 +1 101 1 +2 100 1 + +# 8.7 A GROUP BY that reads no column keeps the whole GROUP BY list as its key. +query I +SELECT count(*) FROM orders +JOIN (SELECT 1 AS k FROM sales WHERE false GROUP BY 'x') g ON o_item_sk = g.k; +---- +0 + +# 8.8 `upper(i_desc)` reads only `i_desc`, which `i_item_sk` determines, so +# `i_item_sk` is a key of the inner aggregate and `c` can be selected. +query II +SELECT i_item_sk, c +FROM (SELECT i_item_sk, upper(i_desc) AS u, count(*) AS c + FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, upper(i_desc)) q +GROUP BY i_item_sk ORDER BY i_item_sk; +---- +1 2 +2 1 + +# 8.9 `s_date + 1` reads `s_date`, which `i_item_sk` does not determine. +statement error +SELECT i_item_sk, c +FROM (SELECT i_item_sk, s_date + 1 AS d, count(*) AS c + FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, s_date + 1) q +GROUP BY i_item_sk; +---- +DataFusion error: Error during planning: Column in SELECT must be in GROUP BY or an aggregate function: While expanding wildcard, column "q.c" must appear in the GROUP BY clause or must be part of an aggregate function, currently only "q.i_item_sk" appears in the SELECT clause satisfies this requirement + + +# 8.10 A volatile expression is never determined: the join stays an inner join. +query TT +EXPLAIN SELECT sum(o_amount) FROM orders +JOIN (SELECT i_item_sk FROM item JOIN sales ON i_item_sk = s_item_sk + GROUP BY i_item_sk, random() > 0.5) g + ON o_item_sk = g.i_item_sk; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[sum(CAST(orders.o_amount AS Int64))]] +02)--Projection: orders.o_amount +03)----Inner Join: orders.o_item_sk = g.i_item_sk +04)------TableScan: orders projection=[o_item_sk, o_amount] +05)------SubqueryAlias: g +06)--------Projection: item.i_item_sk +07)----------Aggregate: groupBy=[[item.i_item_sk, random() > Float64(0.5), item.i_desc]], aggr=[[]] +08)------------Projection: item.i_item_sk, item.i_desc +09)--------------Inner Join: item.i_item_sk = sales.s_item_sk +10)----------------TableScan: item projection=[i_item_sk, i_desc] +11)----------------TableScan: sales projection=[s_item_sk] + +statement ok +DROP VIEW item_dates; + +statement ok +DROP TABLE item; + +statement ok +DROP TABLE sales; + +statement ok +DROP TABLE orders; + ########## ## Cleanup ########## diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md index 769d3fdbeb1db..6d60c70ca5df7 100644 --- a/docs/source/library-user-guide/upgrading/56.0.0.md +++ b/docs/source/library-user-guide/upgrading/56.0.0.md @@ -1123,6 +1123,34 @@ let indices: Vec> = exprs let target = get_target_functional_dependencies(schema, &indices); ``` +### `aggregate_functional_dependencies` is deprecated + +`aggregate_functional_dependencies` is deprecated; use +`aggregate_functional_dependencies_with_computed_inputs` instead. Its new +`computed_input_indices: &[Option>]` parameter lists, for each GROUP +BY expression, the input fields that a deterministic computed expression reads, +or is `None` for a column reference or a volatile expression. A computed +expression is determined by a key when every field it reads is, so the key +stays a key of the aggregate output. The deprecated function treats every +computed expression as not determined. + +**Migration guide:** + +```rust,ignore +// Before +let deps = aggregate_functional_dependencies(input_schema, &group_by_input_indices, aggr_schema); + +// After +let deps = aggregate_functional_dependencies_with_computed_inputs( + input_schema, + &group_by_input_indices, + &computed_input_indices, + aggr_schema, +); +``` + +See [issue #27112](https://github.com/apache/datafusion/issues/27112) for details. + ### `FileSource::exact_filter` controls scan equivalence properties `FileScanConfig` no longer derives equivalence properties (constant columns