Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 168 additions & 31 deletions datafusion/common/src/functional_dependencies.rs
Original file line number Diff line number Diff line change
Expand Up @@ -445,39 +445,64 @@ impl Deref for FunctionalDependencies {
/// the expression computes a new value. Only a column reference can carry an
/// input dependency; a computed expression such as `CAST(pk AS INT)` cannot,
/// even though its output name may equal the column's name.
///
/// Every computed GROUP BY expression is treated as not determined by any
/// determinant.
#[deprecated(
since = "57.0.0",
note = "Use `aggregate_functional_dependencies_with_computed_inputs` instead"
)]
pub fn aggregate_functional_dependencies(
aggr_input_schema: &DFSchema,
group_by_input_indices: &[Option<usize>],
aggr_schema: &DFSchema,
) -> FunctionalDependencies {
aggregate_functional_dependencies_with_computed_inputs(
aggr_input_schema,
group_by_input_indices,
&vec![None; group_by_input_indices.len()],
aggr_schema,
)
}

/// Calculates functional dependencies for aggregate output, when there is a GROUP BY expression.
///
/// `group_by_input_indices[i]` is the index of the input field that GROUP BY
/// expression `i` passes through unchanged (a column reference), or `None` if
/// the expression computes a new value. Only a column reference can carry an
/// input dependency; a computed expression such as `CAST(pk AS INT)` cannot,
/// even though its output name may equal the column's name.
///
/// `computed_input_indices[i]` lists the input fields that a deterministic
/// computed GROUP BY expression `i` reads, or is `None` for a column
/// reference or a volatile expression. A computed expression whose fields
/// are all determined by a determinant is determined by it too.
pub fn aggregate_functional_dependencies_with_computed_inputs(
aggr_input_schema: &DFSchema,
group_by_input_indices: &[Option<usize>],
computed_input_indices: &[Option<Vec<usize>>],
aggr_schema: &DFSchema,
) -> FunctionalDependencies {
let mut aggregate_func_dependencies = vec![];
let aggr_fields = aggr_schema.fields();
// Association covers the whole table:
let target_indices = (0..aggr_schema.fields().len()).collect::<Vec<_>>();
let all_output_indices = (0..aggr_schema.fields().len()).collect::<Vec<_>>();
// Get functional dependencies of the schema:
let func_dependencies = aggr_input_schema.functional_dependencies();
// The loop below only re-expresses input dependencies. Skip it when the
// input has none. The GROUP BY-key dependency below always runs.
if !func_dependencies.is_empty() {
// Compare the columns covered by every dependency, whatever its NULL
// semantics: leaving out the ones that do not hold across NULLs would
// hide GROUP BY expressions this determinant does not cover and
// upgrade its mode to `Single` wrongly.
let existing_target_indices = get_target_indices_from_dependencies(
func_dependencies.iter(),
group_by_input_indices,
);
for FunctionalDependence {
source_indices,
nullable,
null_equality,
mode,
mode: input_mode,
..
} in &func_dependencies.deps
{
// Indices into the GROUP BY list for this determinant:
let mut new_source_indices = vec![];
let mut new_source_input_indices = vec![];
let mut source_output_indices = vec![];
let mut grouped_source_indices = vec![];
for (idx, input_idx) in group_by_input_indices.iter().enumerate() {
// When one of the input determinant columns is a GROUP BY
// expression, add the index of the GROUP BY expression as a new
Expand All @@ -486,42 +511,49 @@ pub fn aggregate_functional_dependencies(
&& source_indices.contains(input_idx)
// A key column can be in the GROUP BY list more than one
// time (`x, x AS k`). Count it one time.
&& !new_source_input_indices.contains(&Some(*input_idx))
&& !grouped_source_indices.contains(&Some(*input_idx))
{
new_source_indices.push(idx);
new_source_input_indices.push(Some(*input_idx));
source_output_indices.push(idx);
grouped_source_indices.push(Some(*input_idx));
}
}
let new_target_indices = get_target_indices_from_dependencies(
let determined_input_indices = get_target_indices_from_dependencies(
func_dependencies.iter(),
&new_source_input_indices,
&grouped_source_indices,
)
.unwrap_or_default();
let determined_output_indices = group_by_positions_determined_by(
&determined_input_indices,
group_by_input_indices,
computed_input_indices,
);
let mode = if existing_target_indices == new_target_indices
&& new_target_indices.is_some()
{
// If dependency covers all GROUP BY expressions, mode will be `Single`:
Dependency::Single
// A determinant is a key of the output, and so determines the
// aggregate values too, if it is a key of the input or determines
// every GROUP BY expression.
let is_output_key = *input_mode == Dependency::Single
|| determined_output_indices.len() == group_by_input_indices.len();
let (output_mode, output_target_indices) = if is_output_key {
(Dependency::Single, all_output_indices.clone())
} else {
// Otherwise, existing mode is preserved:
*mode
(*input_mode, determined_output_indices)
};
// All of the composite indices occur in the GROUP BY expression:
if new_source_indices.len() == source_indices.len() {
if source_output_indices.len() == source_indices.len() {
// GROUP BY treats NULLs as equal: a determinant covering the
// complete grouping key gets at most one output row per NULL too.
let output_null_equality =
if new_source_indices.len() == group_by_input_indices.len() {
if source_output_indices.len() == group_by_input_indices.len() {
NullEquality::NullEqualsNull
} else {
*null_equality
};
aggregate_func_dependencies.push(
FunctionalDependence::new(
new_source_indices,
target_indices.clone(),
source_output_indices,
output_target_indices,
*nullable,
)
.with_mode(mode)
.with_mode(output_mode)
.with_null_equality(output_null_equality),
);
}
Expand All @@ -532,7 +564,14 @@ pub fn aggregate_functional_dependencies(
// aggregation:
if !group_by_input_indices.is_empty() {
let count = group_by_input_indices.len();
let source_indices = (0..count).collect::<Vec<_>>();
// Remove extraneous columns (the GROUP BY expressions that the others
// determine) from the key, so that the key still exists after a
// projection removes them.
let source_indices = group_by_key_indices(
aggr_input_schema,
group_by_input_indices,
computed_input_indices,
);
let nullable = source_indices
.iter()
.any(|idx| aggr_fields[*idx].is_nullable());
Expand All @@ -544,14 +583,16 @@ pub fn aggregate_functional_dependencies(
// The following simple comparison is working well because
// GROUP BY expressions come here as a prefix.
item.source_indices.iter().all(|idx| idx < &count)
// Only a key determines the aggregate values:
&& item.mode == Dependency::Single
// A dependency that is not valid across NULLs cannot replace
// the whole GROUP BY key dependency:
&& item.is_valid_across_nulls(aggr_schema)
}) {
// Add a new functional dependency associated with the whole table:
// Use nullable property of the GROUP BY expression:
aggregate_func_dependencies.push(
FunctionalDependence::new(source_indices, target_indices, nullable)
FunctionalDependence::new(source_indices, all_output_indices, nullable)
.with_mode(Dependency::Single)
// Grouping collapses NULL keys into a single group:
.with_null_equality(NullEquality::NullEqualsNull),
Expand All @@ -561,6 +602,102 @@ pub fn aggregate_functional_dependencies(
FunctionalDependencies::new(aggregate_func_dependencies)
}

/// Returns the positions of the GROUP BY expressions that the input fields
/// `fields` determine: when two input rows have the same value in every field
/// of `fields`, they also have the same value for each of these expressions.
/// NULL counts as the same value as NULL, as in GROUP BY.
///
/// A column reference is determined when its field is in `fields`. A computed
/// expression is determined when it is deterministic (its
/// `computed_input_indices` entry is `Some`) and every field it reads is in
/// `fields`.
///
/// For example, for `GROUP BY i_item_sk, s_date, upper(i_desc)` over input
/// fields `[i_item_sk, i_desc, s_item_sk, s_date]`, the fields `[0, 1]`
/// (`i_item_sk` and `i_desc`) determine positions `[0, 2]`, but not `s_date`
/// at position 1.
fn group_by_positions_determined_by(
fields: &[usize],
group_by_input_indices: &[Option<usize>],
computed_input_indices: &[Option<Vec<usize>>],
) -> Vec<usize> {
group_by_input_indices
.iter()
.zip(computed_input_indices)
.enumerate()
.filter(|(_, item)| match item {
(Some(field), _) => fields.contains(field),
(None, Some(read)) => read.iter().all(|field| fields.contains(field)),
(None, None) => false,
})
.map(|(position, _)| position)
.collect()
}

/// Returns the positions of the GROUP BY expressions that the GROUP BY key
/// needs: the GROUP BY list without its extraneous expressions. An expression
/// is extraneous when the other expressions of the key determine it.
///
/// Only column references can be key columns, because a computed expression
/// does not expose the fields it reads. The key columns are the column
/// references without the extraneous ones. They determine every column
/// reference of the GROUP BY list, and through the input dependencies some
/// more fields; every expression that these fields determine is left out of
/// the key.
///
/// For example, `GROUP BY i_item_sk, s_date, upper(i_desc), i_desc` over a
/// join where `i_item_sk` determines `i_desc` needs only `i_item_sk` and
/// `s_date`.
fn group_by_key_indices(
schema: &DFSchema,
group_by_input_indices: &[Option<usize>],
computed_input_indices: &[Option<Vec<usize>>],
) -> Vec<usize> {
let all = || (0..group_by_input_indices.len()).collect();
let column_inputs = group_by_input_indices
.iter()
.filter(|input_idx| input_idx.is_some())
.copied()
.collect::<Vec<_>>();
let Some(required) = get_required_group_by_exprs_indices(schema, &column_inputs)
else {
return all();
};
let key_inputs = required
.iter()
.map(|idx| column_inputs[*idx])
.collect::<Vec<_>>();
let mut fields = column_inputs.iter().flatten().copied().collect::<Vec<_>>();
fields.extend(
get_target_functional_dependencies(schema, &key_inputs).unwrap_or_default(),
);
let determined = group_by_positions_determined_by(
&fields,
group_by_input_indices,
computed_input_indices,
);
// A column can be in the GROUP BY list more than one time (`x, x AS k`):
// the key keeps its first position.
let key_positions = key_inputs
.iter()
.filter_map(|key_input| {
group_by_input_indices
.iter()
.position(|input| input == key_input)
})
.collect::<Vec<_>>();
let key = (0..group_by_input_indices.len())
.filter(|position| {
key_positions.contains(position) || !determined.contains(position)
})
.collect::<Vec<_>>();
// A key with no columns is left as the whole GROUP BY list.
if key.is_empty() {
return all();
}
key
}

/// Returns target indices, for the determinant keys that are inside
/// group by expressions.
///
Expand Down
7 changes: 5 additions & 2 deletions datafusion/common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,13 @@ pub use file_options::file_type::{
DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,
DEFAULT_JSON_EXTENSION, DEFAULT_PARQUET_EXTENSION, GetExt,
};
#[expect(deprecated)]
pub use functional_dependencies::aggregate_functional_dependencies;
pub use functional_dependencies::{
Constraint, Constraints, Dependency, FunctionalDependence, FunctionalDependencies,
aggregate_functional_dependencies, get_required_group_by_exprs_indices,
get_required_sort_exprs_indices, get_target_functional_dependencies,
aggregate_functional_dependencies_with_computed_inputs,
get_required_group_by_exprs_indices, get_required_sort_exprs_indices,
get_target_functional_dependencies,
};
use hashbrown::DefaultHashBuilder;
pub use join_type::{JoinConstraint, JoinSide, JoinType};
Expand Down
54 changes: 45 additions & 9 deletions datafusion/expr/src/logical_plan/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ use datafusion_common::{
Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency,
FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result,
ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions,
aggregate_functional_dependencies, assert_eq_or_internal_err, assert_or_internal_err,
internal_err, plan_datafusion_err, plan_err, validate_range_split_points,
aggregate_functional_dependencies_with_computed_inputs, assert_eq_or_internal_err,
assert_or_internal_err, internal_err, plan_datafusion_err, plan_err,
validate_range_split_points,
};
use itertools::Itertools as _;

Expand Down Expand Up @@ -4529,21 +4530,56 @@ fn calc_func_dependencies_for_aggregate(
// - Otherwise, it may be possible to propagate functional dependencies.
if !contains_grouping_set(group_expr) {
// One entry per GROUP BY output field, in the order of `aggr_schema`
let group_by_input_indices = grouping_set_to_exprlist(group_expr)?
.into_iter()
let group_exprs = grouping_set_to_exprlist(group_expr)?;
let group_by_input_indices = group_exprs
.iter()
.map(|item| passthrough_field_index(item, input.schema()))
.collect::<Vec<_>>();
let aggregate_func_dependencies = aggregate_functional_dependencies(
input.schema(),
&group_by_input_indices,
aggr_schema,
);
let computed_input_indices = group_exprs
.iter()
.zip(&group_by_input_indices)
.map(|(item, input_idx)| match input_idx {
Some(_) => None,
None => deterministic_input_indices(item, input.schema()),
})
.collect::<Vec<_>>();
let aggregate_func_dependencies =
aggregate_functional_dependencies_with_computed_inputs(
input.schema(),
&group_by_input_indices,
&computed_input_indices,
aggr_schema,
);
Ok(aggregate_func_dependencies)
} else {
Ok(FunctionalDependencies::empty())
}
}

/// Returns the input fields that `expr` reads, or `None` if `expr` is volatile,
/// contains a subquery or reads a column that is not an input field.
fn deterministic_input_indices(expr: &Expr, schema: &DFSchema) -> Option<Vec<usize>> {
let has_subquery = expr
.exists(|e| {
Ok(matches!(
e,
Expr::ScalarSubquery(_)
| Expr::Exists(_)
| Expr::InSubquery(_)
| Expr::SetComparison(_)
| Expr::OuterReferenceColumn(..)
))
})
.unwrap_or(true);
if expr.is_volatile() || has_subquery {
return None;
}
expr.column_refs()
.into_iter()
.map(|column| schema.index_of_column(column).ok())
.collect()
}

/// This function projects functional dependencies of the `input` plan according
/// to projection expressions `exprs`.
fn calc_func_dependencies_for_project(
Expand Down
Loading
Loading