Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions datafusion/core/src/datasource/listing/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -641,12 +641,12 @@ impl ListingTable {
}
else {
Err(DataFusionError::Plan(
format!("Expected single column references in output_ordering, got {expr:?}")
format!("Expected single column references in output_ordering, got {expr}")
))
}
} else {
Err(DataFusionError::Plan(
format!("Expected Expr::Sort in output_ordering, but got {expr:?}")
format!("Expected Expr::Sort in output_ordering, but got {expr}")
))
}
})
Expand Down
169 changes: 85 additions & 84 deletions datafusion/core/src/physical_optimizer/dist_enforcement.rs

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions datafusion/core/src/physical_optimizer/join_selection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -687,9 +687,9 @@ mod tests {
// has an exact cardinality of 10_000 rows).
let expected = [
"ProjectionExec: expr=[medium_col@2 as medium_col, big_col@0 as big_col, small_col@1 as small_col]",
" HashJoinExec: mode=CollectLeft, join_type=Right, on=[(Column { name: \"small_col\", index: 1 }, Column { name: \"medium_col\", index: 0 })]",
" HashJoinExec: mode=CollectLeft, join_type=Right, on=[(small_col@1, medium_col@0)]",
" ProjectionExec: expr=[big_col@1 as big_col, small_col@0 as small_col]",
" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(Column { name: \"small_col\", index: 0 }, Column { name: \"big_col\", index: 0 })]",
" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(small_col@0, big_col@0)]",
" StatisticsExec: col_count=1, row_count=Some(1000)",
" StatisticsExec: col_count=1, row_count=Some(100000)",
" StatisticsExec: col_count=1, row_count=Some(10000)",
Expand Down
28 changes: 16 additions & 12 deletions datafusion/core/src/physical_optimizer/sort_enforcement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2381,10 +2381,12 @@ mod tests {
];
let physical_plan = sort_preserving_merge_exec(sort_exprs.clone(), join);

let join_plan =
format!("SortMergeJoin: join_type={join_type}, on=[(Column {{ name: \"nullable_col\", index: 0 }}, Column {{ name: \"col_a\", index: 0 }})]");
let join_plan2 =
format!(" SortMergeJoin: join_type={join_type}, on=[(Column {{ name: \"nullable_col\", index: 0 }}, Column {{ name: \"col_a\", index: 0 }})]");
let join_plan = format!(
"SortMergeJoin: join_type={join_type}, on=[(nullable_col@0, col_a@0)]"
);
let join_plan2 = format!(
" SortMergeJoin: join_type={join_type}, on=[(nullable_col@0, col_a@0)]"
);
let expected_input = vec![
"SortPreservingMergeExec: [nullable_col@0 ASC,non_nullable_col@1 ASC]",
join_plan2.as_str(),
Expand Down Expand Up @@ -2452,16 +2454,18 @@ mod tests {
];
let physical_plan = sort_preserving_merge_exec(sort_exprs, join);

let join_plan =
format!("SortMergeJoin: join_type={join_type}, on=[(Column {{ name: \"nullable_col\", index: 0 }}, Column {{ name: \"col_a\", index: 0 }})]");
let join_plan = format!(
"SortMergeJoin: join_type={join_type}, on=[(nullable_col@0, col_a@0)]"
);
let spm_plan = match join_type {
JoinType::RightAnti => {
"SortPreservingMergeExec: [col_a@0 ASC,col_b@1 ASC]"
}
_ => "SortPreservingMergeExec: [col_a@2 ASC,col_b@3 ASC]",
};
let join_plan2 =
format!(" SortMergeJoin: join_type={join_type}, on=[(Column {{ name: \"nullable_col\", index: 0 }}, Column {{ name: \"col_a\", index: 0 }})]");
let join_plan2 = format!(
" SortMergeJoin: join_type={join_type}, on=[(nullable_col@0, col_a@0)]"
);
let expected_input = vec![
spm_plan,
join_plan2.as_str(),
Expand Down Expand Up @@ -2521,15 +2525,15 @@ mod tests {

let expected_input = vec![
"SortPreservingMergeExec: [col_b@3 ASC,col_a@2 ASC]",
" SortMergeJoin: join_type=Inner, on=[(Column { name: \"nullable_col\", index: 0 }, Column { name: \"col_a\", index: 0 })]",
" SortMergeJoin: join_type=Inner, on=[(nullable_col@0, col_a@0)]",
" ParquetExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col]",
" ParquetExec: file_groups={1 group: [[x]]}, projection=[col_a, col_b]",
];

// can not push down the sort requirements, need to add SortExec
let expected_optimized = vec![
"SortExec: expr=[col_b@3 ASC,col_a@2 ASC]",
" SortMergeJoin: join_type=Inner, on=[(Column { name: \"nullable_col\", index: 0 }, Column { name: \"col_a\", index: 0 })]",
" SortMergeJoin: join_type=Inner, on=[(nullable_col@0, col_a@0)]",
" SortExec: expr=[nullable_col@0 ASC]",
" ParquetExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col]",
" SortExec: expr=[col_a@0 ASC]",
Expand All @@ -2547,15 +2551,15 @@ mod tests {

let expected_input = vec![
"SortPreservingMergeExec: [nullable_col@0 ASC,col_b@3 ASC,col_a@2 ASC]",
" SortMergeJoin: join_type=Inner, on=[(Column { name: \"nullable_col\", index: 0 }, Column { name: \"col_a\", index: 0 })]",
" SortMergeJoin: join_type=Inner, on=[(nullable_col@0, col_a@0)]",
" ParquetExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col]",
" ParquetExec: file_groups={1 group: [[x]]}, projection=[col_a, col_b]",
];

// can not push down the sort requirements, need to add SortExec
let expected_optimized = vec![
"SortExec: expr=[nullable_col@0 ASC,col_b@3 ASC,col_a@2 ASC]",
" SortMergeJoin: join_type=Inner, on=[(Column { name: \"nullable_col\", index: 0 }, Column { name: \"col_a\", index: 0 })]",
" SortMergeJoin: join_type=Inner, on=[(nullable_col@0, col_a@0)]",
" SortExec: expr=[nullable_col@0 ASC]",
" ParquetExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col]",
" SortExec: expr=[col_a@0 ASC]",
Expand Down
10 changes: 8 additions & 2 deletions datafusion/core/src/physical_plan/joins/hash_join.rs
Original file line number Diff line number Diff line change
Expand Up @@ -423,10 +423,16 @@ impl ExecutionPlan for HashJoinExec {
|| "".to_string(),
|f| format!(", filter={}", f.expression()),
);
let on = self
.on
.iter()
.map(|(c1, c2)| format!("({}, {})", c1, c2))
.collect::<Vec<String>>()
.join(", ");
write!(
f,
"HashJoinExec: mode={:?}, join_type={:?}, on={:?}{}",
self.mode, self.join_type, self.on, display_filter
"HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}",
self.mode, self.join_type, on, display_filter
)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ impl ExecutionPlan for NestedLoopJoinExec {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
let display_filter = self.filter.as_ref().map_or_else(
|| "".to_string(),
|f| format!(", filter={:?}", f.expression()),
|f| format!(", filter={}", f.expression()),
);
write!(
f,
Expand Down
10 changes: 8 additions & 2 deletions datafusion/core/src/physical_plan/joins/sort_merge_join.rs
Original file line number Diff line number Diff line change
Expand Up @@ -364,10 +364,16 @@ impl ExecutionPlan for SortMergeJoinExec {
fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
match t {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
let on = self
.on
.iter()
.map(|(c1, c2)| format!("({}, {})", c1, c2))
.collect::<Vec<String>>()
.join(", ");
write!(
f,
"SortMergeJoin: join_type={:?}, on={:?}",
self.join_type, self.on
"SortMergeJoin: join_type={:?}, on=[{}]",
self.join_type, on
)
}
}
Expand Down
22 changes: 14 additions & 8 deletions datafusion/core/src/physical_plan/joins/symmetric_hash_join.rs
Original file line number Diff line number Diff line change
Expand Up @@ -467,10 +467,16 @@ impl ExecutionPlan for SymmetricHashJoinExec {
|| "".to_string(),
|f| format!(", filter={}", f.expression()),
);
let on = self
.on
.iter()
.map(|(c1, c2)| format!("({}, {})", c1, c2))
.collect::<Vec<String>>()
.join(", ");
write!(
f,
"SymmetricHashJoinExec: join_type={:?}, on={:?}{}",
self.join_type, self.on, display_filter
"SymmetricHashJoinExec: join_type={:?}, on=[{}]{}",
self.join_type, on, display_filter
)
}
}
Expand Down Expand Up @@ -2627,12 +2633,12 @@ mod tests {
let formatted = displayable(physical_plan.as_ref()).indent(true).to_string();
let expected = {
[
"SymmetricHashJoinExec: join_type=Full, on=[(Column { name: \"a2\", index: 1 }, Column { name: \"a2\", index: 1 })], filter=CAST(a1@0 AS Int64) > CAST(a1@1 AS Int64) + 3 AND CAST(a1@0 AS Int64) < CAST(a1@1 AS Int64) + 10",
"SymmetricHashJoinExec: join_type=Full, on=[(a2@1, a2@1)], filter=CAST(a1@0 AS Int64) > CAST(a1@1 AS Int64) + 3 AND CAST(a1@0 AS Int64) < CAST(a1@1 AS Int64) + 10",
" CoalesceBatchesExec: target_batch_size=8192",
" RepartitionExec: partitioning=Hash([Column { name: \"a2\", index: 1 }], 8), input_partitions=1",
" RepartitionExec: partitioning=Hash([a2@1], 8), input_partitions=1",
// " CsvExec: file_groups={1 group: [[tempdir/left.csv]]}, projection=[a1, a2], has_header=false",
" CoalesceBatchesExec: target_batch_size=8192",
" RepartitionExec: partitioning=Hash([Column { name: \"a2\", index: 1 }], 8), input_partitions=1",
" RepartitionExec: partitioning=Hash([a2@1], 8), input_partitions=1",
// " CsvExec: file_groups={1 group: [[tempdir/right.csv]]}, projection=[a1, a2], has_header=false"
]
};
Expand Down Expand Up @@ -2680,12 +2686,12 @@ mod tests {
let formatted = displayable(physical_plan.as_ref()).indent(true).to_string();
let expected = {
[
"SymmetricHashJoinExec: join_type=Full, on=[(Column { name: \"a2\", index: 1 }, Column { name: \"a2\", index: 1 })], filter=CAST(a1@0 AS Int64) > CAST(a1@1 AS Int64) + 3 AND CAST(a1@0 AS Int64) < CAST(a1@1 AS Int64) + 10",
"SymmetricHashJoinExec: join_type=Full, on=[(a2@1, a2@1)], filter=CAST(a1@0 AS Int64) > CAST(a1@1 AS Int64) + 3 AND CAST(a1@0 AS Int64) < CAST(a1@1 AS Int64) + 10",
" CoalesceBatchesExec: target_batch_size=8192",
" RepartitionExec: partitioning=Hash([Column { name: \"a2\", index: 1 }], 8), input_partitions=1",
" RepartitionExec: partitioning=Hash([a2@1], 8), input_partitions=1",
// " CsvExec: file_groups={1 group: [[tempdir/left.csv]]}, projection=[a1, a2], has_header=false",
" CoalesceBatchesExec: target_batch_size=8192",
" RepartitionExec: partitioning=Hash([Column { name: \"a2\", index: 1 }], 8), input_partitions=1",
" RepartitionExec: partitioning=Hash([a2@1], 8), input_partitions=1",
// " CsvExec: file_groups={1 group: [[tempdir/right.csv]]}, projection=[a1, a2], has_header=false"
]
};
Expand Down
18 changes: 18 additions & 0 deletions datafusion/core/src/physical_plan/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,24 @@ pub enum Partitioning {
UnknownPartitioning(usize),
}

impl fmt::Display for Partitioning {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Partitioning::RoundRobinBatch(size) => write!(f, "RoundRobinBatch({size})"),
Partitioning::Hash(phy_exprs, size) => {
let phy_exprs_str = phy_exprs
.iter()
.map(|e| format!("{e}"))
.collect::<Vec<String>>()
.join(", ");
write!(f, "Hash([{phy_exprs_str}], {size})")
}
Partitioning::UnknownPartitioning(size) => {
write!(f, "UnknownPartitioning({size})")
}
}
}
}
impl Partitioning {
/// Returns the number of partitions in this partitioning scheme
pub fn partition_count(&self) -> usize {
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/src/physical_plan/repartition/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ impl ExecutionPlan for RepartitionExec {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
write!(
f,
"RepartitionExec: partitioning={:?}, input_partitions={}",
"RepartitionExec: partitioning={}, input_partitions={}",
self.partitioning,
self.input.output_partitioning().partition_count()
)
Expand Down
6 changes: 3 additions & 3 deletions datafusion/core/src/physical_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,13 @@ fn create_physical_name(e: &Expr, is_first_expr: bool) -> Result<String> {
Expr::Case(case) => {
let mut name = "CASE ".to_string();
if let Some(e) = &case.expr {
let _ = write!(name, "{e:?} ");
let _ = write!(name, "{e} ");
}
for (w, t) in &case.when_then_expr {
let _ = write!(name, "WHEN {w:?} THEN {t:?} ");
let _ = write!(name, "WHEN {w} THEN {t} ");
}
if let Some(e) = &case.else_expr {
let _ = write!(name, "ELSE {e:?} ");
let _ = write!(name, "ELSE {e} ");
}
name += "END";
Ok(name)
Expand Down
8 changes: 4 additions & 4 deletions datafusion/core/tests/sql/explain_analyze.rs
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,7 @@ async fn test_physical_plan_display_indent() {
" ProjectionExec: expr=[c1@0 as c1, MAX(aggregate_test_100.c12)@1 as MAX(aggregate_test_100.c12), MIN(aggregate_test_100.c12)@2 as the_min]",
" AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[MAX(aggregate_test_100.c12), MIN(aggregate_test_100.c12)]",
" CoalesceBatchesExec: target_batch_size=4096",
" RepartitionExec: partitioning=Hash([Column { name: \"c1\", index: 0 }], 9000), input_partitions=9000",
" RepartitionExec: partitioning=Hash([c1@0], 9000), input_partitions=9000",
" AggregateExec: mode=Partial, gby=[c1@0 as c1], aggr=[MAX(aggregate_test_100.c12), MIN(aggregate_test_100.c12)]",
" CoalesceBatchesExec: target_batch_size=4096",
" FilterExec: c12@1 < 10",
Expand Down Expand Up @@ -646,13 +646,13 @@ async fn test_physical_plan_display_indent_multi_children() {
let expected = vec![
"ProjectionExec: expr=[c1@0 as c1]",
" CoalesceBatchesExec: target_batch_size=4096",
" HashJoinExec: mode=Partitioned, join_type=Inner, on=[(Column { name: \"c1\", index: 0 }, Column { name: \"c2\", index: 0 })]",
" HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c1@0, c2@0)]",
" CoalesceBatchesExec: target_batch_size=4096",
" RepartitionExec: partitioning=Hash([Column { name: \"c1\", index: 0 }], 9000), input_partitions=9000",
" RepartitionExec: partitioning=Hash([c1@0], 9000), input_partitions=9000",
" RepartitionExec: partitioning=RoundRobinBatch(9000), input_partitions=1",
" CsvExec: file_groups={1 group: [[ARROW_TEST_DATA/csv/aggregate_test_100.csv]]}, projection=[c1], has_header=true",
" CoalesceBatchesExec: target_batch_size=4096",
" RepartitionExec: partitioning=Hash([Column { name: \"c2\", index: 0 }], 9000), input_partitions=9000",
" RepartitionExec: partitioning=Hash([c2@0], 9000), input_partitions=9000",
" RepartitionExec: partitioning=RoundRobinBatch(9000), input_partitions=1",
" ProjectionExec: expr=[c1@0 as c2]",
" CsvExec: file_groups={1 group: [[ARROW_TEST_DATA/csv/aggregate_test_100.csv]]}, projection=[c1], has_header=true",
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/tests/sql/order.rs
Original file line number Diff line number Diff line change
Expand Up @@ -206,15 +206,15 @@ ORDER BY 1, 2;
" ProjectionExec: expr=[Int64(0)@0 as m, t@1 as t]",
" AggregateExec: mode=FinalPartitioned, gby=[Int64(0)@0 as Int64(0), t@1 as t], aggr=[]",
" CoalesceBatchesExec: target_batch_size=8192",
" RepartitionExec: partitioning=Hash([Column { name: \"Int64(0)\", index: 0 }, Column { name: \"t\", index: 1 }], 2), input_partitions=2",
" RepartitionExec: partitioning=Hash([Int64(0)@0, t@1], 2), input_partitions=2",
" AggregateExec: mode=Partial, gby=[0 as Int64(0), t@0 as t], aggr=[]",
" RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1",
" ProjectionExec: expr=[column1@0 as t]",
" ValuesExec",
" ProjectionExec: expr=[Int64(1)@0 as m, t@1 as t]",
" AggregateExec: mode=FinalPartitioned, gby=[Int64(1)@0 as Int64(1), t@1 as t], aggr=[]",
" CoalesceBatchesExec: target_batch_size=8192",
" RepartitionExec: partitioning=Hash([Column { name: \"Int64(1)\", index: 0 }, Column { name: \"t\", index: 1 }], 2), input_partitions=2",
" RepartitionExec: partitioning=Hash([Int64(1)@0, t@1], 2), input_partitions=2",
" AggregateExec: mode=Partial, gby=[1 as Int64(1), t@0 as t], aggr=[]",
" RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1",
" ProjectionExec: expr=[column1@0 as t]",
Expand Down
Loading