From 0443adcdd81e4c53c913bcd3a0018b42e501b1a1 Mon Sep 17 00:00:00 2001 From: Wang Date: Wed, 26 Oct 2022 22:25:39 +0800 Subject: [PATCH 1/3] [Part1] Partition and Sort Enforcement, PhysicalExpr enhancement --- datafusion-cli/Cargo.lock | 1 + datafusion-cli/src/main.rs | 2 +- datafusion-cli/src/object_storage.rs | 12 +- datafusion/physical-expr/Cargo.toml | 1 + .../physical-expr/src/expressions/binary.rs | 25 ++ .../physical-expr/src/expressions/case.rs | 201 ++++++++++ .../physical-expr/src/expressions/cast.rs | 31 ++ .../physical-expr/src/expressions/column.rs | 22 + .../physical-expr/src/expressions/datetime.rs | 34 +- .../src/expressions/get_indexed_field.rs | 24 ++ .../physical-expr/src/expressions/in_list.rs | 52 ++- .../src/expressions/is_not_null.rs | 20 + .../physical-expr/src/expressions/is_null.rs | 20 + .../physical-expr/src/expressions/literal.rs | 23 +- .../physical-expr/src/expressions/mod.rs | 2 + .../physical-expr/src/expressions/negative.rs | 21 + .../physical-expr/src/expressions/no_op.rs | 87 ++++ .../physical-expr/src/expressions/not.rs | 21 + .../physical-expr/src/expressions/try_cast.rs | 24 ++ datafusion/physical-expr/src/physical_expr.rs | 375 +++++++++++++++++- .../physical-expr/src/scalar_function.rs | 22 + datafusion/physical-expr/src/sort_expr.rs | 6 + 22 files changed, 1011 insertions(+), 15 deletions(-) create mode 100644 datafusion/physical-expr/src/expressions/no_op.rs diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 364c412fb02de..8c43a0eaa3c0b 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -649,6 +649,7 @@ dependencies = [ "datafusion-expr", "datafusion-row", "hashbrown", + "itertools", "lazy_static", "md-5", "ordered-float 3.2.0", diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index cbb74bbf9b4ce..0944564f88928 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -147,7 +147,7 @@ fn create_runtime_env() -> Result { ObjectStoreRegistry::new_with_provider(Some(Arc::new(object_store_provider))); let rn_config = RuntimeConfig::new().with_object_store_registry(Arc::new(object_store_registry)); - return RuntimeEnv::new(rn_config); + RuntimeEnv::new(rn_config) } fn is_valid_file(dir: &str) -> std::result::Result<(), String> { diff --git a/datafusion-cli/src/object_storage.rs b/datafusion-cli/src/object_storage.rs index 19993e751c26d..df5235328a2cc 100644 --- a/datafusion-cli/src/object_storage.rs +++ b/datafusion-cli/src/object_storage.rs @@ -68,7 +68,7 @@ fn build_gcs_object_store(url: &Url) -> Result Result Result<&str> { - url.host_str().ok_or(DataFusionError::Execution(format!( - "Not able to parse hostname from url, {}", - url.as_str() - ))) + url.host_str().ok_or_else(|| { + DataFusionError::Execution(format!( + "Not able to parse hostname from url, {}", + url.as_str() + )) + }) } #[cfg(test)] diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index e0703829d394a..d281b9543b281 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -48,6 +48,7 @@ datafusion-common = { path = "../common", version = "13.0.0" } datafusion-expr = { path = "../expr", version = "13.0.0" } datafusion-row = { path = "../row", version = "13.0.0" } hashbrown = { version = "0.12", features = ["raw"] } +itertools = { version = "0.10", features = ["use_std"] } lazy_static = { version = "^1.4.0" } md-5 = { version = "^0.10.0", optional = true } ordered-float = "3.0" diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index d6c1240a0d87b..8b93f49c1a034 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -74,6 +74,7 @@ use kernels_arrow::{ use arrow::datatypes::{DataType, Schema, TimeUnit}; use arrow::record_batch::RecordBatch; +use crate::physical_expr::down_cast_any_ref; use crate::{ExprBoundaries, PhysicalExpr, PhysicalExprStats}; use datafusion_common::{ColumnStatistics, ScalarValue}; use datafusion_common::{DataFusionError, Result}; @@ -649,6 +650,30 @@ impl PhysicalExpr for BinaryExpr { right: Arc::clone(self.right()), }) } + + fn children(&self) -> Vec> { + vec![self.left.clone(), self.right.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(BinaryExpr::new( + children[0].clone(), + self.op, + children[1].clone(), + ))) + } +} + +impl PartialEq for BinaryExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.left.eq(&x.left) && self.op == x.op && self.right.eq(&x.right)) + .unwrap_or(false) + } } struct BinaryExprStats { diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs index 581d9abdce382..9bac8479ebf48 100644 --- a/datafusion/physical-expr/src/expressions/case.rs +++ b/datafusion/physical-expr/src/expressions/case.rs @@ -18,6 +18,8 @@ use std::{any::Any, sync::Arc}; use crate::expressions::try_cast; +use crate::expressions::NoOp; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use arrow::array::*; use arrow::compute::kernels::zip::zip; @@ -27,6 +29,8 @@ use arrow::record_batch::RecordBatch; use datafusion_common::{DataFusionError, Result}; use datafusion_expr::ColumnarValue; +use itertools::Itertools; + type WhenThen = (Arc, Arc); /// The CASE expression is similar to a series of nested if/else and there are two forms that @@ -286,6 +290,89 @@ impl PhysicalExpr for CaseExpr { self.case_when_no_expr(batch) } } + + fn children(&self) -> Vec> { + let mut chileren = vec![]; + match &self.expr { + Some(expr) => chileren.push(expr.clone()), + None => chileren.push(Arc::new(NoOp::new())), + } + self.when_then_expr.iter().for_each(|(cond, value)| { + chileren.push(cond.clone()); + chileren.push(value.clone()); + }); + + match &self.else_expr { + Some(expr) => chileren.push(expr.clone()), + None => chileren.push(Arc::new(NoOp::new())), + } + chileren + } + + // For physical CaseExpr, we do not allow modifying children size + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + if children.len() != self.children().len() { + Err(DataFusionError::Internal( + "CaseExpr: Wrong number of children".to_string(), + )) + } else { + assert_eq!(children.len() % 2, 0); + let expr = match children[0].clone().as_any().downcast_ref::() { + Some(_) => None, + _ => Some(children[0].clone()), + }; + let else_expr = match children[children.len() - 1] + .clone() + .as_any() + .downcast_ref::() + { + Some(_) => None, + _ => Some(children[children.len() - 1].clone()), + }; + + let branches = children[1..children.len() - 1].to_vec(); + let mut when_then_expr: Vec = vec![]; + for (prev, next) in branches.into_iter().tuples() { + when_then_expr.push((prev, next)); + } + Ok(Arc::new(CaseExpr::try_new( + expr, + when_then_expr, + else_expr, + )?)) + } + } +} + +impl PartialEq for CaseExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| { + let expr_eq = match (&self.expr, &x.expr) { + (Some(expr1), Some(expr2)) => expr1.eq(expr2), + (None, None) => true, + _ => false, + }; + let else_expr_eq = match (&self.else_expr, &x.else_expr) { + (Some(expr1), Some(expr2)) => expr1.eq(expr2), + (None, None) => true, + _ => false, + }; + expr_eq + && else_expr_eq + && self.when_then_expr.len() == x.when_then_expr.len() + && self.when_then_expr.iter().zip(x.when_then_expr.iter()).all( + |((when1, then1), (when2, then2))| { + when1.eq(when2) && then1.eq(then2) + }, + ) + }) + .unwrap_or(false) + } } /// Create a CASE expression @@ -303,6 +390,7 @@ mod tests { use crate::expressions::col; use crate::expressions::lit; use crate::expressions::{binary, cast}; + use crate::physical_expr::TreeNodeRewritable; use arrow::array::StringArray; use arrow::buffer::Buffer; use arrow::datatypes::DataType::Float64; @@ -728,6 +816,119 @@ mod tests { Ok(()) } + #[test] + fn case_eq() -> Result<()> { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + + let when1 = lit("foo"); + let then1 = lit(123i32); + let when2 = lit("bar"); + let then2 = lit(456i32); + let else_value = lit(999i32); + + let expr1 = generate_case_when_with_type_coercion( + Some(col("a", &schema)?), + vec![ + (when1.clone(), then1.clone()), + (when2.clone(), then2.clone()), + ], + Some(else_value.clone()), + &schema, + )?; + + let expr2 = generate_case_when_with_type_coercion( + Some(col("a", &schema)?), + vec![ + (when1.clone(), then1.clone()), + (when2.clone(), then2.clone()), + ], + Some(else_value.clone()), + &schema, + )?; + + let expr3 = generate_case_when_with_type_coercion( + Some(col("a", &schema)?), + vec![(when1.clone(), then1.clone()), (when2, then2)], + None, + &schema, + )?; + + let expr4 = generate_case_when_with_type_coercion( + Some(col("a", &schema)?), + vec![(when1, then1)], + Some(else_value), + &schema, + )?; + + assert!(expr1.eq(&expr2)); + assert!(expr2.eq(&expr1)); + + assert!(expr2.ne(&expr3)); + assert!(expr3.ne(&expr2)); + + assert!(expr1.ne(&expr4)); + assert!(expr4.ne(&expr1)); + + Ok(()) + } + + #[test] + fn case_tranform() -> Result<()> { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + + let when1 = lit("foo"); + let then1 = lit(123i32); + let when2 = lit("bar"); + let then2 = lit(456i32); + let else_value = lit(999i32); + + let expr = generate_case_when_with_type_coercion( + Some(col("a", &schema)?), + vec![ + (when1.clone(), then1.clone()), + (when2.clone(), then2.clone()), + ], + Some(else_value.clone()), + &schema, + )?; + + let expr2 = expr + .clone() + .transform( + &|e| match e.as_any().downcast_ref::() { + Some(lit_value) => match lit_value.value() { + ScalarValue::Utf8(Some(str_value)) => { + Some(lit(str_value.to_uppercase())) + } + _ => None, + }, + _ => None, + }, + ) + .unwrap(); + + let expr3 = expr + .clone() + .transform_down(&|e| match e + .as_any() + .downcast_ref::() + { + Some(lit_value) => match lit_value.value() { + ScalarValue::Utf8(Some(str_value)) => { + Some(lit(str_value.to_uppercase())) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + + assert!(expr.ne(&expr2)); + assert!(expr2.eq(&expr3)); + + Ok(()) + } + fn generate_case_when_with_type_coercion( expr: Option>, when_thens: Vec, diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 99d2859e710af..02a110f5aae07 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -19,6 +19,7 @@ use std::any::Any; use std::fmt; use std::sync::Arc; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use arrow::compute; use arrow::compute::kernels; @@ -93,6 +94,36 @@ impl PhysicalExpr for CastExpr { let value = self.expr.evaluate(batch)?; cast_column(&value, &self.cast_type, &self.cast_options) } + + fn children(&self) -> Vec> { + vec![self.expr.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(CastExpr::new( + children[0].clone(), + self.cast_type.clone(), + CastOptions { + safe: self.cast_options.safe, + }, + ))) + } +} + +impl PartialEq for CastExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| { + self.expr.eq(&x.expr) + && self.cast_type == x.cast_type + && self.cast_options.safe == x.cast_options.safe + }) + .unwrap_or(false) + } } /// Internal cast function for casting ColumnarValue -> ColumnarValue for cast_type diff --git a/datafusion/physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rs index dfb6eca40d3ff..450919aa3f2cc 100644 --- a/datafusion/physical-expr/src/expressions/column.rs +++ b/datafusion/physical-expr/src/expressions/column.rs @@ -17,6 +17,7 @@ //! Column expression +use std::any::Any; use std::sync::Arc; use arrow::{ @@ -24,6 +25,7 @@ use arrow::{ record_batch::RecordBatch, }; +use crate::physical_expr::down_cast_any_ref; use crate::{ExprBoundaries, PhysicalExpr, PhysicalExprStats}; use datafusion_common::{ColumnStatistics, DataFusionError, Result}; use datafusion_expr::ColumnarValue; @@ -94,6 +96,26 @@ impl PhysicalExpr for Column { fn expr_stats(&self) -> Arc { Arc::new(ColumnExprStats { index: self.index }) } + + fn children(&self) -> Vec> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> Result> { + Ok(self) + } +} + +impl PartialEq for Column { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self == x) + .unwrap_or(false) + } } #[derive(Debug, Clone)] diff --git a/datafusion/physical-expr/src/expressions/datetime.rs b/datafusion/physical-expr/src/expressions/datetime.rs index fa021f61a940f..3c7096c7256cd 100644 --- a/datafusion/physical-expr/src/expressions/datetime.rs +++ b/datafusion/physical-expr/src/expressions/datetime.rs @@ -16,6 +16,7 @@ // under the License. use crate::expressions::delta::shift_months; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use arrow::array::{ Array, ArrayRef, Date32Array, Date64Array, TimestampMicrosecondArray, @@ -42,6 +43,7 @@ pub struct DateTimeIntervalExpr { lhs: Arc, op: Operator, rhs: Arc, + input_schema: Schema, } impl DateTimeIntervalExpr { @@ -56,7 +58,12 @@ impl DateTimeIntervalExpr { DataType::Date32 | DataType::Date64 | DataType::Timestamp(_, _) => { match rhs.data_type(input_schema)? { DataType::Interval(_) => match &op { - Operator::Plus | Operator::Minus => Ok(Self { lhs, op, rhs }), + Operator::Plus | Operator::Minus => Ok(Self { + lhs, + op, + rhs, + input_schema: input_schema.clone(), + }), _ => Err(DataFusionError::Execution(format!( "Invalid operator '{}' for DateIntervalExpr", op @@ -140,6 +147,31 @@ impl PhysicalExpr for DateTimeIntervalExpr { ColumnarValue::Array(array) => evaluate_array(array, sign, intervals), } } + + fn children(&self) -> Vec> { + vec![self.lhs.clone(), self.rhs.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(DateTimeIntervalExpr::try_new( + children[0].clone(), + self.op, + children[1].clone(), + &self.input_schema, + )?)) + } +} + +impl PartialEq for DateTimeIntervalExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.lhs.eq(&x.lhs) && self.op == x.op && self.rhs.eq(&x.rhs)) + .unwrap_or(false) + } } pub fn evaluate_array( diff --git a/datafusion/physical-expr/src/expressions/get_indexed_field.rs b/datafusion/physical-expr/src/expressions/get_indexed_field.rs index 5d9b1594d4383..ff10c06e2206f 100644 --- a/datafusion/physical-expr/src/expressions/get_indexed_field.rs +++ b/datafusion/physical-expr/src/expressions/get_indexed_field.rs @@ -22,6 +22,7 @@ use arrow::array::Array; use arrow::array::{ListArray, StructArray}; use arrow::compute::concat; +use crate::physical_expr::down_cast_any_ref; use arrow::{ datatypes::{DataType, Schema}, record_batch::RecordBatch, @@ -132,6 +133,29 @@ impl PhysicalExpr for GetIndexedFieldExpr { (dt, key) => Err(DataFusionError::Execution(format!("get indexed field is only possible on lists with int64 indexes or struct with utf8 indexes. Tried {:?} with {:?} index", dt, key))), } } + + fn children(&self) -> Vec> { + vec![self.arg.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(GetIndexedFieldExpr::new( + children[0].clone(), + self.key.clone(), + ))) + } +} + +impl PartialEq for GetIndexedFieldExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.arg.eq(&x.arg) && self.key == x.key) + .unwrap_or(false) + } } #[cfg(test)] diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 2d34115220ece..a43b6042b2485 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -19,6 +19,7 @@ use std::any::Any; use std::collections::HashSet; +use std::fmt::Debug; use std::sync::Arc; use arrow::array::GenericStringArray; @@ -33,6 +34,7 @@ use arrow::{ record_batch::RecordBatch, }; +use crate::physical_expr::{down_cast_any_ref, expr_list_eq_any_order}; use crate::PhysicalExpr; use arrow::array::*; use arrow::datatypes::TimeUnit; @@ -52,16 +54,27 @@ use datafusion_expr::ColumnarValue; static OPTIMIZER_INSET_THRESHOLD: usize = 30; /// InList -#[derive(Debug)] pub struct InListExpr { expr: Arc, list: Vec>, negated: bool, set: Option, + input_schema: Schema, +} + +impl Debug for InListExpr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("InListExpr") + .field("expr", &self.expr) + .field("list", &self.list) + .field("negated", &self.negated) + .field("set", &self.set) + .finish() + } } /// InSet -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct InSet { // TODO: optimization: In the `IN` or `NOT IN` we don't need to consider the NULL value // The data type is same, we can use set: HashSet @@ -386,6 +399,7 @@ impl InListExpr { set: Some(InSet::new(set)), list, negated, + input_schema: schema.clone(), }; } } @@ -394,6 +408,7 @@ impl InListExpr { list, negated, set: None, + input_schema: schema.clone(), } } @@ -899,6 +914,39 @@ impl PhysicalExpr for InListExpr { } } } + + fn children(&self) -> Vec> { + let mut children = vec![]; + children.push(self.expr.clone()); + children.extend(self.list.clone()); + children + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + in_list( + children[0].clone(), + children[1..].to_vec(), + &self.negated, + &self.input_schema, + ) + } +} + +impl PartialEq for InListExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| { + self.expr.eq(&x.expr) + && expr_list_eq_any_order(&self.list, &x.list) + && self.negated == x.negated + && self.set == x.set + }) + .unwrap_or(false) + } } /// Creates a unary expression InList diff --git a/datafusion/physical-expr/src/expressions/is_not_null.rs b/datafusion/physical-expr/src/expressions/is_not_null.rs index 6b614f3d98ca1..b5003ac232c2d 100644 --- a/datafusion/physical-expr/src/expressions/is_not_null.rs +++ b/datafusion/physical-expr/src/expressions/is_not_null.rs @@ -19,6 +19,7 @@ use std::{any::Any, sync::Arc}; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use arrow::compute; use arrow::{ @@ -79,8 +80,27 @@ impl PhysicalExpr for IsNotNullExpr { )), } } + + fn children(&self) -> Vec> { + vec![self.arg.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(IsNotNullExpr::new(children[0].clone()))) + } } +impl PartialEq for IsNotNullExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.arg.eq(&x.arg)) + .unwrap_or(false) + } +} /// Create an IS NOT NULL expression pub fn is_not_null(arg: Arc) -> Result> { Ok(Arc::new(IsNotNullExpr::new(arg))) diff --git a/datafusion/physical-expr/src/expressions/is_null.rs b/datafusion/physical-expr/src/expressions/is_null.rs index e5dbfbdc74819..976df2bc04c87 100644 --- a/datafusion/physical-expr/src/expressions/is_null.rs +++ b/datafusion/physical-expr/src/expressions/is_null.rs @@ -25,6 +25,7 @@ use arrow::{ record_batch::RecordBatch, }; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use datafusion_common::Result; use datafusion_common::ScalarValue; @@ -80,8 +81,27 @@ impl PhysicalExpr for IsNullExpr { )), } } + + fn children(&self) -> Vec> { + vec![self.arg.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(IsNullExpr::new(children[0].clone()))) + } } +impl PartialEq for IsNullExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.arg.eq(&x.arg)) + .unwrap_or(false) + } +} /// Create an IS NULL expression pub fn is_null(arg: Arc) -> Result> { Ok(Arc::new(IsNullExpr::new(arg))) diff --git a/datafusion/physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rs index b3e4a7a320aca..7957aaef240e8 100644 --- a/datafusion/physical-expr/src/expressions/literal.rs +++ b/datafusion/physical-expr/src/expressions/literal.rs @@ -25,13 +25,14 @@ use arrow::{ record_batch::RecordBatch, }; +use crate::physical_expr::down_cast_any_ref; use crate::{ExprBoundaries, PhysicalExpr, PhysicalExprStats}; use datafusion_common::ScalarValue; use datafusion_common::{ColumnStatistics, Result}; use datafusion_expr::{ColumnarValue, Expr}; /// Represents a literal value -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct Literal { value: ScalarValue, } @@ -77,6 +78,26 @@ impl PhysicalExpr for Literal { value: self.value.clone(), }) } + + fn children(&self) -> Vec> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> Result> { + Ok(self) + } +} + +impl PartialEq for Literal { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self == x) + .unwrap_or(false) + } } struct LiteralExprStats { diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index 208e6d0b51fba..e97482075dc27 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -30,6 +30,7 @@ mod is_not_null; mod is_null; mod literal; mod negative; +mod no_op; mod not; mod nullif; mod try_cast; @@ -80,6 +81,7 @@ pub use is_not_null::{is_not_null, IsNotNullExpr}; pub use is_null::{is_null, IsNullExpr}; pub use literal::{lit, Literal}; pub use negative::{negative, NegativeExpr}; +pub use no_op::NoOp; pub use not::{not, NotExpr}; pub use nullif::nullif_func; pub use try_cast::{try_cast, TryCastExpr}; diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs index 4f4def99e8210..1fed005ce3b28 100644 --- a/datafusion/physical-expr/src/expressions/negative.rs +++ b/datafusion/physical-expr/src/expressions/negative.rs @@ -28,6 +28,7 @@ use arrow::{ record_batch::RecordBatch, }; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use datafusion_common::{DataFusionError, Result}; use datafusion_expr::{ @@ -110,6 +111,26 @@ impl PhysicalExpr for NegativeExpr { } } } + + fn children(&self) -> Vec> { + vec![self.arg.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(NegativeExpr::new(children[0].clone()))) + } +} + +impl PartialEq for NegativeExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.arg.eq(&x.arg)) + .unwrap_or(false) + } } /// Creates a unary expression NEGATIVE diff --git a/datafusion/physical-expr/src/expressions/no_op.rs b/datafusion/physical-expr/src/expressions/no_op.rs new file mode 100644 index 0000000000000..b81bcfc30b905 --- /dev/null +++ b/datafusion/physical-expr/src/expressions/no_op.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Literal expressions for physical operations + +use std::any::Any; +use std::sync::Arc; + +use arrow::{ + datatypes::{DataType, Schema}, + record_batch::RecordBatch, +}; + +use crate::physical_expr::down_cast_any_ref; +use crate::PhysicalExpr; +use datafusion_common::Result; +use datafusion_expr::ColumnarValue; + +/// A place holder expressions, can not be evaluated +#[derive(Debug, PartialEq, Eq, Default)] +pub struct NoOp {} + +impl NoOp { + /// Create a NoOp expression + pub fn new() -> Self { + Self {} + } +} + +impl std::fmt::Display for NoOp { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "NoOp") + } +} + +impl PhysicalExpr for NoOp { + /// Return a reference to Any that can be used for downcasting + fn as_any(&self) -> &dyn Any { + self + } + + fn data_type(&self, _input_schema: &Schema) -> Result { + Ok(DataType::Null) + } + + fn nullable(&self, _input_schema: &Schema) -> Result { + Ok(true) + } + + fn evaluate(&self, _batch: &RecordBatch) -> Result { + unimplemented!("NoOp::evaluate"); + } + + fn children(&self) -> Vec> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> Result> { + Ok(self) + } +} + +impl PartialEq for NoOp { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self == x) + .unwrap_or(false) + } +} diff --git a/datafusion/physical-expr/src/expressions/not.rs b/datafusion/physical-expr/src/expressions/not.rs index a7fba60ec362a..00f1670af7dc3 100644 --- a/datafusion/physical-expr/src/expressions/not.rs +++ b/datafusion/physical-expr/src/expressions/not.rs @@ -21,6 +21,7 @@ use std::any::Any; use std::fmt; use std::sync::Arc; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use arrow::array::BooleanArray; use arrow::datatypes::{DataType, Schema}; @@ -103,6 +104,26 @@ impl PhysicalExpr for NotExpr { } } } + + fn children(&self) -> Vec> { + vec![self.arg.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(NotExpr::new(children[0].clone()))) + } +} + +impl PartialEq for NotExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.arg.eq(&x.arg)) + .unwrap_or(false) + } } /// Creates a unary expression NOT diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs index 6a03a6b5306c3..299d29ec6072d 100644 --- a/datafusion/physical-expr/src/expressions/try_cast.rs +++ b/datafusion/physical-expr/src/expressions/try_cast.rs @@ -19,6 +19,7 @@ use std::any::Any; use std::fmt; use std::sync::Arc; +use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; use arrow::compute; use arrow::compute::kernels; @@ -90,6 +91,29 @@ impl PhysicalExpr for TryCastExpr { } } } + + fn children(&self) -> Vec> { + vec![self.expr.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(TryCastExpr::new( + children[0].clone(), + self.cast_type.clone(), + ))) + } +} + +impl PartialEq for TryCastExpr { + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| self.expr.eq(&x.expr) && self.cast_type == x.cast_type) + .unwrap_or(false) + } } /// Return a PhysicalExpression representing `expr` casted to diff --git a/datafusion/physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rs index df82cc07f1eeb..21a27700f8981 100644 --- a/datafusion/physical-expr/src/physical_expr.rs +++ b/datafusion/physical-expr/src/physical_expr.rs @@ -19,19 +19,20 @@ use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; -use datafusion_common::{ColumnStatistics, Result, ScalarValue}; - +use crate::PhysicalSortExpr; +use datafusion_common::{ColumnStatistics, DataFusionError, Result, ScalarValue}; use datafusion_expr::ColumnarValue; -use std::fmt::{Debug, Display}; -use std::sync::Arc; use arrow::array::{make_array, Array, ArrayRef, BooleanArray, MutableArrayData}; use arrow::compute::{and_kleene, filter_record_batch, is_not_null, SlicesIterator}; + use std::any::Any; +use std::fmt::{Debug, Display}; +use std::sync::Arc; /// Expression that can be evaluated against a RecordBatch /// A Physical expression knows its type, nullability and how to evaluate itself. -pub trait PhysicalExpr: Send + Sync + Display + Debug { +pub trait PhysicalExpr: Send + Sync + Display + Debug + PartialEq { /// Returns the physical expression as [`Any`](std::any::Any) so that it can be /// downcast to a specific implementation. fn as_any(&self) -> &dyn Any; @@ -66,6 +67,14 @@ pub trait PhysicalExpr: Send + Sync + Display + Debug { fn expr_stats(&self) -> Arc { Arc::new(BasicExpressionStats {}) } + /// Get a list of child PhysicalExpr that provide the input for this expr. + fn children(&self) -> Vec>; + + /// Returns a new PhysicalExpr where all children were replaced by new exprs. + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result>; } /// Statistics about the result of a single expression. @@ -128,6 +137,232 @@ impl PhysicalExprStats for BasicExpressionStats { } } +/// a Trait for marking tree node types that are rewritable +pub trait TreeNodeRewritable: Clone { + /// Transform the tree node using the given [TreeNodeRewriter] + /// It performs a depth first walk of an node and its children. + /// + /// For an node tree such as + /// ```text + /// ParentNode + /// left: ChildNode1 + /// right: ChildNode2 + /// ``` + /// + /// The nodes are visited using the following order + /// ```text + /// pre_visit(ParentNode) + /// pre_visit(ChildNode1) + /// mutatate(ChildNode1) + /// pre_visit(ChildNode2) + /// mutate(ChildNode2) + /// mutate(ParentNode) + /// ``` + /// + /// If an Err result is returned, recursion is stopped immediately + /// + /// If [`false`] is returned on a call to pre_visit, no + /// children of that node are visited, nor is mutate + /// called on that node + /// + fn transform_using>( + self, + rewriter: &mut R, + ) -> Result { + let need_mutate = match rewriter.pre_visit(&self)? { + RewriteRecursion::Mutate => return rewriter.mutate(self), + RewriteRecursion::Stop => return Ok(self), + RewriteRecursion::Continue => true, + RewriteRecursion::Skip => false, + }; + + let after_op_children = + self.map_children(|node| node.transform_using(rewriter))?; + + // now rewrite this node itself + if need_mutate { + rewriter.mutate(after_op_children) + } else { + Ok(after_op_children) + } + } + + /// Convenience utils for writing optimizers rule: recursively apply the given `op` to the node tree. + /// When `op` does not apply to a given node, it is left uncshanged. + /// The default tree traversal direction is transform_up(Postorder Traversal). + fn transform(self, op: &F) -> Result + where + F: Fn(Self) -> Option, + { + self.transform_up(op) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given 'op' to the node and all of its + /// children(Preorder Traversal). + /// When the `op` does not apply to a given node, it is left unchanged. + fn transform_down(self, op: &F) -> Result + where + F: Fn(Self) -> Option, + { + let node_cloned = self.clone(); + let after_op = match op(node_cloned) { + Some(value) => value, + None => self, + }; + after_op.map_children(|node| node.transform_down(op)) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given 'op' first to all of its + /// children and then itself(Postorder Traversal). + /// When the `op` does not apply to a given node, it is left unchanged. + fn transform_up(self, op: &F) -> Result + where + F: Fn(Self) -> Option, + { + let after_op_children = self.map_children(|node| node.transform_up(op))?; + + let after_op_children_clone = after_op_children.clone(); + let new_node = match op(after_op_children) { + Some(value) => value, + None => after_op_children_clone, + }; + Ok(new_node) + } + + /// Apply transform `F` to the node's children, the transform `F` might have a direction(Preorder or Postorder) + fn map_children(self, transform: F) -> Result + where + F: FnMut(Self) -> Result; +} + +/// Trait for potentially recursively transform an [`TreeNodeRewritable`] node +/// tree. When passed to `TreeNodeRewritable::transform_using`, `TreeNodeRewriter::mutate` is +/// invoked recursively on all nodes of a tree. +pub trait TreeNodeRewriter: Sized { + /// Invoked before (Preorder) any children of `node` are rewritten / + /// visited. Default implementation returns `Ok(RewriteRecursion::Continue)` + fn pre_visit(&mut self, _node: &N) -> Result { + Ok(RewriteRecursion::Continue) + } + + /// Invoked after (Postorder) all children of `node` have been mutated and + /// returns a potentially modified node. + fn mutate(&mut self, node: N) -> Result; +} + +/// Controls how the [TreeNodeRewriter] recursion should proceed. +#[allow(dead_code)] +pub enum RewriteRecursion { + /// Continue rewrite / visit this node tree. + Continue, + /// Call 'op' immediately and return. + Mutate, + /// Do not rewrite / visit the children of this node. + Stop, + /// Keep recursive but skip apply op on this node + Skip, +} + +impl TreeNodeRewritable for Arc { + fn map_children(self, transform: F) -> Result + where + F: FnMut(Self) -> Result, + { + if !self.children().is_empty() { + let new_children: Result> = + self.children().into_iter().map(transform).collect(); + with_new_children_if_necessary(self, new_children?) + } else { + Ok(self) + } + } +} + +/// Returns a copy of this expr if we change any child according to the pointer comparison. +/// The size of `children` must be equal to the size of `PhysicalExpr::children()`. +/// Allow the vtable address comparisons for PhysicalExpr Trait Objects,it is harmless even +/// in the case of 'false-native'. +#[allow(clippy::vtable_address_comparisons)] +pub fn with_new_children_if_necessary( + expr: Arc, + children: Vec>, +) -> Result> { + if children.len() != expr.children().len() { + Err(DataFusionError::Internal( + "PhysicalExpr: Wrong number of children".to_string(), + )) + } else if children.is_empty() + || children + .iter() + .zip(expr.children().iter()) + .any(|(c1, c2)| !Arc::ptr_eq(c1, c2)) + { + expr.with_new_children(children) + } else { + Ok(expr) + } +} + +/// Compare the two expr lists are equal no matter the order. +/// For example two InListExpr can be considered to be equals no matter the order: +/// +/// In('a','b','c') == In('c','b','a') +pub fn expr_list_eq_any_order( + list1: &[Arc], + list2: &[Arc], +) -> bool { + if list1.len() == list2.len() { + let mut expr_vec1 = list1.to_vec(); + let mut expr_vec2 = list2.to_vec(); + while let Some(expr1) = expr_vec1.pop() { + if let Some(idx) = expr_vec2.iter().position(|expr2| expr1.eq(expr2)) { + expr_vec2.swap_remove(idx); + } else { + break; + } + } + expr_vec1.is_empty() && expr_vec2.is_empty() + } else { + false + } +} + +/// Strictly compare the two expr lists are equal in the given order. +#[allow(dead_code)] +pub fn expr_list_eq_strict_order( + list1: &[Arc], + list2: &[Arc], +) -> bool { + list1.len() == list2.len() && list1.iter().zip(list2.iter()).all(|(e1, e2)| e1.eq(e2)) +} + +/// Strictly compare the two sort expr lists in the given order. +/// +/// For Physical Sort Exprs, the order matters: +/// +/// SortExpr('a','b','c') != SortExpr('c','b','a') +#[allow(dead_code)] +pub fn sort_expr_list_eq_strict_order( + list1: &[PhysicalSortExpr], + list2: &[PhysicalSortExpr], +) -> bool { + list1.len() == list2.len() && list1.iter().zip(list2.iter()).all(|(e1, e2)| e1.eq(e2)) +} + +pub fn down_cast_any_ref(any: &dyn Any) -> &dyn Any { + if any.is::>() { + any.downcast_ref::>() + .unwrap() + .as_any() + } else if any.is::>() { + any.downcast_ref::>() + .unwrap() + .as_any() + } else { + any + } +} + /// Scatter `truthy` array by boolean mask. When the mask evaluates `true`, next values of `truthy` /// are taken, when the mask evaluates `false` values null values are filled. /// @@ -176,7 +411,10 @@ mod tests { use std::sync::Arc; use super::*; + use crate::expressions::Column; + use crate::PhysicalSortExpr; use arrow::array::Int32Array; + use arrow::compute::SortOptions; use datafusion_common::Result; #[test] @@ -271,4 +509,131 @@ mod tests { Ok(()) } + + #[test] + fn expr_list_eq_any_order_test() -> Result<()> { + let list1: Vec> = vec![ + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("b", 1)), + ]; + let list2: Vec> = vec![ + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("a", 0)), + ]; + assert!(!expr_list_eq_any_order(list1.as_slice(), list2.as_slice())); + assert!(!expr_list_eq_any_order(list2.as_slice(), list1.as_slice())); + + let list3: Vec> = vec![ + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("c", 2)), + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("b", 1)), + ]; + let list4: Vec> = vec![ + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("c", 2)), + Arc::new(Column::new("a", 0)), + ]; + assert!(expr_list_eq_any_order(list3.as_slice(), list4.as_slice())); + assert!(expr_list_eq_any_order(list4.as_slice(), list3.as_slice())); + assert!(expr_list_eq_any_order(list3.as_slice(), list3.as_slice())); + assert!(expr_list_eq_any_order(list4.as_slice(), list4.as_slice())); + + Ok(()) + } + + #[test] + fn sort_expr_list_eq_strict_order_test() -> Result<()> { + let list1: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + ]; + + let list2: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + ]; + + assert!(!sort_expr_list_eq_strict_order( + list1.as_slice(), + list2.as_slice() + )); + assert!(!sort_expr_list_eq_strict_order( + list2.as_slice(), + list1.as_slice() + )); + + let list3: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("c", 2)), + options: SortOptions::default(), + }, + ]; + let list4: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("c", 2)), + options: SortOptions::default(), + }, + ]; + + assert!(sort_expr_list_eq_strict_order( + list3.as_slice(), + list4.as_slice() + )); + assert!(sort_expr_list_eq_strict_order( + list4.as_slice(), + list3.as_slice() + )); + assert!(sort_expr_list_eq_strict_order( + list3.as_slice(), + list3.as_slice() + )); + assert!(sort_expr_list_eq_strict_order( + list4.as_slice(), + list4.as_slice() + )); + + Ok(()) + } } diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index 1350d49510d58..93968b60f8b2d 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -145,4 +145,26 @@ impl PhysicalExpr for ScalarFunctionExpr { let fun = self.fun.as_ref(); (fun)(&inputs) } + + fn children(&self) -> Vec> { + self.args.clone() + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(ScalarFunctionExpr::new( + &self.name, + self.fun.clone(), + children, + self.return_type(), + ))) + } +} + +impl PartialEq for ScalarFunctionExpr { + fn eq(&self, _other: &dyn Any) -> bool { + false + } } diff --git a/datafusion/physical-expr/src/sort_expr.rs b/datafusion/physical-expr/src/sort_expr.rs index 79656725d4f44..a173cc8ba8b3b 100644 --- a/datafusion/physical-expr/src/sort_expr.rs +++ b/datafusion/physical-expr/src/sort_expr.rs @@ -33,6 +33,12 @@ pub struct PhysicalSortExpr { pub options: SortOptions, } +impl PartialEq for PhysicalSortExpr { + fn eq(&self, other: &PhysicalSortExpr) -> bool { + self.options == other.options && self.expr.eq(&other.expr) + } +} + impl std::fmt::Display for PhysicalSortExpr { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { let opts_string = match (self.options.descending, self.options.nulls_first) { From 1c05ac8522774630cc72846e8110ab89982ef356 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 27 Oct 2022 23:28:29 +0800 Subject: [PATCH 2/3] Resolve review comments --- .../physical-expr/src/expressions/case.rs | 18 +- .../physical-expr/src/expressions/datetime.rs | 2 + .../physical-expr/src/expressions/in_list.rs | 3 +- .../physical-expr/src/expressions/no_op.rs | 12 +- datafusion/physical-expr/src/lib.rs | 2 + datafusion/physical-expr/src/physical_expr.rs | 318 ------------------ datafusion/physical-expr/src/rewrite.rs | 163 +++++++++ .../physical-expr/src/scalar_function.rs | 14 +- datafusion/physical-expr/src/utils.rs | 205 +++++++++++ 9 files changed, 403 insertions(+), 334 deletions(-) create mode 100644 datafusion/physical-expr/src/rewrite.rs create mode 100644 datafusion/physical-expr/src/utils.rs diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs index 9bac8479ebf48..e1da3dd2b2101 100644 --- a/datafusion/physical-expr/src/expressions/case.rs +++ b/datafusion/physical-expr/src/expressions/case.rs @@ -292,21 +292,21 @@ impl PhysicalExpr for CaseExpr { } fn children(&self) -> Vec> { - let mut chileren = vec![]; + let mut children = vec![]; match &self.expr { - Some(expr) => chileren.push(expr.clone()), - None => chileren.push(Arc::new(NoOp::new())), + Some(expr) => children.push(expr.clone()), + None => children.push(Arc::new(NoOp::new())), } self.when_then_expr.iter().for_each(|(cond, value)| { - chileren.push(cond.clone()); - chileren.push(value.clone()); + children.push(cond.clone()); + children.push(value.clone()); }); match &self.else_expr { - Some(expr) => chileren.push(expr.clone()), - None => chileren.push(Arc::new(NoOp::new())), + Some(expr) => children.push(expr.clone()), + None => children.push(Arc::new(NoOp::new())), } - chileren + children } // For physical CaseExpr, we do not allow modifying children size @@ -390,7 +390,7 @@ mod tests { use crate::expressions::col; use crate::expressions::lit; use crate::expressions::{binary, cast}; - use crate::physical_expr::TreeNodeRewritable; + use crate::rewrite::TreeNodeRewritable; use arrow::array::StringArray; use arrow::buffer::Buffer; use arrow::datatypes::DataType::Float64; diff --git a/datafusion/physical-expr/src/expressions/datetime.rs b/datafusion/physical-expr/src/expressions/datetime.rs index 4b4b43fe9991e..608368dda1cac 100644 --- a/datafusion/physical-expr/src/expressions/datetime.rs +++ b/datafusion/physical-expr/src/expressions/datetime.rs @@ -44,6 +44,8 @@ pub struct DateTimeIntervalExpr { lhs: Arc, op: Operator, rhs: Arc, + // TODO: move type checking to the planning phase and not in the physical expr + // so we can remove this input_schema: Schema, } diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index a43b6042b2485..4ec3394ab8884 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -34,7 +34,8 @@ use arrow::{ record_batch::RecordBatch, }; -use crate::physical_expr::{down_cast_any_ref, expr_list_eq_any_order}; +use crate::physical_expr::down_cast_any_ref; +use crate::utils::expr_list_eq_any_order; use crate::PhysicalExpr; use arrow::array::*; use arrow::datatypes::TimeUnit; diff --git a/datafusion/physical-expr/src/expressions/no_op.rs b/datafusion/physical-expr/src/expressions/no_op.rs index b81bcfc30b905..7c7d8cc8977dc 100644 --- a/datafusion/physical-expr/src/expressions/no_op.rs +++ b/datafusion/physical-expr/src/expressions/no_op.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Literal expressions for physical operations +//! NoOp placeholder for physical operations use std::any::Any; use std::sync::Arc; @@ -27,10 +27,12 @@ use arrow::{ use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; -use datafusion_common::Result; +use datafusion_common::{DataFusionError, Result}; use datafusion_expr::ColumnarValue; -/// A place holder expressions, can not be evaluated +/// A place holder expression, can not be evaluated. +/// +/// Used in some cases where an `Arc` is needed, such as `children()` #[derive(Debug, PartialEq, Eq, Default)] pub struct NoOp {} @@ -62,7 +64,9 @@ impl PhysicalExpr for NoOp { } fn evaluate(&self, _batch: &RecordBatch) -> Result { - unimplemented!("NoOp::evaluate"); + Err(DataFusionError::Plan( + "NoOp::evaluate() should not be called".to_owned(), + )) } fn children(&self) -> Vec> { diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rs index 965ca330056df..9087c2c2d9732 100644 --- a/datafusion/physical-expr/src/lib.rs +++ b/datafusion/physical-expr/src/lib.rs @@ -29,6 +29,7 @@ mod physical_expr; pub mod planner; #[cfg(feature = "regex_expressions")] pub mod regex_expressions; +pub mod rewrite; mod scalar_function; mod sort_expr; pub mod string_expressions; @@ -37,6 +38,7 @@ pub mod type_coercion; pub mod udf; #[cfg(feature = "unicode_expressions")] pub mod unicode_expressions; +pub mod utils; pub mod var_provider; pub mod window; diff --git a/datafusion/physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rs index 21a27700f8981..772a02097bfd5 100644 --- a/datafusion/physical-expr/src/physical_expr.rs +++ b/datafusion/physical-expr/src/physical_expr.rs @@ -19,7 +19,6 @@ use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; -use crate::PhysicalSortExpr; use datafusion_common::{ColumnStatistics, DataFusionError, Result, ScalarValue}; use datafusion_expr::ColumnarValue; @@ -137,147 +136,6 @@ impl PhysicalExprStats for BasicExpressionStats { } } -/// a Trait for marking tree node types that are rewritable -pub trait TreeNodeRewritable: Clone { - /// Transform the tree node using the given [TreeNodeRewriter] - /// It performs a depth first walk of an node and its children. - /// - /// For an node tree such as - /// ```text - /// ParentNode - /// left: ChildNode1 - /// right: ChildNode2 - /// ``` - /// - /// The nodes are visited using the following order - /// ```text - /// pre_visit(ParentNode) - /// pre_visit(ChildNode1) - /// mutatate(ChildNode1) - /// pre_visit(ChildNode2) - /// mutate(ChildNode2) - /// mutate(ParentNode) - /// ``` - /// - /// If an Err result is returned, recursion is stopped immediately - /// - /// If [`false`] is returned on a call to pre_visit, no - /// children of that node are visited, nor is mutate - /// called on that node - /// - fn transform_using>( - self, - rewriter: &mut R, - ) -> Result { - let need_mutate = match rewriter.pre_visit(&self)? { - RewriteRecursion::Mutate => return rewriter.mutate(self), - RewriteRecursion::Stop => return Ok(self), - RewriteRecursion::Continue => true, - RewriteRecursion::Skip => false, - }; - - let after_op_children = - self.map_children(|node| node.transform_using(rewriter))?; - - // now rewrite this node itself - if need_mutate { - rewriter.mutate(after_op_children) - } else { - Ok(after_op_children) - } - } - - /// Convenience utils for writing optimizers rule: recursively apply the given `op` to the node tree. - /// When `op` does not apply to a given node, it is left uncshanged. - /// The default tree traversal direction is transform_up(Postorder Traversal). - fn transform(self, op: &F) -> Result - where - F: Fn(Self) -> Option, - { - self.transform_up(op) - } - - /// Convenience utils for writing optimizers rule: recursively apply the given 'op' to the node and all of its - /// children(Preorder Traversal). - /// When the `op` does not apply to a given node, it is left unchanged. - fn transform_down(self, op: &F) -> Result - where - F: Fn(Self) -> Option, - { - let node_cloned = self.clone(); - let after_op = match op(node_cloned) { - Some(value) => value, - None => self, - }; - after_op.map_children(|node| node.transform_down(op)) - } - - /// Convenience utils for writing optimizers rule: recursively apply the given 'op' first to all of its - /// children and then itself(Postorder Traversal). - /// When the `op` does not apply to a given node, it is left unchanged. - fn transform_up(self, op: &F) -> Result - where - F: Fn(Self) -> Option, - { - let after_op_children = self.map_children(|node| node.transform_up(op))?; - - let after_op_children_clone = after_op_children.clone(); - let new_node = match op(after_op_children) { - Some(value) => value, - None => after_op_children_clone, - }; - Ok(new_node) - } - - /// Apply transform `F` to the node's children, the transform `F` might have a direction(Preorder or Postorder) - fn map_children(self, transform: F) -> Result - where - F: FnMut(Self) -> Result; -} - -/// Trait for potentially recursively transform an [`TreeNodeRewritable`] node -/// tree. When passed to `TreeNodeRewritable::transform_using`, `TreeNodeRewriter::mutate` is -/// invoked recursively on all nodes of a tree. -pub trait TreeNodeRewriter: Sized { - /// Invoked before (Preorder) any children of `node` are rewritten / - /// visited. Default implementation returns `Ok(RewriteRecursion::Continue)` - fn pre_visit(&mut self, _node: &N) -> Result { - Ok(RewriteRecursion::Continue) - } - - /// Invoked after (Postorder) all children of `node` have been mutated and - /// returns a potentially modified node. - fn mutate(&mut self, node: N) -> Result; -} - -/// Controls how the [TreeNodeRewriter] recursion should proceed. -#[allow(dead_code)] -pub enum RewriteRecursion { - /// Continue rewrite / visit this node tree. - Continue, - /// Call 'op' immediately and return. - Mutate, - /// Do not rewrite / visit the children of this node. - Stop, - /// Keep recursive but skip apply op on this node - Skip, -} - -impl TreeNodeRewritable for Arc { - fn map_children(self, transform: F) -> Result - where - F: FnMut(Self) -> Result, - { - if !self.children().is_empty() { - let new_children: Result> = - self.children().into_iter().map(transform).collect(); - with_new_children_if_necessary(self, new_children?) - } else { - Ok(self) - } - } -} - /// Returns a copy of this expr if we change any child according to the pointer comparison. /// The size of `children` must be equal to the size of `PhysicalExpr::children()`. /// Allow the vtable address comparisons for PhysicalExpr Trait Objects,it is harmless even @@ -303,52 +161,6 @@ pub fn with_new_children_if_necessary( } } -/// Compare the two expr lists are equal no matter the order. -/// For example two InListExpr can be considered to be equals no matter the order: -/// -/// In('a','b','c') == In('c','b','a') -pub fn expr_list_eq_any_order( - list1: &[Arc], - list2: &[Arc], -) -> bool { - if list1.len() == list2.len() { - let mut expr_vec1 = list1.to_vec(); - let mut expr_vec2 = list2.to_vec(); - while let Some(expr1) = expr_vec1.pop() { - if let Some(idx) = expr_vec2.iter().position(|expr2| expr1.eq(expr2)) { - expr_vec2.swap_remove(idx); - } else { - break; - } - } - expr_vec1.is_empty() && expr_vec2.is_empty() - } else { - false - } -} - -/// Strictly compare the two expr lists are equal in the given order. -#[allow(dead_code)] -pub fn expr_list_eq_strict_order( - list1: &[Arc], - list2: &[Arc], -) -> bool { - list1.len() == list2.len() && list1.iter().zip(list2.iter()).all(|(e1, e2)| e1.eq(e2)) -} - -/// Strictly compare the two sort expr lists in the given order. -/// -/// For Physical Sort Exprs, the order matters: -/// -/// SortExpr('a','b','c') != SortExpr('c','b','a') -#[allow(dead_code)] -pub fn sort_expr_list_eq_strict_order( - list1: &[PhysicalSortExpr], - list2: &[PhysicalSortExpr], -) -> bool { - list1.len() == list2.len() && list1.iter().zip(list2.iter()).all(|(e1, e2)| e1.eq(e2)) -} - pub fn down_cast_any_ref(any: &dyn Any) -> &dyn Any { if any.is::>() { any.downcast_ref::>() @@ -411,10 +223,7 @@ mod tests { use std::sync::Arc; use super::*; - use crate::expressions::Column; - use crate::PhysicalSortExpr; use arrow::array::Int32Array; - use arrow::compute::SortOptions; use datafusion_common::Result; #[test] @@ -509,131 +318,4 @@ mod tests { Ok(()) } - - #[test] - fn expr_list_eq_any_order_test() -> Result<()> { - let list1: Vec> = vec![ - Arc::new(Column::new("a", 0)), - Arc::new(Column::new("a", 0)), - Arc::new(Column::new("b", 1)), - ]; - let list2: Vec> = vec![ - Arc::new(Column::new("b", 1)), - Arc::new(Column::new("b", 1)), - Arc::new(Column::new("a", 0)), - ]; - assert!(!expr_list_eq_any_order(list1.as_slice(), list2.as_slice())); - assert!(!expr_list_eq_any_order(list2.as_slice(), list1.as_slice())); - - let list3: Vec> = vec![ - Arc::new(Column::new("a", 0)), - Arc::new(Column::new("b", 1)), - Arc::new(Column::new("c", 2)), - Arc::new(Column::new("a", 0)), - Arc::new(Column::new("b", 1)), - ]; - let list4: Vec> = vec![ - Arc::new(Column::new("b", 1)), - Arc::new(Column::new("b", 1)), - Arc::new(Column::new("a", 0)), - Arc::new(Column::new("c", 2)), - Arc::new(Column::new("a", 0)), - ]; - assert!(expr_list_eq_any_order(list3.as_slice(), list4.as_slice())); - assert!(expr_list_eq_any_order(list4.as_slice(), list3.as_slice())); - assert!(expr_list_eq_any_order(list3.as_slice(), list3.as_slice())); - assert!(expr_list_eq_any_order(list4.as_slice(), list4.as_slice())); - - Ok(()) - } - - #[test] - fn sort_expr_list_eq_strict_order_test() -> Result<()> { - let list1: Vec = vec![ - PhysicalSortExpr { - expr: Arc::new(Column::new("a", 0)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("a", 0)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("b", 1)), - options: SortOptions::default(), - }, - ]; - - let list2: Vec = vec![ - PhysicalSortExpr { - expr: Arc::new(Column::new("b", 1)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("a", 0)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("a", 0)), - options: SortOptions::default(), - }, - ]; - - assert!(!sort_expr_list_eq_strict_order( - list1.as_slice(), - list2.as_slice() - )); - assert!(!sort_expr_list_eq_strict_order( - list2.as_slice(), - list1.as_slice() - )); - - let list3: Vec = vec![ - PhysicalSortExpr { - expr: Arc::new(Column::new("a", 0)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("b", 1)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("c", 2)), - options: SortOptions::default(), - }, - ]; - let list4: Vec = vec![ - PhysicalSortExpr { - expr: Arc::new(Column::new("a", 0)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("b", 1)), - options: SortOptions::default(), - }, - PhysicalSortExpr { - expr: Arc::new(Column::new("c", 2)), - options: SortOptions::default(), - }, - ]; - - assert!(sort_expr_list_eq_strict_order( - list3.as_slice(), - list4.as_slice() - )); - assert!(sort_expr_list_eq_strict_order( - list4.as_slice(), - list3.as_slice() - )); - assert!(sort_expr_list_eq_strict_order( - list3.as_slice(), - list3.as_slice() - )); - assert!(sort_expr_list_eq_strict_order( - list4.as_slice(), - list4.as_slice() - )); - - Ok(()) - } } diff --git a/datafusion/physical-expr/src/rewrite.rs b/datafusion/physical-expr/src/rewrite.rs new file mode 100644 index 0000000000000..ff91571c906f0 --- /dev/null +++ b/datafusion/physical-expr/src/rewrite.rs @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::physical_expr::with_new_children_if_necessary; +use crate::PhysicalExpr; +use datafusion_common::Result; + +use std::sync::Arc; + +/// a Trait for marking tree node types that are rewritable +pub trait TreeNodeRewritable: Clone { + /// Transform the tree node using the given [TreeNodeRewriter] + /// It performs a depth first walk of an node and its children. + /// + /// For an node tree such as + /// ```text + /// ParentNode + /// left: ChildNode1 + /// right: ChildNode2 + /// ``` + /// + /// The nodes are visited using the following order + /// ```text + /// pre_visit(ParentNode) + /// pre_visit(ChildNode1) + /// mutatate(ChildNode1) + /// pre_visit(ChildNode2) + /// mutate(ChildNode2) + /// mutate(ParentNode) + /// ``` + /// + /// If an Err result is returned, recursion is stopped immediately + /// + /// If [`false`] is returned on a call to pre_visit, no + /// children of that node are visited, nor is mutate + /// called on that node + /// + fn transform_using>( + self, + rewriter: &mut R, + ) -> Result { + let need_mutate = match rewriter.pre_visit(&self)? { + RewriteRecursion::Mutate => return rewriter.mutate(self), + RewriteRecursion::Stop => return Ok(self), + RewriteRecursion::Continue => true, + RewriteRecursion::Skip => false, + }; + + let after_op_children = + self.map_children(|node| node.transform_using(rewriter))?; + + // now rewrite this node itself + if need_mutate { + rewriter.mutate(after_op_children) + } else { + Ok(after_op_children) + } + } + + /// Convenience utils for writing optimizers rule: recursively apply the given `op` to the node tree. + /// When `op` does not apply to a given node, it is left unchanged. + /// The default tree traversal direction is transform_up(Postorder Traversal). + fn transform(self, op: &F) -> Result + where + F: Fn(Self) -> Option, + { + self.transform_up(op) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given 'op' to the node and all of its + /// children(Preorder Traversal). + /// When the `op` does not apply to a given node, it is left unchanged. + fn transform_down(self, op: &F) -> Result + where + F: Fn(Self) -> Option, + { + let node_cloned = self.clone(); + let after_op = match op(node_cloned) { + Some(value) => value, + None => self, + }; + after_op.map_children(|node| node.transform_down(op)) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given 'op' first to all of its + /// children and then itself(Postorder Traversal). + /// When the `op` does not apply to a given node, it is left unchanged. + fn transform_up(self, op: &F) -> Result + where + F: Fn(Self) -> Option, + { + let after_op_children = self.map_children(|node| node.transform_up(op))?; + + let after_op_children_clone = after_op_children.clone(); + let new_node = match op(after_op_children) { + Some(value) => value, + None => after_op_children_clone, + }; + Ok(new_node) + } + + /// Apply transform `F` to the node's children, the transform `F` might have a direction(Preorder or Postorder) + fn map_children(self, transform: F) -> Result + where + F: FnMut(Self) -> Result; +} + +/// Trait for potentially recursively transform an [`TreeNodeRewritable`] node +/// tree. When passed to `TreeNodeRewritable::transform_using`, `TreeNodeRewriter::mutate` is +/// invoked recursively on all nodes of a tree. +pub trait TreeNodeRewriter: Sized { + /// Invoked before (Preorder) any children of `node` are rewritten / + /// visited. Default implementation returns `Ok(RewriteRecursion::Continue)` + fn pre_visit(&mut self, _node: &N) -> Result { + Ok(RewriteRecursion::Continue) + } + + /// Invoked after (Postorder) all children of `node` have been mutated and + /// returns a potentially modified node. + fn mutate(&mut self, node: N) -> Result; +} + +/// Controls how the [TreeNodeRewriter] recursion should proceed. +#[allow(dead_code)] +pub enum RewriteRecursion { + /// Continue rewrite / visit this node tree. + Continue, + /// Call 'op' immediately and return. + Mutate, + /// Do not rewrite / visit the children of this node. + Stop, + /// Keep recursive but skip apply op on this node + Skip, +} + +impl TreeNodeRewritable for Arc { + fn map_children(self, transform: F) -> Result + where + F: FnMut(Self) -> Result, + { + if !self.children().is_empty() { + let new_children: Result> = + self.children().into_iter().map(transform).collect(); + with_new_children_if_necessary(self, new_children?) + } else { + Ok(self) + } + } +} diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index 93968b60f8b2d..fe674497a1972 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -29,6 +29,8 @@ //! This module also has a set of coercion rules to improve user experience: if an argument i32 is passed //! to a function that supports f64, it is coerced to f64. +use crate::physical_expr::down_cast_any_ref; +use crate::utils::expr_list_eq_strict_order; use crate::PhysicalExpr; use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; @@ -164,7 +166,15 @@ impl PhysicalExpr for ScalarFunctionExpr { } impl PartialEq for ScalarFunctionExpr { - fn eq(&self, _other: &dyn Any) -> bool { - false + /// Comparing name, args and return_type + fn eq(&self, other: &dyn Any) -> bool { + down_cast_any_ref(other) + .downcast_ref::() + .map(|x| { + self.name == x.name + && expr_list_eq_strict_order(&self.args, &x.args) + && self.return_type == x.return_type + }) + .unwrap_or(false) } } diff --git a/datafusion/physical-expr/src/utils.rs b/datafusion/physical-expr/src/utils.rs new file mode 100644 index 0000000000000..7ee947c0a7f99 --- /dev/null +++ b/datafusion/physical-expr/src/utils.rs @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::PhysicalExpr; +use crate::PhysicalSortExpr; + +use std::sync::Arc; + +/// Compare the two expr lists are equal no matter the order. +/// For example two InListExpr can be considered to be equals no matter the order: +/// +/// In('a','b','c') == In('c','b','a') +pub fn expr_list_eq_any_order( + list1: &[Arc], + list2: &[Arc], +) -> bool { + if list1.len() == list2.len() { + let mut expr_vec1 = list1.to_vec(); + let mut expr_vec2 = list2.to_vec(); + while let Some(expr1) = expr_vec1.pop() { + if let Some(idx) = expr_vec2.iter().position(|expr2| expr1.eq(expr2)) { + expr_vec2.swap_remove(idx); + } else { + break; + } + } + expr_vec1.is_empty() && expr_vec2.is_empty() + } else { + false + } +} + +/// Strictly compare the two expr lists are equal in the given order. +pub fn expr_list_eq_strict_order( + list1: &[Arc], + list2: &[Arc], +) -> bool { + list1.len() == list2.len() && list1.iter().zip(list2.iter()).all(|(e1, e2)| e1.eq(e2)) +} + +/// Strictly compare the two sort expr lists in the given order. +/// +/// For Physical Sort Exprs, the order matters: +/// +/// SortExpr('a','b','c') != SortExpr('c','b','a') +#[allow(dead_code)] +pub fn sort_expr_list_eq_strict_order( + list1: &[PhysicalSortExpr], + list2: &[PhysicalSortExpr], +) -> bool { + list1.len() == list2.len() && list1.iter().zip(list2.iter()).all(|(e1, e2)| e1.eq(e2)) +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::expressions::Column; + use crate::PhysicalSortExpr; + use arrow::compute::SortOptions; + use datafusion_common::Result; + + use std::sync::Arc; + + #[test] + fn expr_list_eq_any_order_test() -> Result<()> { + let list1: Vec> = vec![ + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("b", 1)), + ]; + let list2: Vec> = vec![ + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("a", 0)), + ]; + assert!(!expr_list_eq_any_order(list1.as_slice(), list2.as_slice())); + assert!(!expr_list_eq_any_order(list2.as_slice(), list1.as_slice())); + + let list3: Vec> = vec![ + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("c", 2)), + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("b", 1)), + ]; + let list4: Vec> = vec![ + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("b", 1)), + Arc::new(Column::new("a", 0)), + Arc::new(Column::new("c", 2)), + Arc::new(Column::new("a", 0)), + ]; + assert!(expr_list_eq_any_order(list3.as_slice(), list4.as_slice())); + assert!(expr_list_eq_any_order(list4.as_slice(), list3.as_slice())); + assert!(expr_list_eq_any_order(list3.as_slice(), list3.as_slice())); + assert!(expr_list_eq_any_order(list4.as_slice(), list4.as_slice())); + + Ok(()) + } + + #[test] + fn sort_expr_list_eq_strict_order_test() -> Result<()> { + let list1: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + ]; + + let list2: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + ]; + + assert!(!sort_expr_list_eq_strict_order( + list1.as_slice(), + list2.as_slice() + )); + assert!(!sort_expr_list_eq_strict_order( + list2.as_slice(), + list1.as_slice() + )); + + let list3: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("c", 2)), + options: SortOptions::default(), + }, + ]; + let list4: Vec = vec![ + PhysicalSortExpr { + expr: Arc::new(Column::new("a", 0)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("b", 1)), + options: SortOptions::default(), + }, + PhysicalSortExpr { + expr: Arc::new(Column::new("c", 2)), + options: SortOptions::default(), + }, + ]; + + assert!(sort_expr_list_eq_strict_order( + list3.as_slice(), + list4.as_slice() + )); + assert!(sort_expr_list_eq_strict_order( + list4.as_slice(), + list3.as_slice() + )); + assert!(sort_expr_list_eq_strict_order( + list3.as_slice(), + list3.as_slice() + )); + assert!(sort_expr_list_eq_strict_order( + list4.as_slice(), + list4.as_slice() + )); + + Ok(()) + } +} From cbf3158a41d47889cef2c1fdda79a855948d5840 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 27 Oct 2022 23:54:31 +0800 Subject: [PATCH 3/3] tiny fix to doc --- datafusion/physical-expr/src/rewrite.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/rewrite.rs b/datafusion/physical-expr/src/rewrite.rs index ff91571c906f0..ccd9210dfc7b4 100644 --- a/datafusion/physical-expr/src/rewrite.rs +++ b/datafusion/physical-expr/src/rewrite.rs @@ -37,7 +37,7 @@ pub trait TreeNodeRewritable: Clone { /// ```text /// pre_visit(ParentNode) /// pre_visit(ChildNode1) - /// mutatate(ChildNode1) + /// mutate(ChildNode1) /// pre_visit(ChildNode2) /// mutate(ChildNode2) /// mutate(ParentNode)