Repository navigation
[Part1] Partition and Sort Enforcement, PhysicalExpr enhancement #3969
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
0443adc
b705574
1c05ac8
cb941b2
cbf3158
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,6 +18,8 @@ | |
| use std::{any::Any, sync::Arc}; | ||
|
|
||
| use crate::expressions::try_cast; | ||
| use crate::expressions::NoOp; | ||
| use crate::physical_expr::down_cast_any_ref; | ||
| use crate::PhysicalExpr; | ||
| use arrow::array::*; | ||
| use arrow::compute::kernels::zip::zip; | ||
|
|
@@ -27,6 +29,8 @@ use arrow::record_batch::RecordBatch; | |
| use datafusion_common::{DataFusionError, Result}; | ||
| use datafusion_expr::ColumnarValue; | ||
|
|
||
| use itertools::Itertools; | ||
|
|
||
| type WhenThen = (Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>); | ||
|
|
||
| /// The CASE expression is similar to a series of nested if/else and there are two forms that | ||
|
|
@@ -286,6 +290,89 @@ impl PhysicalExpr for CaseExpr { | |
| self.case_when_no_expr(batch) | ||
| } | ||
| } | ||
|
|
||
| fn children(&self) -> Vec<Arc<dyn PhysicalExpr>> { | ||
| let mut children = vec![]; | ||
| match &self.expr { | ||
| Some(expr) => children.push(expr.clone()), | ||
| None => children.push(Arc::new(NoOp::new())), | ||
| } | ||
| self.when_then_expr.iter().for_each(|(cond, value)| { | ||
| children.push(cond.clone()); | ||
| children.push(value.clone()); | ||
| }); | ||
|
|
||
| match &self.else_expr { | ||
| Some(expr) => children.push(expr.clone()), | ||
| None => children.push(Arc::new(NoOp::new())), | ||
| } | ||
| children | ||
| } | ||
|
|
||
| // For physical CaseExpr, we do not allow modifying children size | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 👍 |
||
| fn with_new_children( | ||
| self: Arc<Self>, | ||
| children: Vec<Arc<dyn PhysicalExpr>>, | ||
| ) -> Result<Arc<dyn PhysicalExpr>> { | ||
| if children.len() != self.children().len() { | ||
| Err(DataFusionError::Internal( | ||
| "CaseExpr: Wrong number of children".to_string(), | ||
| )) | ||
| } else { | ||
| assert_eq!(children.len() % 2, 0); | ||
| let expr = match children[0].clone().as_any().downcast_ref::<NoOp>() { | ||
| Some(_) => None, | ||
| _ => Some(children[0].clone()), | ||
| }; | ||
| let else_expr = match children[children.len() - 1] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i would find it easier to read if the |
||
| .clone() | ||
| .as_any() | ||
| .downcast_ref::<NoOp>() | ||
| { | ||
| Some(_) => None, | ||
| _ => Some(children[children.len() - 1].clone()), | ||
| }; | ||
|
|
||
| let branches = children[1..children.len() - 1].to_vec(); | ||
| let mut when_then_expr: Vec<WhenThen> = vec![]; | ||
| for (prev, next) in branches.into_iter().tuples() { | ||
| when_then_expr.push((prev, next)); | ||
| } | ||
| Ok(Arc::new(CaseExpr::try_new( | ||
| expr, | ||
| when_then_expr, | ||
| else_expr, | ||
| )?)) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl PartialEq<dyn Any> for CaseExpr { | ||
| fn eq(&self, other: &dyn Any) -> bool { | ||
| down_cast_any_ref(other) | ||
| .downcast_ref::<Self>() | ||
| .map(|x| { | ||
| let expr_eq = match (&self.expr, &x.expr) { | ||
| (Some(expr1), Some(expr2)) => expr1.eq(expr2), | ||
| (None, None) => true, | ||
| _ => false, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You could also do
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would prefer the current implementation than returning from the Closure. Returning from the Closure might |
||
| }; | ||
| let else_expr_eq = match (&self.else_expr, &x.else_expr) { | ||
| (Some(expr1), Some(expr2)) => expr1.eq(expr2), | ||
| (None, None) => true, | ||
| _ => false, | ||
| }; | ||
| expr_eq | ||
| && else_expr_eq | ||
| && self.when_then_expr.len() == x.when_then_expr.len() | ||
| && self.when_then_expr.iter().zip(x.when_then_expr.iter()).all( | ||
| |((when1, then1), (when2, then2))| { | ||
| when1.eq(when2) && then1.eq(then2) | ||
| }, | ||
| ) | ||
| }) | ||
| .unwrap_or(false) | ||
| } | ||
| } | ||
|
|
||
| /// Create a CASE expression | ||
|
|
@@ -303,6 +390,7 @@ mod tests { | |
| use crate::expressions::col; | ||
| use crate::expressions::lit; | ||
| use crate::expressions::{binary, cast}; | ||
| use crate::rewrite::TreeNodeRewritable; | ||
| use arrow::array::StringArray; | ||
| use arrow::buffer::Buffer; | ||
| use arrow::datatypes::DataType::Float64; | ||
|
|
@@ -728,6 +816,119 @@ mod tests { | |
| Ok(()) | ||
| } | ||
|
|
||
| #[test] | ||
| fn case_eq() -> Result<()> { | ||
| let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); | ||
|
|
||
| let when1 = lit("foo"); | ||
| let then1 = lit(123i32); | ||
| let when2 = lit("bar"); | ||
| let then2 = lit(456i32); | ||
| let else_value = lit(999i32); | ||
|
|
||
| let expr1 = generate_case_when_with_type_coercion( | ||
| Some(col("a", &schema)?), | ||
| vec![ | ||
| (when1.clone(), then1.clone()), | ||
| (when2.clone(), then2.clone()), | ||
| ], | ||
| Some(else_value.clone()), | ||
| &schema, | ||
| )?; | ||
|
|
||
| let expr2 = generate_case_when_with_type_coercion( | ||
| Some(col("a", &schema)?), | ||
| vec![ | ||
| (when1.clone(), then1.clone()), | ||
| (when2.clone(), then2.clone()), | ||
| ], | ||
| Some(else_value.clone()), | ||
| &schema, | ||
| )?; | ||
|
|
||
| let expr3 = generate_case_when_with_type_coercion( | ||
| Some(col("a", &schema)?), | ||
| vec![(when1.clone(), then1.clone()), (when2, then2)], | ||
| None, | ||
| &schema, | ||
| )?; | ||
|
|
||
| let expr4 = generate_case_when_with_type_coercion( | ||
| Some(col("a", &schema)?), | ||
| vec![(when1, then1)], | ||
| Some(else_value), | ||
| &schema, | ||
| )?; | ||
|
|
||
| assert!(expr1.eq(&expr2)); | ||
| assert!(expr2.eq(&expr1)); | ||
|
|
||
| assert!(expr2.ne(&expr3)); | ||
| assert!(expr3.ne(&expr2)); | ||
|
|
||
| assert!(expr1.ne(&expr4)); | ||
| assert!(expr4.ne(&expr1)); | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| #[test] | ||
| fn case_tranform() -> Result<()> { | ||
| let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); | ||
|
|
||
| let when1 = lit("foo"); | ||
| let then1 = lit(123i32); | ||
| let when2 = lit("bar"); | ||
| let then2 = lit(456i32); | ||
| let else_value = lit(999i32); | ||
|
|
||
| let expr = generate_case_when_with_type_coercion( | ||
| Some(col("a", &schema)?), | ||
| vec![ | ||
| (when1.clone(), then1.clone()), | ||
| (when2.clone(), then2.clone()), | ||
| ], | ||
| Some(else_value.clone()), | ||
| &schema, | ||
| )?; | ||
|
|
||
| let expr2 = expr | ||
| .clone() | ||
| .transform( | ||
| &|e| match e.as_any().downcast_ref::<crate::expressions::Literal>() { | ||
| Some(lit_value) => match lit_value.value() { | ||
| ScalarValue::Utf8(Some(str_value)) => { | ||
| Some(lit(str_value.to_uppercase())) | ||
| } | ||
| _ => None, | ||
| }, | ||
| _ => None, | ||
| }, | ||
| ) | ||
| .unwrap(); | ||
|
|
||
| let expr3 = expr | ||
| .clone() | ||
| .transform_down(&|e| match e | ||
| .as_any() | ||
| .downcast_ref::<crate::expressions::Literal>() | ||
| { | ||
| Some(lit_value) => match lit_value.value() { | ||
| ScalarValue::Utf8(Some(str_value)) => { | ||
| Some(lit(str_value.to_uppercase())) | ||
| } | ||
| _ => None, | ||
| }, | ||
| _ => None, | ||
| }) | ||
| .unwrap(); | ||
|
|
||
| assert!(expr.ne(&expr2)); | ||
| assert!(expr2.eq(&expr3)); | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| fn generate_case_when_with_type_coercion( | ||
| expr: Option<Arc<dyn PhysicalExpr>>, | ||
| when_thens: Vec<WhenThen>, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ use std::any::Any; | |
| use std::fmt; | ||
| use std::sync::Arc; | ||
|
|
||
| use crate::physical_expr::down_cast_any_ref; | ||
| use crate::PhysicalExpr; | ||
| use arrow::compute; | ||
| use arrow::compute::kernels; | ||
|
|
@@ -93,6 +94,36 @@ impl PhysicalExpr for CastExpr { | |
| let value = self.expr.evaluate(batch)?; | ||
| cast_column(&value, &self.cast_type, &self.cast_options) | ||
| } | ||
|
|
||
| fn children(&self) -> Vec<Arc<dyn PhysicalExpr>> { | ||
| vec![self.expr.clone()] | ||
| } | ||
|
|
||
| fn with_new_children( | ||
| self: Arc<Self>, | ||
| children: Vec<Arc<dyn PhysicalExpr>>, | ||
| ) -> Result<Arc<dyn PhysicalExpr>> { | ||
| Ok(Arc::new(CastExpr::new( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. recommend assert / error check for children.len() == 1 |
||
| children[0].clone(), | ||
| self.cast_type.clone(), | ||
| CastOptions { | ||
| safe: self.cast_options.safe, | ||
| }, | ||
| ))) | ||
| } | ||
| } | ||
|
|
||
| impl PartialEq<dyn Any> for CastExpr { | ||
| fn eq(&self, other: &dyn Any) -> bool { | ||
| down_cast_any_ref(other) | ||
| .downcast_ref::<Self>() | ||
| .map(|x| { | ||
| self.expr.eq(&x.expr) | ||
| && self.cast_type == x.cast_type | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added a todo note in #4012 |
||
| && self.cast_options.safe == x.cast_options.safe | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This seems to be a bit error-prone, considering that
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nevermind, it seems like we can't change it here. Created a ticket on arrow-rs. |
||
| }) | ||
| .unwrap_or(false) | ||
| } | ||
| } | ||
|
|
||
| /// Internal cast function for casting ColumnarValue -> ColumnarValue for cast_type | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I suggest we should also
assert_eq!(children.len(), 2)to catch bugs earlierUh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I already add the length check in the method
with_new_children_if_necessary(), and expr.with_new_children() should not be called directly. This is also aligned with theExecutionPlan.