Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions datafusion-cli/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion datafusion-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ fn create_runtime_env() -> Result<RuntimeEnv> {
ObjectStoreRegistry::new_with_provider(Some(Arc::new(object_store_provider)));
let rn_config =
RuntimeConfig::new().with_object_store_registry(Arc::new(object_store_registry));
return RuntimeEnv::new(rn_config);
RuntimeEnv::new(rn_config)
}

fn is_valid_file(dir: &str) -> std::result::Result<(), String> {
Expand Down
12 changes: 7 additions & 5 deletions datafusion-cli/src/object_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ fn build_gcs_object_store(url: &Url) -> Result<Arc<dyn object_store::ObjectStore
let host = get_host_name(url)?;
let mut builder = GoogleCloudStorageBuilder::new().with_bucket_name(host);

if let Some(path) = env::var("GCP_SERVICE_ACCOUNT_PATH").ok() {
if let Ok(path) = env::var("GCP_SERVICE_ACCOUNT_PATH") {
builder = builder.with_service_account_path(path);
}
match builder.build() {
Expand All @@ -78,10 +78,12 @@ fn build_gcs_object_store(url: &Url) -> Result<Arc<dyn object_store::ObjectStore
}

fn get_host_name(url: &Url) -> Result<&str> {
url.host_str().ok_or(DataFusionError::Execution(format!(
"Not able to parse hostname from url, {}",
url.as_str()
)))
url.host_str().ok_or_else(|| {
DataFusionError::Execution(format!(
"Not able to parse hostname from url, {}",
url.as_str()
))
})
}

#[cfg(test)]
Expand Down
1 change: 1 addition & 0 deletions datafusion/physical-expr/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ datafusion-common = { path = "../common", version = "13.0.0" }
datafusion-expr = { path = "../expr", version = "13.0.0" }
datafusion-row = { path = "../row", version = "13.0.0" }
hashbrown = { version = "0.12", features = ["raw"] }
itertools = { version = "0.10", features = ["use_std"] }
lazy_static = { version = "^1.4.0" }
md-5 = { version = "^0.10.0", optional = true }
ordered-float = "3.0"
Expand Down
25 changes: 25 additions & 0 deletions datafusion/physical-expr/src/expressions/binary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ use kernels_arrow::{
use arrow::datatypes::{DataType, Schema, TimeUnit};
use arrow::record_batch::RecordBatch;

use crate::physical_expr::down_cast_any_ref;
use crate::{ExprBoundaries, PhysicalExpr, PhysicalExprStats};
use datafusion_common::{ColumnStatistics, ScalarValue};
use datafusion_common::{DataFusionError, Result};
Expand Down Expand Up @@ -649,6 +650,30 @@ impl PhysicalExpr for BinaryExpr {
right: Arc::clone(self.right()),
})
}

fn children(&self) -> Vec<Arc<dyn PhysicalExpr>> {
vec![self.left.clone(), self.right.clone()]
}

fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn PhysicalExpr>>,
) -> Result<Arc<dyn PhysicalExpr>> {
Ok(Arc::new(BinaryExpr::new(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest we should also assert_eq!(children.len(), 2) to catch bugs earlier

@mingmwang mingmwang Oct 27, 2022 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I already add the length check in the method with_new_children_if_necessary(), and expr.with_new_children() should not be called directly. This is also aligned with the ExecutionPlan.

pub fn with_new_children_if_necessary(
    expr: Arc<dyn PhysicalExpr>,
    children: Vec<Arc<dyn PhysicalExpr>>,
) -> Result<Arc<dyn PhysicalExpr>> {
    if children.len() != expr.children().len() {
        Err(DataFusionError::Internal(
            "PhysicalExpr: Wrong number of children".to_string(),
        ))
    } else if children.is_empty()
        || children
            .iter()
            .zip(expr.children().iter())
            .any(|(c1, c2)| !Arc::ptr_eq(c1, c2))
    {
        expr.with_new_children(children)
    } else {
        Ok(expr)
    }
}

children[0].clone(),
self.op,
children[1].clone(),
)))
}
}

impl PartialEq<dyn Any> for BinaryExpr {
fn eq(&self, other: &dyn Any) -> bool {
down_cast_any_ref(other)
.downcast_ref::<Self>()
.map(|x| self.left.eq(&x.left) && self.op == x.op && self.right.eq(&x.right))
.unwrap_or(false)
}
}

struct BinaryExprStats {
Expand Down
201 changes: 201 additions & 0 deletions datafusion/physical-expr/src/expressions/case.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
use std::{any::Any, sync::Arc};

use crate::expressions::try_cast;
use crate::expressions::NoOp;
use crate::physical_expr::down_cast_any_ref;
use crate::PhysicalExpr;
use arrow::array::*;
use arrow::compute::kernels::zip::zip;
Expand All @@ -27,6 +29,8 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::{DataFusionError, Result};
use datafusion_expr::ColumnarValue;

use itertools::Itertools;

type WhenThen = (Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>);

/// The CASE expression is similar to a series of nested if/else and there are two forms that
Expand Down Expand Up @@ -286,6 +290,89 @@ impl PhysicalExpr for CaseExpr {
self.case_when_no_expr(batch)
}
}

fn children(&self) -> Vec<Arc<dyn PhysicalExpr>> {
let mut children = vec![];
match &self.expr {
Some(expr) => children.push(expr.clone()),
None => children.push(Arc::new(NoOp::new())),
}
self.when_then_expr.iter().for_each(|(cond, value)| {
children.push(cond.clone());
children.push(value.clone());
});

match &self.else_expr {
Some(expr) => children.push(expr.clone()),
None => children.push(Arc::new(NoOp::new())),
}
children
}

// For physical CaseExpr, we do not allow modifying children size

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn PhysicalExpr>>,
) -> Result<Arc<dyn PhysicalExpr>> {
if children.len() != self.children().len() {
Err(DataFusionError::Internal(
"CaseExpr: Wrong number of children".to_string(),
))
} else {
assert_eq!(children.len() % 2, 0);
let expr = match children[0].clone().as_any().downcast_ref::<NoOp>() {
Some(_) => None,
_ => Some(children[0].clone()),
};
let else_expr = match children[children.len() - 1]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i would find it easier to read if the else_expr handling was after the when_then_expr handling to mirror the children() but I think this is fine too

.clone()
.as_any()
.downcast_ref::<NoOp>()
{
Some(_) => None,
_ => Some(children[children.len() - 1].clone()),
};

let branches = children[1..children.len() - 1].to_vec();
let mut when_then_expr: Vec<WhenThen> = vec![];
for (prev, next) in branches.into_iter().tuples() {
when_then_expr.push((prev, next));
}
Ok(Arc::new(CaseExpr::try_new(
expr,
when_then_expr,
else_expr,
)?))
}
}
}

impl PartialEq<dyn Any> for CaseExpr {
fn eq(&self, other: &dyn Any) -> bool {
down_cast_any_ref(other)
.downcast_ref::<Self>()
.map(|x| {
let expr_eq = match (&self.expr, &x.expr) {
(Some(expr1), Some(expr2)) => expr1.eq(expr2),
(None, None) => true,
_ => false,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You could also do return false here rather than checking expr_eq again later

@mingmwang mingmwang Oct 27, 2022 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer the current implementation than returning from the Closure. Returning from the Closure might
confuse people, does it mean return from the Closure or return from the outer method? In Rust it is returned from the Closure but in Scala or Java(I remember) it is returned from the outer method. In my opinion, it is not clear and more lines of code.


fn eq(&self, other: &dyn Any) -> bool {
        down_cast_any_ref(other)
            .downcast_ref::<Self>()
            .map(|x| {
                match (&self.expr, &x.expr) {
                    (Some(expr1), Some(expr2)) =>{
                        if expr1.ne(expr2) {
                            return false;
                        }
                    },
                    (None, None) => {}
                    _ => return false,
                };
                match (&self.else_expr, &x.else_expr) {
                    (Some(expr1), Some(expr2)) => {
                        if expr1.ne(expr2) {
                            return false;
                        }
                    },
                    (None, None) => {}
                    _ => return false,
                };
                self.when_then_expr.len() == x.when_then_expr.len()
                    && self.when_then_expr.iter().zip(x.when_then_expr.iter()).all(
                        |((when1, then1), (when2, then2))| {
                            when1.eq(when2) && then1.eq(then2)
                        },
                    )
            })
            .unwrap_or(false)
    }

};
let else_expr_eq = match (&self.else_expr, &x.else_expr) {
(Some(expr1), Some(expr2)) => expr1.eq(expr2),
(None, None) => true,
_ => false,
};
expr_eq
&& else_expr_eq
&& self.when_then_expr.len() == x.when_then_expr.len()
&& self.when_then_expr.iter().zip(x.when_then_expr.iter()).all(
|((when1, then1), (when2, then2))| {
when1.eq(when2) && then1.eq(then2)
},
)
})
.unwrap_or(false)
}
}

/// Create a CASE expression
Expand All @@ -303,6 +390,7 @@ mod tests {
use crate::expressions::col;
use crate::expressions::lit;
use crate::expressions::{binary, cast};
use crate::rewrite::TreeNodeRewritable;
use arrow::array::StringArray;
use arrow::buffer::Buffer;
use arrow::datatypes::DataType::Float64;
Expand Down Expand Up @@ -728,6 +816,119 @@ mod tests {
Ok(())
}

#[test]
fn case_eq() -> Result<()> {
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);

let when1 = lit("foo");
let then1 = lit(123i32);
let when2 = lit("bar");
let then2 = lit(456i32);
let else_value = lit(999i32);

let expr1 = generate_case_when_with_type_coercion(
Some(col("a", &schema)?),
vec![
(when1.clone(), then1.clone()),
(when2.clone(), then2.clone()),
],
Some(else_value.clone()),
&schema,
)?;

let expr2 = generate_case_when_with_type_coercion(
Some(col("a", &schema)?),
vec![
(when1.clone(), then1.clone()),
(when2.clone(), then2.clone()),
],
Some(else_value.clone()),
&schema,
)?;

let expr3 = generate_case_when_with_type_coercion(
Some(col("a", &schema)?),
vec![(when1.clone(), then1.clone()), (when2, then2)],
None,
&schema,
)?;

let expr4 = generate_case_when_with_type_coercion(
Some(col("a", &schema)?),
vec![(when1, then1)],
Some(else_value),
&schema,
)?;

assert!(expr1.eq(&expr2));
assert!(expr2.eq(&expr1));

assert!(expr2.ne(&expr3));
assert!(expr3.ne(&expr2));

assert!(expr1.ne(&expr4));
assert!(expr4.ne(&expr1));

Ok(())
}

#[test]
fn case_tranform() -> Result<()> {
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);

let when1 = lit("foo");
let then1 = lit(123i32);
let when2 = lit("bar");
let then2 = lit(456i32);
let else_value = lit(999i32);

let expr = generate_case_when_with_type_coercion(
Some(col("a", &schema)?),
vec![
(when1.clone(), then1.clone()),
(when2.clone(), then2.clone()),
],
Some(else_value.clone()),
&schema,
)?;

let expr2 = expr
.clone()
.transform(
&|e| match e.as_any().downcast_ref::<crate::expressions::Literal>() {
Some(lit_value) => match lit_value.value() {
ScalarValue::Utf8(Some(str_value)) => {
Some(lit(str_value.to_uppercase()))
}
_ => None,
},
_ => None,
},
)
.unwrap();

let expr3 = expr
.clone()
.transform_down(&|e| match e
.as_any()
.downcast_ref::<crate::expressions::Literal>()
{
Some(lit_value) => match lit_value.value() {
ScalarValue::Utf8(Some(str_value)) => {
Some(lit(str_value.to_uppercase()))
}
_ => None,
},
_ => None,
})
.unwrap();

assert!(expr.ne(&expr2));
assert!(expr2.eq(&expr3));

Ok(())
}

fn generate_case_when_with_type_coercion(
expr: Option<Arc<dyn PhysicalExpr>>,
when_thens: Vec<WhenThen>,
Expand Down
31 changes: 31 additions & 0 deletions datafusion/physical-expr/src/expressions/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use std::any::Any;
use std::fmt;
use std::sync::Arc;

use crate::physical_expr::down_cast_any_ref;
use crate::PhysicalExpr;
use arrow::compute;
use arrow::compute::kernels;
Expand Down Expand Up @@ -93,6 +94,36 @@ impl PhysicalExpr for CastExpr {
let value = self.expr.evaluate(batch)?;
cast_column(&value, &self.cast_type, &self.cast_options)
}

fn children(&self) -> Vec<Arc<dyn PhysicalExpr>> {
vec![self.expr.clone()]
}

fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn PhysicalExpr>>,
) -> Result<Arc<dyn PhysicalExpr>> {
Ok(Arc::new(CastExpr::new(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

recommend assert / error check for children.len() == 1

children[0].clone(),
self.cast_type.clone(),
CastOptions {
safe: self.cast_options.safe,
},
)))
}
}

impl PartialEq<dyn Any> for CastExpr {
fn eq(&self, other: &dyn Any) -> bool {
down_cast_any_ref(other)
.downcast_ref::<Self>()
.map(|x| {
self.expr.eq(&x.expr)
&& self.cast_type == x.cast_type

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a todo note in #4012

&& self.cast_options.safe == x.cast_options.safe

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to be a bit error-prone, considering that cast_options might get new members and this part will not count them. Can we consider adding equality comparison support to CastOptions directly?

@isidentical isidentical Oct 28, 2022 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nevermind, it seems like we can't change it here. Created a ticket on arrow-rs.

})
.unwrap_or(false)
}
}

/// Internal cast function for casting ColumnarValue -> ColumnarValue for cast_type
Expand Down
Loading