diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index 54247cbcf07ce..658aad3512afa 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -79,6 +79,7 @@ num-traits = { version = "0.2", optional = true } pyo3 = { version = "0.15", optional = true } tempfile = "3" parking_lot = "0.12" +uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] criterion = "0.3" diff --git a/datafusion/src/datasource/listing/helpers.rs b/datafusion/src/datasource/listing/helpers.rs index 912179c36f060..97bf13ebcdfdc 100644 --- a/datafusion/src/datasource/listing/helpers.rs +++ b/datafusion/src/datasource/listing/helpers.rs @@ -107,7 +107,8 @@ impl ExpressionVisitor for ApplicabilityVisitor<'_> { | Expr::AggregateFunction { .. } | Expr::Sort { .. } | Expr::WindowFunction { .. } - | Expr::Wildcard => { + | Expr::Wildcard + | Expr::Exists(_) => { *self.is_applicable = false; Recursion::Stop(self) } diff --git a/datafusion/src/logical_plan/dfschema.rs b/datafusion/src/logical_plan/dfschema.rs index 7b6471f64dd73..e6cc03fd59352 100644 --- a/datafusion/src/logical_plan/dfschema.rs +++ b/datafusion/src/logical_plan/dfschema.rs @@ -32,7 +32,7 @@ use std::fmt::{Display, Formatter}; pub type DFSchemaRef = Arc; /// DFSchema wraps an Arrow schema and adds relation names -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct DFSchema { /// Fields fields: Vec, @@ -403,7 +403,7 @@ impl Display for DFSchema { } /// DFField wraps an Arrow field and adds an optional qualifier -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct DFField { /// Optional qualifier (usually a table or relation name) qualifier: Option, diff --git a/datafusion/src/logical_plan/expr.rs b/datafusion/src/logical_plan/expr.rs index 300c75137d8a0..811204a357813 100644 --- a/datafusion/src/logical_plan/expr.rs +++ b/datafusion/src/logical_plan/expr.rs @@ -373,6 +373,8 @@ pub enum Expr { }, /// Represents a reference to all fields in a schema. Wildcard, + /// Exists subquery + Exists(Box), } /// Fixed seed for the hashing so that Ords are consistent across runs @@ -508,9 +510,163 @@ impl Expr { get_indexed_field(&data_type, key).map(|x| x.data_type().clone()) } + Expr::Exists(logical_plan) => { + let df_fields = logical_plan.schema().fields(); + let mut fields = Vec::with_capacity(df_fields.len()); + let _ = df_fields + .iter() + .map(|df_field| fields.push(df_field.field().clone())); + match fields.len() { + 1 => Ok(fields[0].data_type().clone()), + _ => Ok(DataType::Struct(fields)), + } + } } } + /// rewrite the subquery to apply to the outer logical plan + pub fn rewrite_to(&self, outer: &mut LogicalPlan) -> Result { + Ok(match self { + Expr::Alias(expr, name) => { + Expr::Alias(Box::new(expr.rewrite_to(outer).unwrap()), name.clone()) + } + Expr::Column(col) => Expr::Column(col.clone()), + Expr::ScalarVariable(vars) => Expr::ScalarVariable(vars.clone()), + Expr::Literal(val) => Expr::Literal(val.clone()), + Expr::BinaryExpr { left, op, right } => Expr::BinaryExpr { + left: Box::new(left.rewrite_to(outer).unwrap()), + op: op.clone(), + right: Box::new(right.rewrite_to(outer).unwrap()), + }, + Expr::Not(expr) => Expr::Not(Box::new(expr.rewrite_to(outer).unwrap())), + Expr::IsNotNull(expr) => { + Expr::IsNotNull(Box::new(expr.rewrite_to(outer).unwrap())) + } + Expr::IsNull(expr) => Expr::IsNull(Box::new(expr.rewrite_to(outer).unwrap())), + Expr::Negative(expr) => { + Expr::Negative(Box::new(expr.rewrite_to(outer).unwrap())) + } + Expr::GetIndexedField { expr, key } => Expr::GetIndexedField { + expr: Box::new(expr.rewrite_to(outer).unwrap()), + key: key.clone(), + }, + Expr::Between { + expr, + negated, + low, + high, + } => Expr::Between { + expr: Box::new(expr.rewrite_to(outer).unwrap()), + negated: *negated, + low: Box::new(low.rewrite_to(outer).unwrap()), + high: Box::new(high.rewrite_to(outer).unwrap()), + }, + Expr::Case { + expr, + when_then_expr, + else_expr, + } => Expr::Case { + expr: if let Some(e) = expr { + Option::from(Box::new(e.rewrite_to(outer).unwrap())) + } else { + None + }, + when_then_expr: when_then_expr + .into_iter() + .map(|(when, then)| { + ( + Box::new(when.rewrite_to(outer).unwrap()), + Box::new(then.rewrite_to(outer).unwrap()), + ) + }) + .collect(), + else_expr: if let Some(e) = else_expr { + Option::from(Box::new(e.rewrite_to(outer).unwrap())) + } else { + None + }, + }, + Expr::Cast { expr, data_type } => Expr::Cast { + expr: Box::new(expr.rewrite_to(outer).unwrap()), + data_type: data_type.clone(), + }, + Expr::TryCast { expr, data_type } => Expr::TryCast { + expr: Box::new(expr.rewrite_to(outer).unwrap()), + data_type: data_type.clone(), + }, + Expr::Sort { + expr, + asc, + nulls_first, + } => Expr::Sort { + expr: Box::new(expr.rewrite_to(outer).unwrap()), + asc: *asc, + nulls_first: *nulls_first, + }, + Expr::ScalarFunction { fun, args } => Expr::ScalarFunction { + fun: fun.clone(), + args: args.iter().map(|a| a.rewrite_to(outer).unwrap()).collect(), + }, + Expr::ScalarUDF { fun, args } => Expr::ScalarUDF { + fun: fun.clone(), + args: args.iter().map(|a| a.rewrite_to(outer).unwrap()).collect(), + }, + Expr::AggregateFunction { + fun, + args, + distinct, + } => Expr::AggregateFunction { + fun: fun.clone(), + args: args.iter().map(|a| a.rewrite_to(outer).unwrap()).collect(), + distinct: *distinct, + }, + Expr::WindowFunction { + fun, + args, + partition_by, + order_by, + window_frame, + } => Expr::WindowFunction { + fun: fun.clone(), + args: args.iter().map(|a| a.rewrite_to(outer).unwrap()).collect(), + partition_by: partition_by + .iter() + .map(|e| e.rewrite_to(outer).unwrap()) + .collect(), + order_by: order_by + .iter() + .map(|e| e.rewrite_to(outer).unwrap()) + .collect(), + window_frame: window_frame.clone(), + }, + Expr::AggregateUDF { fun, args } => Expr::AggregateUDF { + fun: fun.clone(), + args: args.iter().map(|a| a.rewrite_to(outer).unwrap()).collect(), + }, + Expr::InList { + expr, + list, + negated, + } => Expr::InList { + expr: Box::new(expr.rewrite_to(outer).unwrap()), + list: list.iter().map(|e| e.rewrite_to(outer).unwrap()).collect(), + negated: *negated, + }, + Expr::Wildcard => Expr::Wildcard, + Expr::Exists(logical_plan) => { + *outer = outer + .clone() + .process_subquery(logical_plan.as_ref()) + .unwrap(); + // finally, return the column contains `bool` + Expr::Column(Column { + relation: None, + name: "exists_subquery_col".to_string(), + }) + } + }) + } + /// Returns the nullability of the expression based on [ExprSchema]. /// /// Note: [DFSchema] implements [ExprSchema]. @@ -569,6 +725,14 @@ impl Expr { let data_type = expr.get_type(input_schema)?; get_indexed_field(&data_type, key).map(|x| x.is_nullable()) } + Expr::Exists(logical_plan) => { + for df_field in logical_plan.schema().fields() { + if !df_field.is_nullable() { + return Ok(false); + } + } + Ok(true) + } } } @@ -752,7 +916,7 @@ impl Expr { /// pre_visit(Column("foo")) /// pre_visit(Column("bar")) /// post_visit(Column("bar")) - /// post_visit(Column("bar")) + /// post_visit(Column("foo")) /// post_visit(BinaryExpr(GT)) /// ``` /// @@ -783,7 +947,8 @@ impl Expr { Expr::Column(_) | Expr::ScalarVariable(_) | Expr::Literal(_) - | Expr::Wildcard => Ok(visitor), + | Expr::Wildcard + | Expr::Exists(_) => Ok(visitor), Expr::BinaryExpr { left, right, .. } => { let visitor = left.accept(visitor)?; right.accept(visitor) @@ -872,7 +1037,7 @@ impl Expr { /// ```text /// pre_visit(BinaryExpr(GT)) /// pre_visit(Column("foo")) - /// mutatate(Column("foo")) + /// mutate(Column("foo")) /// pre_visit(Column("bar")) /// mutate(Column("bar")) /// mutate(BinaryExpr(GT)) @@ -1010,6 +1175,7 @@ impl Expr { expr: rewrite_boxed(expr, rewriter)?, key, }, + Expr::Exists(logical_plan) => Expr::Exists(logical_plan), }; // now rewrite this expression itself @@ -2063,6 +2229,9 @@ impl fmt::Debug for Expr { Expr::GetIndexedField { ref expr, key } => { write!(f, "({:?})[{}]", expr, key) } + Expr::Exists(logical_plan) => { + write!(f, "exists subquery: {:?}", logical_plan) + } } } } @@ -2224,6 +2393,9 @@ fn create_name(e: &Expr, input_schema: &DFSchema) -> Result { Expr::Wildcard => Err(DataFusionError::Internal( "Create name does not support wildcard".to_string(), )), + Expr::Exists(logical_plan) => { + Ok(format!("Exists subquery {}", logical_plan.display())) + } } } diff --git a/datafusion/src/logical_plan/plan.rs b/datafusion/src/logical_plan/plan.rs index 3d49e5484eab1..7dd326b1f6665 100644 --- a/datafusion/src/logical_plan/plan.rs +++ b/datafusion/src/logical_plan/plan.rs @@ -20,20 +20,28 @@ use super::display::{GraphvizVisitor, IndentVisitor}; use super::expr::{Column, Expr}; use super::extension::UserDefinedLogicalNode; +use crate::datasource::empty::EmptyTable; use crate::datasource::TableProvider; -use crate::error::DataFusionError; +use crate::error::{DataFusionError, Result}; +use crate::from_slice::FromSlice; +use crate::logical_plan::builder::project_with_alias; use crate::logical_plan::dfschema::DFSchemaRef; +use crate::logical_plan::{col, LogicalPlanBuilder}; use crate::sql::parser::FileType; +use arrow::array::Int8Array; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; use std::fmt::Formatter; +use std::hash::{Hash, Hasher}; use std::{ collections::HashSet, fmt::{self, Display}, sync::Arc, }; +use uuid::Uuid; /// Join type -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum JoinType { /// Inner Join Inner, @@ -64,7 +72,7 @@ impl Display for JoinType { } /// Join constraint -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Hash)] pub enum JoinConstraint { /// Join ON On, @@ -74,7 +82,7 @@ pub enum JoinConstraint { /// Evaluates an arbitrary list of expressions (essentially a /// SELECT with an expression list) on its input. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Projection { /// The list of expressions pub expr: Vec, @@ -94,7 +102,7 @@ pub struct Projection { /// If the value of `` is true, the input row is passed to /// the output. If the value of `` is false, the row is /// discarded. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Filter { /// The predicate expression, which must have Boolean type. pub predicate: Expr, @@ -103,7 +111,7 @@ pub struct Filter { } /// Window its input based on a set of window spec and window function (e.g. SUM or RANK) -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Window { /// The incoming logical plan pub input: Arc, @@ -130,8 +138,20 @@ pub struct TableScan { pub limit: Option, } +impl PartialEq for TableScan { + fn eq(&self, other: &Self) -> bool { + self.table_name == other.table_name + } +} + +impl Hash for TableScan { + fn hash(&self, _state: &mut H) { + todo!() + } +} + /// Apply Cross Join to two logical plans -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct CrossJoin { /// Left input pub left: Arc, @@ -142,7 +162,7 @@ pub struct CrossJoin { } /// Repartition the plan based on a partitioning scheme. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Repartition { /// The incoming logical plan pub input: Arc, @@ -151,7 +171,7 @@ pub struct Repartition { } /// Union multiple inputs -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Union { /// Inputs to merge pub inputs: Vec, @@ -162,7 +182,7 @@ pub struct Union { } /// Creates an in memory table. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct CreateMemoryTable { /// The table name pub name: String, @@ -171,7 +191,7 @@ pub struct CreateMemoryTable { } /// Creates an external table. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct CreateExternalTable { /// The table schema pub schema: DFSchemaRef, @@ -186,7 +206,7 @@ pub struct CreateExternalTable { } /// Drops a table. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct DropTable { /// The table name pub name: String, @@ -198,7 +218,7 @@ pub struct DropTable { /// Produces a relation with string representations of /// various parts of the plan -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Explain { /// Should extra (detailed, intermediate plans) be included? pub verbose: bool, @@ -212,7 +232,7 @@ pub struct Explain { /// Runs the actual plan, and then prints the physical plan with /// with execution metrics. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Analyze { /// Should extra detail be included? pub verbose: bool, @@ -229,8 +249,20 @@ pub struct Extension { pub node: Arc, } +impl PartialEq for Extension { + fn eq(&self, _other: &Self) -> bool { + todo!() + } +} + +impl Hash for Extension { + fn hash(&self, _state: &mut H) { + todo!() + } +} + /// Produces no rows: An empty relation with an empty schema -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct EmptyRelation { /// Whether to produce a placeholder row pub produce_one_row: bool, @@ -239,7 +271,7 @@ pub struct EmptyRelation { } /// Produces the first `n` tuples from its input and discards the rest. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Limit { /// The limit pub n: usize, @@ -250,7 +282,7 @@ pub struct Limit { /// Values expression. See /// [Postgres VALUES](https://www.postgresql.org/docs/current/queries-values.html) /// documentation for more details. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Values { /// The table schema pub schema: DFSchemaRef, @@ -260,7 +292,7 @@ pub struct Values { /// Aggregates its input based on a set of grouping and aggregate /// expressions (e.g. SUM). -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Aggregate { /// The incoming logical plan pub input: Arc, @@ -273,7 +305,7 @@ pub struct Aggregate { } /// Sorts its input according to a list of sort expressions. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Sort { /// The sort expressions pub expr: Vec, @@ -282,7 +314,7 @@ pub struct Sort { } /// Join two logical plans on one or more join columns -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub struct Join { /// Left input pub left: Arc, @@ -299,6 +331,7 @@ pub struct Join { /// If null_equals_null is true, null == null else null != null pub null_equals_null: bool, } + /// A LogicalPlan represents the different types of relational /// operators (such as Projection, Filter, etc) and can be created by /// the SQL query planner and the DataFrame API. @@ -307,7 +340,7 @@ pub struct Join { /// an output relation (table) with a (potentially) different /// schema. A plan represents a dataflow tree where data flows /// from leaves up to the root to produce the query result. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] pub enum LogicalPlan { /// Evaluates an arbitrary list of expressions (essentially a /// SELECT with an expression list) on its input. @@ -526,7 +559,7 @@ impl LogicalPlan { } /// returns all `Using` join columns in a logical plan - pub fn using_columns(&self) -> Result>, DataFusionError> { + pub fn using_columns(&self) -> Result>> { struct UsingJoinColumnVisitor { using_columns: Vec>, } @@ -534,7 +567,7 @@ impl LogicalPlan { impl PlanVisitor for UsingJoinColumnVisitor { type Error = DataFusionError; - fn pre_visit(&mut self, plan: &LogicalPlan) -> Result { + fn pre_visit(&mut self, plan: &LogicalPlan) -> Result { if let LogicalPlan::Join(Join { join_constraint: JoinConstraint::Using, on, @@ -559,10 +592,184 @@ impl LogicalPlan { self.accept(&mut visitor)?; Ok(visitor.using_columns) } + + /// rewrite logical plan to flatten the subqueries + pub fn rewrite_to(&self, outer: LogicalPlan) -> Result { + if let LogicalPlan::TableScan(TableScan { .. }) = &outer { + } else { + return Err(DataFusionError::Plan(format!( + "outer_table must be a TableScan, but found: {:?}", + outer + ))); + } + match self { + LogicalPlan::Projection(Projection { + expr, + input, + schema: _, + alias, + }) => { + let outer_cols = outer + .schema() + .fields() + .into_iter() + .map(|field| col(field.name())) + .collect::>(); + let outer_projection = outer_cols + .into_iter() + .chain(expr.clone().into_iter()) + .collect::>(); + let input = input.rewrite_to(outer).unwrap(); + project_with_alias(input, outer_projection, alias.clone()) + } + LogicalPlan::Filter(Filter { predicate, input }) => { + let mut input = input.rewrite_to(outer).unwrap().clone(); + dbg!(input.schema().clone()); + // `predicate` may contain subqueries, so need rewrite to process subqueries + let new_predicate = predicate.rewrite_to(&mut input).unwrap(); + dbg!(new_predicate.clone()); + // use `new_predicate` instead of `predicate + // todo: delete the columns that are used to process subqueries + Ok(LogicalPlanBuilder::from(input) + .filter(new_predicate) + .unwrap() + .build() + .unwrap()) + } + LogicalPlan::Window(Window { .. }) => todo!(), + LogicalPlan::Aggregate(Aggregate { .. }) => todo!(), + LogicalPlan::Sort(Sort { .. }) => todo!(), + LogicalPlan::Join(Join { .. }) => todo!(), + LogicalPlan::CrossJoin(CrossJoin { .. }) => todo!(), + LogicalPlan::Repartition(Repartition { .. }) => todo!(), + LogicalPlan::Union(Union { inputs, .. }) => { + let mut first = inputs[0].rewrite_to(outer).unwrap(); + for input in inputs.into_iter().skip(1) { + first = LogicalPlanBuilder::from(first) + .union(input.clone())? + .build() + .unwrap() + } + Ok(first) + } + LogicalPlan::TableScan(TableScan { + table_name, + source, + projection, + projected_schema, + filters, + limit, + }) => LogicalPlanBuilder::from(outer) + .cross_join(&LogicalPlan::TableScan(TableScan { + table_name: table_name.to_string(), + source: source.clone(), + projection: projection.clone(), + projected_schema: projected_schema.clone(), + filters: filters.clone(), + limit: limit.clone(), + }))? + .build(), + LogicalPlan::EmptyRelation(EmptyRelation { .. }) => todo!(), + LogicalPlan::Limit(Limit { .. }) => todo!(), + LogicalPlan::CreateExternalTable(CreateExternalTable { .. }) => todo!(), + LogicalPlan::CreateMemoryTable(CreateMemoryTable { .. }) => todo!(), + LogicalPlan::DropTable(DropTable { .. }) => todo!(), + LogicalPlan::Values(Values { .. }) => todo!(), + LogicalPlan::Explain(Explain { .. }) => todo!(), + LogicalPlan::Analyze(Analyze { .. }) => todo!(), + LogicalPlan::Extension(Extension { .. }) => todo!(), + } + } + // Get TableScan from Logical Plan + fn get_table_scan(&self) -> Result { + Ok(if let LogicalPlan::TableScan(TableScan { .. }) = self { + self.clone() + } else { + let name = Uuid::new_v4().to_string(); + let schema = SchemaRef::new(Schema::new( + self.schema() + .fields() + .iter() + .map(|f| f.field().clone()) + .collect(), + )); + dbg!(schema.clone()); + let fields_size = self.schema().fields().len(); + LogicalPlanBuilder::scan( + name, + Arc::new(EmptyTable::new(schema)), + Option::from((0..fields_size).collect::>()), + )? + .build() + .unwrap() + }) + } + + /// process subqueries + pub fn process_subquery(self, expr: &LogicalPlan) -> Result { + let outer_table = self.get_table_scan().unwrap(); + let distinct_outer_table = LogicalPlanBuilder::from(outer_table.clone()) + .distinct()? + .build() + .unwrap(); + dbg!(distinct_outer_table.schema().fields()); // yes + let result_plan = distinct_outer_table + .get_table_scan() + .unwrap() + .compute(expr) + .unwrap(); + dbg!(result_plan.schema().fields()); + dbg!(outer_table.schema().fields()); + // join with outer_table + let join_keys = outer_table + .schema() + .fields() + .iter() + .zip( + result_plan.schema().fields().as_slice() + [0..outer_table.schema().fields().len()] + .iter(), + ) + .map(|(left_field, right_field)| { + ( + (Column::from_name(left_field.name())), + (Column::from_name(right_field.name())), + ) + }) + .unzip(); + LogicalPlanBuilder::from(outer_table) + .join(&result_plan, JoinType::Inner, join_keys)? + .build() + } + + /// compute every row in outer_table for subquery + fn compute(self, expr: &LogicalPlan) -> Result { + let compute_result = + LogicalPlanBuilder::from(expr.clone().rewrite_to(self).unwrap()) + .distinct()? + .build() + .unwrap(); + dbg!(compute_result.schema().clone()); + // Add a column contains true to compute_result + // 1. create a table contains one col on row, data type: bool, value: true + // 2. cross join with `compute_result` + let schema = SchemaRef::new(Schema::new(vec![Field::new( + "exists_subquery_col", + DataType::Int8, + false, + )])); + let partitions = vec![vec![RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int8Array::from_slice(&[1]))], + )?]]; + let plan_builder = + LogicalPlanBuilder::scan_memory(partitions, schema, None).unwrap(); + plan_builder.cross_join(&compute_result).unwrap().build() + } } /// Logical partitioning schemes supported by the repartition operator. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] pub enum Partitioning { /// Allocate batches using a round-robin algorithm and the specified number of partitions RoundRobinBatch(usize), @@ -1025,7 +1232,7 @@ impl fmt::Debug for LogicalPlan { /// Represents which type of plan, when storing multiple /// for use in EXPLAIN plans -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Hash)] pub enum PlanType { /// The initial LogicalPlan provided to DataFusion InitialLogicalPlan, @@ -1065,7 +1272,7 @@ impl fmt::Display for PlanType { } /// Represents some sort of execution plan, in String form -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Hash)] #[allow(clippy::rc_buffer)] pub struct StringifiedPlan { /// An identifier of what type of plan this string represents diff --git a/datafusion/src/optimizer/common_subexpr_eliminate.rs b/datafusion/src/optimizer/common_subexpr_eliminate.rs index 947073409d056..f47efca33696b 100644 --- a/datafusion/src/optimizer/common_subexpr_eliminate.rs +++ b/datafusion/src/optimizer/common_subexpr_eliminate.rs @@ -457,6 +457,9 @@ impl ExprIdentifierVisitor<'_> { desc.push_str("GetIndexedField-"); desc.push_str(&key.to_string()); } + Expr::Exists(_) => { + desc.push_str("Exists-subquery"); + } } desc diff --git a/datafusion/src/optimizer/simplify_expressions.rs b/datafusion/src/optimizer/simplify_expressions.rs index c000bdbc2beae..b21c117611f40 100644 --- a/datafusion/src/optimizer/simplify_expressions.rs +++ b/datafusion/src/optimizer/simplify_expressions.rs @@ -375,7 +375,8 @@ impl<'a> ConstEvaluator<'a> { | Expr::Column(_) | Expr::WindowFunction { .. } | Expr::Sort { .. } - | Expr::Wildcard => false, + | Expr::Wildcard + | Expr::Exists(_) => false, Expr::ScalarFunction { fun, .. } => Self::volatility_ok(fun.volatility()), Expr::ScalarUDF { fun, .. } => Self::volatility_ok(fun.signature.volatility), Expr::Literal(_) diff --git a/datafusion/src/optimizer/utils.rs b/datafusion/src/optimizer/utils.rs index f7ab836b398cb..74ef1b8d24a34 100644 --- a/datafusion/src/optimizer/utils.rs +++ b/datafusion/src/optimizer/utils.rs @@ -82,7 +82,8 @@ impl ExpressionVisitor for ColumnNameVisitor<'_> { | Expr::AggregateUDF { .. } | Expr::InList { .. } | Expr::Wildcard - | Expr::GetIndexedField { .. } => {} + | Expr::GetIndexedField { .. } + | Expr::Exists(_) => {} } Ok(Recursion::Continue(self)) } @@ -347,6 +348,7 @@ pub fn expr_sub_expressions(expr: &Expr) -> Result> { Expr::Wildcard { .. } => Err(DataFusionError::Internal( "Wildcard expressions are not valid in a logical query plan".to_owned(), )), + Expr::Exists(_) => Ok(vec![]), } } @@ -474,7 +476,8 @@ pub fn rewrite_expression(expr: &Expr, expressions: &[Expr]) -> Result { Expr::Column(_) | Expr::Literal(_) | Expr::InList { .. } - | Expr::ScalarVariable(_) => Ok(expr.clone()), + | Expr::ScalarVariable(_) + | Expr::Exists(_) => Ok(expr.clone()), Expr::Sort { asc, nulls_first, .. } => Ok(Expr::Sort { diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/src/physical_plan/planner.rs index bf8be3df720bd..734a8b5681413 100644 --- a/datafusion/src/physical_plan/planner.rs +++ b/datafusion/src/physical_plan/planner.rs @@ -207,6 +207,9 @@ fn create_physical_name(e: &Expr, is_first_expr: bool) -> Result { Expr::Wildcard => Err(DataFusionError::Internal( "Create physical name does not support wildcard".to_string(), )), + Expr::Exists(_) => Err(DataFusionError::Internal( + "Create physical name does not support wildcard".to_string(), + )), } } diff --git a/datafusion/src/sql/parser.rs b/datafusion/src/sql/parser.rs index 3352577644429..1ee11d99b1547 100644 --- a/datafusion/src/sql/parser.rs +++ b/datafusion/src/sql/parser.rs @@ -35,7 +35,7 @@ macro_rules! parser_err { } /// Types of files to parse as DataFrames -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Hash)] pub enum FileType { /// Newline-delimited JSON NdJson, diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 462977274ecbb..0f76cf847f99b 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -66,6 +66,7 @@ use super::{ resolve_aliases_to_exprs, resolve_positions_to_exprs, }, }; +use crate::datasource::empty::EmptyTable; use crate::logical_plan::builder::project_with_alias; use crate::logical_plan::plan::{Analyze, Explain}; @@ -85,6 +86,30 @@ pub struct SqlToRel<'a, S: ContextProvider> { schema_provider: &'a S, } +/// Subqueries context, in the future, maybe instead by QueryCtx +pub struct SubqueryCtx { + /// If query contains subquery + has_subquery: bool, + // todo If sql contains correlated subquery, need get the outer table contained in subquery +} + +impl SubqueryCtx { + /// New a SubqueryCtx + pub fn new(has_subquery: bool) -> Self { + SubqueryCtx { has_subquery } + } + + /// Get the has_subquery, the decide if rewrite logical plan + pub fn has_subquery(&self) -> bool { + self.has_subquery + } + + /// When encounter subquery, set has_subquery to **true** + pub fn set_has_subquery(&mut self, has_subquery: bool) { + self.has_subquery = has_subquery + } +} + fn plan_key(key: Value) -> ScalarValue { match key { Value::Number(s, _) => ScalarValue::Int64(Some(s.parse().unwrap())), @@ -121,12 +146,36 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { pub fn statement_to_plan(&self, statement: &DFStatement) -> Result { match statement { DFStatement::CreateExternalTable(s) => self.external_table_to_plan(s), - DFStatement::Statement(s) => self.sql_statement_to_plan(s), + DFStatement::Statement(s) => { + let mut subquery_ctx = SubqueryCtx::new(false); + let plan = self.sql_statement_to_plan(s, &mut subquery_ctx); + // If contains subqueries, rewrite the plan to process subqueries in logical plan level + if subquery_ctx.has_subquery() { + return self.rewrite_plan(&mut plan.unwrap()); + } + plan + } } } + /// Rewrite the logical plan generated by `statement_to_plan`, include processing subquery + fn rewrite_plan(&self, plan: &mut LogicalPlan) -> Result { + let outer_table = LogicalPlanBuilder::scan( + "outer_table".to_string(), + Arc::new(EmptyTable::new(SchemaRef::new(Schema::new(vec![])))), + None, + )? + .build() + .unwrap(); + plan.clone().rewrite_to(outer_table) + } + /// Generate a logical plan from an SQL statement - pub fn sql_statement_to_plan(&self, sql: &Statement) -> Result { + pub fn sql_statement_to_plan( + &self, + sql: &Statement, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { match sql { Statement::Explain { verbose, @@ -134,7 +183,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { analyze, describe_alias: _, } => self.explain_statement_to_plan(*verbose, *analyze, statement), - Statement::Query(query) => self.query_to_plan(query), + Statement::Query(query) => self.query_to_plan(query, subquery_ctx), Statement::ShowVariable { variable } => self.show_variable_to_plan(variable), Statement::CreateTable { query: Some(query), @@ -158,7 +207,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { && table_properties.is_empty() && with_options.is_empty() => { - let plan = self.query_to_plan(query)?; + let plan = self.query_to_plan(query, subquery_ctx)?; Ok(LogicalPlan::CreateMemoryTable(CreateMemoryTable { name: name.to_string(), @@ -200,8 +249,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } /// Generate a logic plan from an SQL query - pub fn query_to_plan(&self, query: &Query) -> Result { - self.query_to_plan_with_alias(query, None, &mut HashMap::new()) + pub fn query_to_plan( + &self, + query: &Query, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { + self.query_to_plan_with_alias(query, None, &mut HashMap::new(), subquery_ctx) } /// Generate a logic plan from an SQL query with optional alias @@ -210,6 +263,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { query: &Query, alias: Option, ctes: &mut HashMap, + subquery_ctx: &mut SubqueryCtx, ) -> Result { let set_expr = &query.body; if let Some(with) = &query.with { @@ -229,15 +283,24 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &cte.query, Some(cte.alias.name.value.clone()), &mut ctes.clone(), + subquery_ctx, )?; ctes.insert(cte.alias.name.value.clone(), logical_plan); } } - let plan = self.set_expr_to_plan(set_expr, alias, ctes)?; + let plan = self.set_expr_to_plan(set_expr, alias, ctes, subquery_ctx)?; - let plan = self.order_by(plan, &query.order_by)?; + let plan = self.order_by(plan, &query.order_by, subquery_ctx)?; - self.limit(plan, &query.limit) + self.limit(plan, &query.limit, subquery_ctx) + } + + fn plan_subquery( + &self, + query: &Query, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { + self.query_to_plan(query, subquery_ctx) } fn set_expr_to_plan( @@ -245,18 +308,23 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { set_expr: &SetExpr, alias: Option, ctes: &mut HashMap, + subquery_ctx: &mut SubqueryCtx, ) -> Result { match set_expr { - SetExpr::Select(s) => self.select_to_plan(s.as_ref(), ctes, alias), - SetExpr::Values(v) => self.sql_values_to_plan(v), + SetExpr::Select(s) => { + self.select_to_plan(s.as_ref(), ctes, alias, subquery_ctx) + } + SetExpr::Values(v) => self.sql_values_to_plan(v, subquery_ctx), SetExpr::SetOperation { op, left, right, all, } => { - let left_plan = self.set_expr_to_plan(left.as_ref(), None, ctes)?; - let right_plan = self.set_expr_to_plan(right.as_ref(), None, ctes)?; + let left_plan = + self.set_expr_to_plan(left.as_ref(), None, ctes, subquery_ctx)?; + let right_plan = + self.set_expr_to_plan(right.as_ref(), None, ctes, subquery_ctx)?; match (op, all) { (SetOperator::Union, true) => { union_with_alias(left_plan, right_plan, alias) @@ -279,6 +347,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } } } + SetExpr::Query(q) => { + let plan = self.plan_subquery(q, subquery_ctx)?; + Ok(plan) + } _ => Err(DataFusionError::NotImplemented(format!( "Query {} not implemented yet", set_expr @@ -333,7 +405,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { analyze: bool, statement: &Statement, ) -> Result { - let plan = self.sql_statement_to_plan(statement)?; + let mut subquery_ctx = SubqueryCtx::new(false); + let plan = self.sql_statement_to_plan(statement, &mut subquery_ctx)?; let plan = Arc::new(plan); let schema = LogicalPlan::explain_schema(); let schema = schema.to_dfschema_ref()?; @@ -401,12 +474,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, from: &[TableWithJoins], ctes: &mut HashMap, + subquery_ctx: &mut SubqueryCtx, ) -> Result> { match from.len() { 0 => Ok(vec![LogicalPlanBuilder::empty(true).build()?]), _ => from .iter() - .map(|t| self.plan_table_with_joins(t, ctes)) + .map(|t| self.plan_table_with_joins(t, ctes, subquery_ctx)) .collect::>>(), } } @@ -415,14 +489,17 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, t: &TableWithJoins, ctes: &mut HashMap, + subquery_ctx: &mut SubqueryCtx, ) -> Result { - let left = self.create_relation(&t.relation, ctes)?; + let left = self.create_relation(&t.relation, ctes, subquery_ctx)?; match t.joins.len() { 0 => Ok(left), n => { - let mut left = self.parse_relation_join(left, &t.joins[0], ctes)?; + let mut left = + self.parse_relation_join(left, &t.joins[0], ctes, subquery_ctx)?; for i in 1..n { - left = self.parse_relation_join(left, &t.joins[i], ctes)?; + left = + self.parse_relation_join(left, &t.joins[i], ctes, subquery_ctx)?; } Ok(left) } @@ -434,20 +511,21 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { left: LogicalPlan, join: &Join, ctes: &mut HashMap, + subquery_ctx: &mut SubqueryCtx, ) -> Result { - let right = self.create_relation(&join.relation, ctes)?; + let right = self.create_relation(&join.relation, ctes, subquery_ctx)?; match &join.join_operator { JoinOperator::LeftOuter(constraint) => { - self.parse_join(left, right, constraint, JoinType::Left) + self.parse_join(left, right, constraint, JoinType::Left, subquery_ctx) } JoinOperator::RightOuter(constraint) => { - self.parse_join(left, right, constraint, JoinType::Right) + self.parse_join(left, right, constraint, JoinType::Right, subquery_ctx) } JoinOperator::Inner(constraint) => { - self.parse_join(left, right, constraint, JoinType::Inner) + self.parse_join(left, right, constraint, JoinType::Inner, subquery_ctx) } JoinOperator::FullOuter(constraint) => { - self.parse_join(left, right, constraint, JoinType::Full) + self.parse_join(left, right, constraint, JoinType::Full, subquery_ctx) } JoinOperator::CrossJoin => self.parse_cross_join(left, &right), other => Err(DataFusionError::NotImplemented(format!( @@ -470,6 +548,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { right: LogicalPlan, constraint: &JoinConstraint, join_type: JoinType, + subquery_ctx: &mut SubqueryCtx, ) -> Result { match constraint { JoinConstraint::On(sql_expr) => { @@ -477,7 +556,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let join_schema = left.schema().join(right.schema())?; // parse ON expression - let expr = self.sql_to_rex(sql_expr, &join_schema)?; + let expr = self.sql_to_rex(sql_expr, &join_schema, subquery_ctx)?; // expression that didn't match equi-join pattern let mut filter = vec![]; @@ -602,6 +681,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, relation: &TableFactor, ctes: &mut HashMap, + subquery_ctx: &mut SubqueryCtx, ) -> Result { let (plan, alias) = match relation { TableFactor::Table { name, alias, .. } => { @@ -640,10 +720,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { "subquery in FROM must have an alias".to_string(), )); } + // subquery_ctx.set_has_subquery(true); let logical_plan = self.query_to_plan_with_alias( subquery, alias.as_ref().map(|a| a.name.value.to_string()), ctes, + subquery_ctx, )?; ( project_with_alias( @@ -658,9 +740,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { alias, ) } - TableFactor::NestedJoin(table_with_joins) => { - (self.plan_table_with_joins(table_with_joins, ctes)?, &None) - } + TableFactor::NestedJoin(table_with_joins) => ( + self.plan_table_with_joins(table_with_joins, ctes, subquery_ctx)?, + &None, + ), // @todo Support TableFactory::TableFunction? _ => { return Err(DataFusionError::NotImplemented(format!( @@ -703,6 +786,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, select: &Select, plans: Vec, + subquery_ctx: &mut SubqueryCtx, ) -> Result { let plan = match &select.selection { Some(predicate_expr) => { @@ -713,7 +797,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } let join_schema = DFSchema::new(fields)?; - let filter_expr = self.sql_to_rex(predicate_expr, &join_schema)?; + let filter_expr = + self.sql_to_rex(predicate_expr, &join_schema, subquery_ctx)?; // look for expressions of the form ` = ` let mut possible_join_keys = vec![]; @@ -830,15 +915,16 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { select: &Select, ctes: &mut HashMap, alias: Option, + subquery_ctx: &mut SubqueryCtx, ) -> Result { // process `from` clause - let plans = self.plan_from_tables(&select.from, ctes)?; + let plans = self.plan_from_tables(&select.from, ctes, subquery_ctx)?; // process `where` clause - let plan = self.plan_selection(select, plans)?; + let plan = self.plan_selection(select, plans, subquery_ctx)?; // process the SELECT expressions, with wildcards expanded. - let select_exprs = self.prepare_select_exprs(&plan, select)?; + let select_exprs = self.prepare_select_exprs(&plan, select, subquery_ctx)?; // having and group by clause may reference aliases defined in select projection let projected_plan = self.project(plan.clone(), select_exprs.clone())?; @@ -853,8 +939,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { .having .as_ref() .map::, _>(|having_expr| { - let having_expr = - self.sql_expr_to_logical_expr(having_expr, &combined_schema)?; + let having_expr = self.sql_expr_to_logical_expr( + having_expr, + &combined_schema, + subquery_ctx, + )?; // This step "dereferences" any aliases in the HAVING clause. // // This is how we support queries with HAVING expressions that @@ -891,7 +980,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { .group_by .iter() .map(|e| { - let group_by_expr = self.sql_expr_to_logical_expr(e, &combined_schema)?; + let group_by_expr = + self.sql_expr_to_logical_expr(e, &combined_schema, subquery_ctx)?; let group_by_expr = resolve_aliases_to_exprs(&group_by_expr, &alias_map)?; let group_by_expr = resolve_positions_to_exprs(&group_by_expr, &select_exprs) @@ -975,12 +1065,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, plan: &LogicalPlan, select: &Select, + subquery_ctx: &mut SubqueryCtx, ) -> Result> { let input_schema = plan.schema(); let projection = &select.projection; projection .iter() - .map(|expr| self.sql_select_to_rex(expr, input_schema)) + .map(|expr| self.sql_select_to_rex(expr, input_schema, subquery_ctx)) .collect::>>()? .into_iter() .map(|expr| { @@ -1073,10 +1164,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } /// Wrap a plan in a limit - fn limit(&self, input: LogicalPlan, limit: &Option) -> Result { + fn limit( + &self, + input: LogicalPlan, + limit: &Option, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { match *limit { Some(ref limit_expr) => { - let n = match self.sql_to_rex(limit_expr, input.schema())? { + let n = match self.sql_to_rex(limit_expr, input.schema(), subquery_ctx)? { Expr::Literal(ScalarValue::Int64(Some(n))) => Ok(n as usize), _ => Err(DataFusionError::Plan( "Unexpected expression for LIMIT clause".to_string(), @@ -1094,6 +1190,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, plan: LogicalPlan, order_by: &[OrderByExpr], + subquery_ctx: &mut SubqueryCtx, ) -> Result { if order_by.is_empty() { return Ok(plan); @@ -1101,14 +1198,19 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let order_by_rex = order_by .iter() - .map(|e| self.order_by_to_sort_expr(e, plan.schema())) + .map(|e| self.order_by_to_sort_expr(e, plan.schema(), subquery_ctx)) .collect::>>()?; LogicalPlanBuilder::from(plan).sort(order_by_rex)?.build() } /// convert sql OrderByExpr to Expr::Sort - fn order_by_to_sort_expr(&self, e: &OrderByExpr, schema: &DFSchema) -> Result { + fn order_by_to_sort_expr( + &self, + e: &OrderByExpr, + schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { let OrderByExpr { asc, expr, @@ -1136,7 +1238,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let field = schema.field(field_index - 1); Expr::Column(field.qualified_column()) } - e => self.sql_expr_to_logical_expr(e, schema)?, + e => self.sql_expr_to_logical_expr(e, schema, subquery_ctx)?, }; Ok({ let asc = asc.unwrap_or(true); @@ -1186,11 +1288,16 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } /// Generate a relational expression from a select SQL expression - fn sql_select_to_rex(&self, sql: &SelectItem, schema: &DFSchema) -> Result { + fn sql_select_to_rex( + &self, + sql: &SelectItem, + schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { match sql { - SelectItem::UnnamedExpr(expr) => self.sql_to_rex(expr, schema), + SelectItem::UnnamedExpr(expr) => self.sql_to_rex(expr, schema, subquery_ctx), SelectItem::ExprWithAlias { expr, alias } => Ok(Alias( - Box::new(self.sql_to_rex(expr, schema)?), + Box::new(self.sql_to_rex(expr, schema, subquery_ctx)?), alias.value.clone(), )), SelectItem::Wildcard => Ok(Expr::Wildcard), @@ -1201,8 +1308,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } /// Generate a relational expression from a SQL expression - pub fn sql_to_rex(&self, sql: &SQLExpr, schema: &DFSchema) -> Result { - let mut expr = self.sql_expr_to_logical_expr(sql, schema)?; + pub fn sql_to_rex( + &self, + sql: &SQLExpr, + schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { + let mut expr = self.sql_expr_to_logical_expr(sql, schema, subquery_ctx)?; expr = self.rewrite_partial_qualifier(expr, schema); self.validate_schema_satisfies_exprs(schema, &[expr.clone()])?; Ok(expr) @@ -1240,12 +1352,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, sql: &FunctionArg, schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, ) -> Result { match sql { FunctionArg::Named { name: _, arg } => { - self.sql_expr_to_logical_expr(arg, schema) + self.sql_expr_to_logical_expr(arg, schema, subquery_ctx) + } + FunctionArg::Unnamed(value) => { + self.sql_expr_to_logical_expr(value, schema, subquery_ctx) } - FunctionArg::Unnamed(value) => self.sql_expr_to_logical_expr(value, schema), } } @@ -1255,6 +1370,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { op: &BinaryOperator, right: &SQLExpr, schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, ) -> Result { let operator = match *op { BinaryOperator::Gt => Ok(Operator::Gt), @@ -1284,9 +1400,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { }?; Ok(Expr::BinaryExpr { - left: Box::new(self.sql_expr_to_logical_expr(left, schema)?), + left: Box::new(self.sql_expr_to_logical_expr(left, schema, subquery_ctx)?), op: operator, - right: Box::new(self.sql_expr_to_logical_expr(right, schema)?), + right: Box::new(self.sql_expr_to_logical_expr( + right, + schema, + subquery_ctx, + )?), }) } @@ -1295,12 +1415,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { op: &UnaryOperator, expr: &SQLExpr, schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, ) -> Result { match op { UnaryOperator::Not => Ok(Expr::Not(Box::new( - self.sql_expr_to_logical_expr(expr, schema)?, + self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?, ))), - UnaryOperator::Plus => Ok(self.sql_expr_to_logical_expr(expr, schema)?), + UnaryOperator::Plus => { + Ok(self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?) + } UnaryOperator::Minus => { match expr { // optimization: if it's a number literal, we apply the negative operator @@ -1316,7 +1439,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { })?)), }, // not a literal, apply negative operator on expression - _ => Ok(Expr::Negative(Box::new(self.sql_expr_to_logical_expr(expr, schema)?))), + _ => Ok(Expr::Negative(Box::new(self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?))), } } _ => Err(DataFusionError::NotImplemented(format!( @@ -1326,7 +1449,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } } - fn sql_values_to_plan(&self, values: &SQLValues) -> Result { + fn sql_values_to_plan( + &self, + values: &SQLValues, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { // values should not be based on any other schema let schema = DFSchema::empty(); let values = values @@ -1344,13 +1471,19 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } SQLExpr::Value(Value::Boolean(n)) => Ok(lit(*n)), SQLExpr::UnaryOp { ref op, ref expr } => { - self.parse_sql_unary_op(op, expr, &schema) + self.parse_sql_unary_op(op, expr, &schema, subquery_ctx) } SQLExpr::BinaryOp { ref left, ref op, ref right, - } => self.parse_sql_binary_op(left, op, right, &schema), + } => self.parse_sql_binary_op( + left, + op, + right, + &schema, + subquery_ctx, + ), other => Err(DataFusionError::NotImplemented(format!( "Unsupported value {:?} in a values list expression", other @@ -1362,7 +1495,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { LogicalPlanBuilder::values(values)?.build() } - fn sql_expr_to_logical_expr(&self, sql: &SQLExpr, schema: &DFSchema) -> Result { + fn sql_expr_to_logical_expr( + &self, + sql: &SQLExpr, + schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, + ) -> Result { match sql { SQLExpr::Value(Value::Number(n, _)) => parse_sql_number(n), SQLExpr::Value(Value::SingleQuotedString(ref s)) => Ok(lit(s.clone())), @@ -1372,7 +1510,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { fun: functions::BuiltinScalarFunction::DatePart, args: vec![ Expr::Literal(ScalarValue::Utf8(Some(format!("{}", field)))), - self.sql_expr_to_logical_expr(expr, schema)?, + self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?, ], }), @@ -1449,20 +1587,28 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { else_result, } => { let expr = if let Some(e) = operand { - Some(Box::new(self.sql_expr_to_logical_expr(e, schema)?)) + Some(Box::new(self.sql_expr_to_logical_expr( + e, + schema, + subquery_ctx, + )?)) } else { None }; let when_expr = conditions .iter() - .map(|e| self.sql_expr_to_logical_expr(e, schema)) + .map(|e| self.sql_expr_to_logical_expr(e, schema, subquery_ctx)) .collect::>>()?; let then_expr = results .iter() - .map(|e| self.sql_expr_to_logical_expr(e, schema)) + .map(|e| self.sql_expr_to_logical_expr(e, schema, subquery_ctx)) .collect::>>()?; let else_expr = if let Some(e) = else_result { - Some(Box::new(self.sql_expr_to_logical_expr(e, schema)?)) + Some(Box::new(self.sql_expr_to_logical_expr( + e, + schema, + subquery_ctx, + )?)) } else { None }; @@ -1482,7 +1628,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { ref expr, ref data_type, } => Ok(Expr::Cast { - expr: Box::new(self.sql_expr_to_logical_expr(expr, schema)?), + expr: Box::new(self.sql_expr_to_logical_expr( + expr, + schema, + subquery_ctx, + )?), data_type: convert_data_type(data_type)?, }), @@ -1490,7 +1640,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { ref expr, ref data_type, } => Ok(Expr::TryCast { - expr: Box::new(self.sql_expr_to_logical_expr(expr, schema)?), + expr: Box::new(self.sql_expr_to_logical_expr( + expr, + schema, + subquery_ctx, + )?), data_type: convert_data_type(data_type)?, }), @@ -1503,27 +1657,43 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { }), SQLExpr::IsNull(ref expr) => Ok(Expr::IsNull(Box::new( - self.sql_expr_to_logical_expr(expr, schema)?, + self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?, ))), SQLExpr::IsNotNull(ref expr) => Ok(Expr::IsNotNull(Box::new( - self.sql_expr_to_logical_expr(expr, schema)?, + self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?, ))), SQLExpr::IsDistinctFrom(left, right) => Ok(Expr::BinaryExpr { - left: Box::new(self.sql_expr_to_logical_expr(left, schema)?), + left: Box::new(self.sql_expr_to_logical_expr( + left, + schema, + subquery_ctx, + )?), op: Operator::IsDistinctFrom, - right: Box::new(self.sql_expr_to_logical_expr(right, schema)?), + right: Box::new(self.sql_expr_to_logical_expr( + right, + schema, + subquery_ctx, + )?), }), SQLExpr::IsNotDistinctFrom(left, right) => Ok(Expr::BinaryExpr { - left: Box::new(self.sql_expr_to_logical_expr(left, schema)?), + left: Box::new(self.sql_expr_to_logical_expr( + left, + schema, + subquery_ctx, + )?), op: Operator::IsNotDistinctFrom, - right: Box::new(self.sql_expr_to_logical_expr(right, schema)?), + right: Box::new(self.sql_expr_to_logical_expr( + right, + schema, + subquery_ctx, + )?), }), SQLExpr::UnaryOp { ref op, ref expr } => { - self.parse_sql_unary_op(op, expr, schema) + self.parse_sql_unary_op(op, expr, schema, subquery_ctx) } SQLExpr::Between { @@ -1532,10 +1702,22 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { ref low, ref high, } => Ok(Expr::Between { - expr: Box::new(self.sql_expr_to_logical_expr(expr, schema)?), + expr: Box::new(self.sql_expr_to_logical_expr( + expr, + schema, + subquery_ctx, + )?), negated: *negated, - low: Box::new(self.sql_expr_to_logical_expr(low, schema)?), - high: Box::new(self.sql_expr_to_logical_expr(high, schema)?), + low: Box::new(self.sql_expr_to_logical_expr( + low, + schema, + subquery_ctx, + )?), + high: Box::new(self.sql_expr_to_logical_expr( + high, + schema, + subquery_ctx, + )?), }), SQLExpr::InList { @@ -1545,11 +1727,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } => { let list_expr = list .iter() - .map(|e| self.sql_expr_to_logical_expr(e, schema)) + .map(|e| self.sql_expr_to_logical_expr(e, schema, subquery_ctx)) .collect::>>()?; Ok(Expr::InList { - expr: Box::new(self.sql_expr_to_logical_expr(expr, schema)?), + expr: Box::new(self.sql_expr_to_logical_expr( + expr, + schema, + subquery_ctx, + )?), list: list_expr, negated: *negated, }) @@ -1559,7 +1745,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { ref left, ref op, ref right, - } => self.parse_sql_binary_op(left, op, right, schema), + } => self.parse_sql_binary_op(left, op, right, schema, subquery_ctx), SQLExpr::Substring { expr, @@ -1568,24 +1754,37 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } => { #[cfg(feature = "unicode_expressions")] { - let arg = self.sql_expr_to_logical_expr(expr, schema)?; + let arg = + self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?; let args = match (substring_from, substring_for) { (Some(from_expr), Some(for_expr)) => { - let from_logic = - self.sql_expr_to_logical_expr(from_expr, schema)?; - let for_logic = - self.sql_expr_to_logical_expr(for_expr, schema)?; + let from_logic = self.sql_expr_to_logical_expr( + from_expr, + schema, + subquery_ctx, + )?; + let for_logic = self.sql_expr_to_logical_expr( + for_expr, + schema, + subquery_ctx, + )?; vec![arg, from_logic, for_logic] } (Some(from_expr), None) => { - let from_logic = - self.sql_expr_to_logical_expr(from_expr, schema)?; + let from_logic = self.sql_expr_to_logical_expr( + from_expr, + schema, + subquery_ctx, + )?; vec![arg, from_logic] } (None, Some(for_expr)) => { let from_logic = Expr::Literal(ScalarValue::Int64(Some(1))); - let for_logic = - self.sql_expr_to_logical_expr(for_expr, schema)?; + let for_logic = self.sql_expr_to_logical_expr( + for_expr, + schema, + subquery_ctx, + )?; vec![arg, from_logic, for_logic] } _ => { @@ -1622,10 +1821,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } None => (functions::BuiltinScalarFunction::Trim, None), }; - let arg = self.sql_expr_to_logical_expr(expr, schema)?; + let arg = self.sql_expr_to_logical_expr(expr, schema, subquery_ctx)?; let args = match where_expr { Some(to_trim) => { - let to_trim = self.sql_expr_to_logical_expr(to_trim, schema)?; + let to_trim = + self.sql_expr_to_logical_expr(to_trim, schema, subquery_ctx)?; vec![arg, to_trim] } None => vec![arg], @@ -1650,7 +1850,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { // first, scalar built-in if let Ok(fun) = functions::BuiltinScalarFunction::from_str(&name) { - let args = self.function_args_to_expr(function, schema)?; + let args = + self.function_args_to_expr(function, schema, subquery_ctx)?; return Ok(Expr::ScalarFunction { fun, args }); }; @@ -1660,12 +1861,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let partition_by = window .partition_by .iter() - .map(|e| self.sql_expr_to_logical_expr(e, schema)) + .map(|e| self.sql_expr_to_logical_expr(e, schema, subquery_ctx)) .collect::>>()?; let order_by = window .order_by .iter() - .map(|e| self.order_by_to_sort_expr(e, schema)) + .map(|e| self.order_by_to_sort_expr(e, schema, subquery_ctx)) .collect::>>()?; let window_frame = window .window_frame @@ -1695,6 +1896,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &aggregate_fun, function, schema, + subquery_ctx, )?, partition_by, order_by, @@ -1708,7 +1910,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { fun: window_functions::WindowFunction::BuiltInWindowFunction( window_fun, ), - args: self.function_args_to_expr(function, schema)?, + args: self.function_args_to_expr(function, schema, subquery_ctx)?, partition_by, order_by, window_frame, @@ -1719,7 +1921,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { // next, aggregate built-ins if let Ok(fun) = aggregates::AggregateFunction::from_str(&name) { - let args = self.aggregate_fn_to_expr(&fun, function, schema)?; + let args = + self.aggregate_fn_to_expr(&fun, function, schema, subquery_ctx)?; return Ok(Expr::AggregateFunction { fun, distinct: function.distinct, @@ -1730,13 +1933,18 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { // finally, user-defined functions (UDF) and UDAF match self.schema_provider.get_function_meta(&name) { Some(fm) => { - let args = self.function_args_to_expr(function, schema)?; + let args = + self.function_args_to_expr(function, schema, subquery_ctx)?; Ok(Expr::ScalarUDF { fun: fm, args }) } None => match self.schema_provider.get_aggregate_meta(&name) { Some(fm) => { - let args = self.function_args_to_expr(function, schema)?; + let args = self.function_args_to_expr( + function, + schema, + subquery_ctx, + )?; Ok(Expr::AggregateUDF { fun: fm, args }) } _ => Err(DataFusionError::Plan(format!( @@ -1747,7 +1955,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } } - SQLExpr::Nested(e) => self.sql_expr_to_logical_expr(e, schema), + SQLExpr::Nested(e) => self.sql_expr_to_logical_expr(e, schema, subquery_ctx), + + SQLExpr::Exists(q) => { + subquery_ctx.set_has_subquery(true); + let logical_plan = self.plan_subquery(q, subquery_ctx)?; + Ok(Expr::Exists(Box::from(logical_plan))) + } _ => Err(DataFusionError::NotImplemented(format!( "Unsupported ast node {:?} in sqltorel", @@ -1760,11 +1974,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, function: &sqlparser::ast::Function, schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, ) -> Result> { function .args .iter() - .map(|a| self.sql_fn_arg_to_logical_expr(a, schema)) + .map(|a| self.sql_fn_arg_to_logical_expr(a, schema, subquery_ctx)) .collect::>>() } @@ -1773,6 +1988,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { fun: &aggregates::AggregateFunction, function: &sqlparser::ast::Function, schema: &DFSchema, + subquery_ctx: &mut SubqueryCtx, ) -> Result> { if *fun == aggregates::AggregateFunction::Count { function @@ -1783,11 +1999,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { Ok(lit(1_u8)) } FunctionArg::Unnamed(SQLExpr::Wildcard) => Ok(lit(1_u8)), - _ => self.sql_fn_arg_to_logical_expr(a, schema), + _ => self.sql_fn_arg_to_logical_expr(a, schema, subquery_ctx), }) .collect::>>() } else { - self.function_args_to_expr(function, schema) + self.function_args_to_expr(function, schema, subquery_ctx) } } diff --git a/datafusion/src/sql/utils.rs b/datafusion/src/sql/utils.rs index 0ede5ad8559e5..0fcf790f6f198 100644 --- a/datafusion/src/sql/utils.rs +++ b/datafusion/src/sql/utils.rs @@ -374,6 +374,9 @@ where expr: Box::new(clone_with_replacement(expr.as_ref(), replacement_fn)?), key: key.clone(), }), + Expr::Exists(logical_plan) => { + Ok(Expr::Exists(Box::from(*logical_plan.clone()))) + } }, } } diff --git a/datafusion/tests/sql/mod.rs b/datafusion/tests/sql/mod.rs index 95623d45e467d..db13904e4d07b 100644 --- a/datafusion/tests/sql/mod.rs +++ b/datafusion/tests/sql/mod.rs @@ -91,6 +91,7 @@ pub mod predicates; pub mod projection; pub mod references; pub mod select; +pub mod subquery; pub mod timestamp; pub mod udf; pub mod union; diff --git a/datafusion/tests/sql/subquery.rs b/datafusion/tests/sql/subquery.rs new file mode 100644 index 0000000000000..6654d53e1b1de --- /dev/null +++ b/datafusion/tests/sql/subquery.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use datafusion::{error::Result, execution::context::ExecutionContext}; + +#[tokio::test] +async fn exist_subquery_basic() -> Result<()> { + let mut ctx = ExecutionContext::new(); + // create t1 table + let create_t1_sql = "CREATE TABLE t1 AS (SELECT 1 AS t1_id, 'a' AS t1_name UNION ALL SELECT 2 AS t1_id, 'b' AS t1_name UNION ALL SELECT 3 AS t1_id, 'c' AS t1_name)"; + ctx.sql(create_t1_sql).await.unwrap(); + + // create t2 table + let create_t2_sql = "CREATE TABLE t2 AS (SELECT 1 AS t2_id, 'a' AS t2_name, 4 AS t2_num UNION ALL SELECT 2 AS t2_id, 'p' AS t2_name, 5 AS t2_num UNION ALL SELECT 3 AS t2_id, 'q' AS t2_name, 6 AS t2_num)"; + ctx.sql(create_t2_sql).await.unwrap(); + + // exist subquery sql + let sql = "SELECT t2.t2_id FROM t2 WHERE EXISTS (SELECT t1.t1_id FROM t1 WHERE t1.t1_name = 'a')"; + + let actual = execute_to_batches(&mut ctx, sql).await; + let expected = vec![ + "+---------+", + "| t2_id |", + "+---------+", + "| 1 |", + "+---------+", + ]; + assert_batches_eq!(expected, &actual); + Ok(()) +}