diff --git a/datafusion-examples/examples/parquet_sql.rs b/datafusion-examples/examples/parquet_sql.rs index e74ed39c68ce2..e19cd1801620b 100644 --- a/datafusion-examples/examples/parquet_sql.rs +++ b/datafusion-examples/examples/parquet_sql.rs @@ -35,13 +35,7 @@ async fn main() -> Result<()> { .await?; // execute the query - let df = ctx - .sql( - "SELECT int_col, double_col, CAST(date_string_col as VARCHAR) \ - FROM alltypes_plain \ - WHERE id > 1 AND tinyint_col < double_col", - ) - .await?; + let df = ctx.sql("SELECT ((SELECT 2))").await?; // print the results df.show().await?; diff --git a/datafusion/src/datasource/listing/helpers.rs b/datafusion/src/datasource/listing/helpers.rs index 912179c36f060..b320f80283471 100644 --- a/datafusion/src/datasource/listing/helpers.rs +++ b/datafusion/src/datasource/listing/helpers.rs @@ -107,7 +107,8 @@ impl ExpressionVisitor for ApplicabilityVisitor<'_> { | Expr::AggregateFunction { .. } | Expr::Sort { .. } | Expr::WindowFunction { .. } - | Expr::Wildcard => { + | Expr::Wildcard + | Expr::Select { .. } => { *self.is_applicable = false; Recursion::Stop(self) } diff --git a/datafusion/src/logical_plan/dfschema.rs b/datafusion/src/logical_plan/dfschema.rs index 31143c4f616d1..5f7c9fd6accad 100644 --- a/datafusion/src/logical_plan/dfschema.rs +++ b/datafusion/src/logical_plan/dfschema.rs @@ -32,7 +32,7 @@ use std::fmt::{Display, Formatter}; pub type DFSchemaRef = Arc; /// DFSchema wraps an Arrow schema and adds relation names -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] pub struct DFSchema { /// Fields fields: Vec, @@ -402,7 +402,7 @@ impl Display for DFSchema { } /// DFField wraps an Arrow field and adds an optional qualifier -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] pub struct DFField { /// Optional qualifier (usually a table or relation name) qualifier: Option, diff --git a/datafusion/src/logical_plan/expr.rs b/datafusion/src/logical_plan/expr.rs index bcdfae7f4d8ec..b216fb12ca061 100644 --- a/datafusion/src/logical_plan/expr.rs +++ b/datafusion/src/logical_plan/expr.rs @@ -368,6 +368,19 @@ pub enum Expr { }, /// Represents a reference to all fields in a schema. Wildcard, + /// Scalar subquery: If the subquery returns no rows, the value is NULL; + /// if it returns more than one row, it is an error. + ScalarSubquery { + logical_plan: Box, + }, + /// Exist subquery + Exist { + logical_plan: Box, + }, + /// In subquery + InSubquery { + logical_plan: Box, + } } impl Expr { @@ -446,6 +459,17 @@ impl Expr { get_indexed_field(&data_type, key).map(|x| x.data_type().clone()) } + Expr::Select { logical_plan, .. } => { + let df_fields = logical_plan.schema().fields(); + let mut fields = Vec::with_capacity(df_fields.len()); + df_fields + .iter() + .map(|df_field| fields.push(df_field.field().clone())); + match fields.len() { + 1 => Ok(fields[0].data_type().clone()), + _ => Ok(DataType::Struct(fields)), + } + } } } @@ -505,6 +529,14 @@ impl Expr { let data_type = expr.get_type(input_schema)?; get_indexed_field(&data_type, key).map(|x| x.is_nullable()) } + Expr::Select { logical_plan, .. } => { + for df_field in logical_plan.schema().fields() { + if !df_field.is_nullable() { + return Ok(false); + } + } + Ok(true) + } } } @@ -684,7 +716,7 @@ impl Expr { /// pre_visit(Column("foo")) /// pre_visit(Column("bar")) /// post_visit(Column("bar")) - /// post_visit(Column("bar")) + /// post_visit(Column("foo")) /// post_visit(BinaryExpr(GT)) /// ``` /// @@ -784,6 +816,7 @@ impl Expr { } Expr::Wildcard => Ok(visitor), Expr::GetIndexedField { ref expr, .. } => expr.accept(visitor), + Expr::Select { .. } => Ok(visitor), }?; visitor.post_visit(self) @@ -948,6 +981,7 @@ impl Expr { expr: rewrite_boxed(expr, rewriter)?, key, }, + Expr::Select { logical_plan, name } => Expr::Select { name, logical_plan }, }; // now rewrite this expression itself @@ -1852,6 +1886,7 @@ impl fmt::Debug for Expr { Expr::GetIndexedField { ref expr, key } => { write!(f, "({:?})[{}]", expr, key) } + Expr::Select { name, .. } => write!(f, "subquery({})", name), } } } @@ -2013,6 +2048,9 @@ fn create_name(e: &Expr, input_schema: &DFSchema) -> Result { Expr::Wildcard => Err(DataFusionError::Internal( "Create name does not support wildcard".to_string(), )), + Expr::Select { logical_plan, .. } => { + Ok(format!("subquery {} in Select", logical_plan.display())) + } } } diff --git a/datafusion/src/logical_plan/plan.rs b/datafusion/src/logical_plan/plan.rs index 952572f4dea3e..8356d0b24c454 100644 --- a/datafusion/src/logical_plan/plan.rs +++ b/datafusion/src/logical_plan/plan.rs @@ -25,6 +25,7 @@ use crate::error::DataFusionError; use crate::logical_plan::dfschema::DFSchemaRef; use crate::sql::parser::FileType; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use std::cmp::Ordering; use std::{ collections::HashSet, fmt::{self, Display}, @@ -32,7 +33,7 @@ use std::{ }; /// Join type -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd)] pub enum JoinType { /// Inner Join Inner, @@ -49,7 +50,7 @@ pub enum JoinType { } /// Join constraint -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] pub enum JoinConstraint { /// Join ON On, @@ -59,7 +60,7 @@ pub enum JoinConstraint { /// Evaluates an arbitrary list of expressions (essentially a /// SELECT with an expression list) on its input. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Projection { /// The list of expressions pub expr: Vec, @@ -79,7 +80,7 @@ pub struct Projection { /// If the value of `` is true, the input row is passed to /// the output. If the value of `` is false, the row is /// discarded. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Filter { /// The predicate expression, which must have Boolean type. pub predicate: Expr, @@ -88,7 +89,7 @@ pub struct Filter { } /// Window its input based on a set of window spec and window function (e.g. SUM or RANK) -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Window { /// The incoming logical plan pub input: Arc, @@ -115,8 +116,20 @@ pub struct TableScan { pub limit: Option, } +impl PartialEq for TableScan { + fn eq(&self, other: &Self) -> bool { + todo!() + } +} + +impl PartialOrd for TableScan { + fn partial_cmp(&self, other: &Self) -> Option { + todo!() + } +} + /// Apply Cross Join to two logical plans -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct CrossJoin { /// Left input pub left: Arc, @@ -127,7 +140,7 @@ pub struct CrossJoin { } /// Repartition the plan based on a partitioning scheme. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Repartition { /// The incoming logical plan pub input: Arc, @@ -136,7 +149,7 @@ pub struct Repartition { } /// Union multiple inputs -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Union { /// Inputs to merge pub inputs: Vec, @@ -147,7 +160,7 @@ pub struct Union { } /// Creates an in memory table. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct CreateMemoryTable { /// The table name pub name: String, @@ -156,7 +169,7 @@ pub struct CreateMemoryTable { } /// Creates an external table. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct CreateExternalTable { /// The table schema pub schema: DFSchemaRef, @@ -171,7 +184,7 @@ pub struct CreateExternalTable { } /// Drops a table. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct DropTable { /// The table name pub name: String, @@ -183,7 +196,7 @@ pub struct DropTable { /// Produces a relation with string representations of /// various parts of the plan -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Explain { /// Should extra (detailed, intermediate plans) be included? pub verbose: bool, @@ -197,7 +210,7 @@ pub struct Explain { /// Runs the actual plan, and then prints the physical plan with /// with execution metrics. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Analyze { /// Should extra detail be included? pub verbose: bool, @@ -214,8 +227,19 @@ pub struct Extension { pub node: Arc, } +impl PartialEq for Extension { + fn eq(&self, other: &Self) -> bool { + todo!() + } +} + +impl PartialOrd for Extension { + fn partial_cmp(&self, other: &Self) -> Option { + todo!() + } +} /// Produces no rows: An empty relation with an empty schema -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct EmptyRelation { /// Whether to produce a placeholder row pub produce_one_row: bool, @@ -224,7 +248,7 @@ pub struct EmptyRelation { } /// Produces the first `n` tuples from its input and discards the rest. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Limit { /// The limit pub n: usize, @@ -235,7 +259,7 @@ pub struct Limit { /// Values expression. See /// [Postgres VALUES](https://www.postgresql.org/docs/current/queries-values.html) /// documentation for more details. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Values { /// The table schema pub schema: DFSchemaRef, @@ -245,7 +269,7 @@ pub struct Values { /// Aggregates its input based on a set of grouping and aggregate /// expressions (e.g. SUM). -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Aggregate { /// The incoming logical plan pub input: Arc, @@ -258,7 +282,7 @@ pub struct Aggregate { } /// Sorts its input according to a list of sort expressions. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Sort { /// The sort expressions pub expr: Vec, @@ -267,7 +291,7 @@ pub struct Sort { } /// Join two logical plans on one or more join columns -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub struct Join { /// Left input pub left: Arc, @@ -292,7 +316,7 @@ pub struct Join { /// an output relation (table) with a (potentially) different /// schema. A plan represents a dataflow tree where data flows /// from leaves up to the root to produce the query result. -#[derive(Clone)] +#[derive(Clone, PartialEq, PartialOrd)] pub enum LogicalPlan { /// Evaluates an arbitrary list of expressions (essentially a /// SELECT with an expression list) on its input. @@ -547,7 +571,7 @@ impl LogicalPlan { } /// Logical partitioning schemes supported by the repartition operator. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, PartialOrd)] pub enum Partitioning { /// Allocate batches using a round-robin algorithm and the specified number of partitions RoundRobinBatch(usize), @@ -1004,7 +1028,7 @@ impl fmt::Debug for LogicalPlan { /// Represents which type of plan, when storing multiple /// for use in EXPLAIN plans -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, PartialOrd)] pub enum PlanType { /// The initial LogicalPlan provided to DataFusion InitialLogicalPlan, @@ -1044,7 +1068,7 @@ impl fmt::Display for PlanType { } /// Represents some sort of execution plan, in String form -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, PartialOrd)] #[allow(clippy::rc_buffer)] pub struct StringifiedPlan { /// An identifier of what type of plan this string represents diff --git a/datafusion/src/optimizer/common_subexpr_eliminate.rs b/datafusion/src/optimizer/common_subexpr_eliminate.rs index 233d112197feb..b40a2d64ccd2c 100644 --- a/datafusion/src/optimizer/common_subexpr_eliminate.rs +++ b/datafusion/src/optimizer/common_subexpr_eliminate.rs @@ -451,6 +451,10 @@ impl ExprIdentifierVisitor<'_> { desc.push_str("GetIndexedField-"); desc.push_str(&key.to_string()); } + Expr::Select { name, .. } => { + desc.push_str("Select-"); + desc.push_str(name); + } } desc diff --git a/datafusion/src/optimizer/simplify_expressions.rs b/datafusion/src/optimizer/simplify_expressions.rs index 0ca9212cf6571..0ee039620a91a 100644 --- a/datafusion/src/optimizer/simplify_expressions.rs +++ b/datafusion/src/optimizer/simplify_expressions.rs @@ -480,6 +480,7 @@ impl ConstEvaluator { Expr::WindowFunction { .. } => false, Expr::Sort { .. } => false, Expr::Wildcard => false, + Expr::Select { .. } => false, Expr::Literal(_) => true, Expr::BinaryExpr { .. } => true, diff --git a/datafusion/src/optimizer/utils.rs b/datafusion/src/optimizer/utils.rs index 541ac67a6117e..c53f602101fac 100644 --- a/datafusion/src/optimizer/utils.rs +++ b/datafusion/src/optimizer/utils.rs @@ -83,6 +83,7 @@ impl ExpressionVisitor for ColumnNameVisitor<'_> { Expr::InList { .. } => {} Expr::Wildcard => {} Expr::GetIndexedField { .. } => {} + Expr::Select { .. } => {} } Ok(Recursion::Continue(self)) } @@ -349,6 +350,7 @@ pub fn expr_sub_expressions(expr: &Expr) -> Result> { "Wildcard expressions are not valid in a logical query plan".to_owned(), )), Expr::GetIndexedField { expr, .. } => Ok(vec![expr.as_ref().to_owned()]), + Expr::Select { .. } => Ok(vec![]), } } @@ -473,9 +475,10 @@ pub fn rewrite_expression(expr: &Expr, expressions: &[Expr]) -> Result { } Expr::Not(_) => Ok(Expr::Not(Box::new(expressions[0].clone()))), Expr::Negative(_) => Ok(Expr::Negative(Box::new(expressions[0].clone()))), - Expr::Column(_) => Ok(expr.clone()), - Expr::Literal(_) => Ok(expr.clone()), - Expr::ScalarVariable(_) => Ok(expr.clone()), + Expr::Column(_) + | Expr::Literal(_) + | Expr::ScalarVariable(_) + | Expr::Select { .. } => Ok(expr.clone()), Expr::Sort { asc, nulls_first, .. } => Ok(Expr::Sort { diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/src/physical_plan/planner.rs index 1302369886f86..74ee8d5d80aef 100644 --- a/datafusion/src/physical_plan/planner.rs +++ b/datafusion/src/physical_plan/planner.rs @@ -207,6 +207,9 @@ fn create_physical_name(e: &Expr, is_first_expr: bool) -> Result { Expr::Wildcard => Err(DataFusionError::Internal( "Create physical name does not support wildcard".to_string(), )), + Expr::Select { logical_plan, .. } => { + Ok(format!("Subquery {}", logical_plan.display())) + } } } diff --git a/datafusion/src/sql/parser.rs b/datafusion/src/sql/parser.rs index 3352577644429..413859d4fce21 100644 --- a/datafusion/src/sql/parser.rs +++ b/datafusion/src/sql/parser.rs @@ -35,7 +35,7 @@ macro_rules! parser_err { } /// Types of files to parse as DataFrames -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] pub enum FileType { /// Newline-delimited JSON NdJson, diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index bbd5aa7c5696b..78535f8a62b78 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -121,10 +121,18 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { pub fn statement_to_plan(&self, statement: &DFStatement) -> Result { match statement { DFStatement::CreateExternalTable(s) => self.external_table_to_plan(s), - DFStatement::Statement(s) => self.sql_statement_to_plan(s), + DFStatement::Statement(s) => { + let mut plan = self.sql_statement_to_plan(s).unwrap(); + self.rewrite_plan(&mut plan) + }, } } + /// Rewrite the logical plan generated by `statement_to_plan`, include processing subquery + fn rewrite_plan(&self, plan: &mut LogicalPlan) -> Result { + // + } + /// Generate a logical plan from an SQL statement pub fn sql_statement_to_plan(&self, sql: &Statement) -> Result { match sql { @@ -232,6 +240,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { self.limit(plan, &query.limit) } + fn plan_subquery(&self, query: &Query) -> Result { + self.query_to_plan(query) + } + fn set_expr_to_plan( &self, set_expr: &SetExpr, @@ -271,6 +283,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } } } + SetExpr::Query(q) => { + let plan = self.plan_subquery(q)?; + Ok(plan) + } _ => Err(DataFusionError::NotImplemented(format!( "Query {} not implemented yet", set_expr @@ -1628,6 +1644,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { SQLExpr::Nested(e) => self.sql_expr_to_logical_expr(e, schema), + SQLExpr::Subquery(q) => { + let logical_plan = self.plan_subquery(q)?; + // TODO: check if only one column + Ok(Expr::Select { + name: "".to_string(), + logical_plan: Box::new(logical_plan), + }) + } + _ => Err(DataFusionError::NotImplemented(format!( "Unsupported ast node {:?} in sqltorel", sql diff --git a/datafusion/src/sql/utils.rs b/datafusion/src/sql/utils.rs index 0ede5ad8559e5..2950320ad2391 100644 --- a/datafusion/src/sql/utils.rs +++ b/datafusion/src/sql/utils.rs @@ -374,6 +374,10 @@ where expr: Box::new(clone_with_replacement(expr.as_ref(), replacement_fn)?), key: key.clone(), }), + Expr::Select { logical_plan, name } => Ok(Expr::Select { + name: name.clone(), + logical_plan: logical_plan.clone(), + }), }, } }