Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions datafusion-examples/examples/parquet_sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,7 @@ async fn main() -> Result<()> {
.await?;

// execute the query
let df = ctx
.sql(
"SELECT int_col, double_col, CAST(date_string_col as VARCHAR) \
FROM alltypes_plain \
WHERE id > 1 AND tinyint_col < double_col",
)
.await?;
let df = ctx.sql("SELECT ((SELECT 2))").await?;

// print the results
df.show().await?;
Expand Down
3 changes: 2 additions & 1 deletion datafusion/src/datasource/listing/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ impl ExpressionVisitor for ApplicabilityVisitor<'_> {
| Expr::AggregateFunction { .. }
| Expr::Sort { .. }
| Expr::WindowFunction { .. }
| Expr::Wildcard => {
| Expr::Wildcard
| Expr::Select { .. } => {
*self.is_applicable = false;
Recursion::Stop(self)
}
Expand Down
4 changes: 2 additions & 2 deletions datafusion/src/logical_plan/dfschema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ use std::fmt::{Display, Formatter};
pub type DFSchemaRef = Arc<DFSchema>;

/// DFSchema wraps an Arrow schema and adds relation names
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)]
pub struct DFSchema {
/// Fields
fields: Vec<DFField>,
Expand Down Expand Up @@ -402,7 +402,7 @@ impl Display for DFSchema {
}

/// DFField wraps an Arrow field and adds an optional qualifier
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)]
pub struct DFField {
/// Optional qualifier (usually a table or relation name)
qualifier: Option<String>,
Expand Down
40 changes: 39 additions & 1 deletion datafusion/src/logical_plan/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,19 @@ pub enum Expr {
},
/// Represents a reference to all fields in a schema.
Wildcard,
/// Scalar subquery: If the subquery returns no rows, the value is NULL;
/// if it returns more than one row, it is an error.
ScalarSubquery {
logical_plan: Box<LogicalPlan>,
},
/// Exist subquery
Exist {
logical_plan: Box<LogicalPlan>,
},
/// In subquery
InSubquery {
logical_plan: Box<LogicalPlan>,
}
}

impl Expr {
Expand Down Expand Up @@ -446,6 +459,17 @@ impl Expr {

get_indexed_field(&data_type, key).map(|x| x.data_type().clone())
}
Expr::Select { logical_plan, .. } => {
let df_fields = logical_plan.schema().fields();
let mut fields = Vec::with_capacity(df_fields.len());
df_fields
.iter()
.map(|df_field| fields.push(df_field.field().clone()));
match fields.len() {
1 => Ok(fields[0].data_type().clone()),
_ => Ok(DataType::Struct(fields)),
}
}
}
}

Expand Down Expand Up @@ -505,6 +529,14 @@ impl Expr {
let data_type = expr.get_type(input_schema)?;
get_indexed_field(&data_type, key).map(|x| x.is_nullable())
}
Expr::Select { logical_plan, .. } => {
for df_field in logical_plan.schema().fields() {
if !df_field.is_nullable() {
return Ok(false);
}
}
Ok(true)
}
}
}

Expand Down Expand Up @@ -684,7 +716,7 @@ impl Expr {
/// pre_visit(Column("foo"))
/// pre_visit(Column("bar"))
/// post_visit(Column("bar"))
/// post_visit(Column("bar"))
/// post_visit(Column("foo"))
/// post_visit(BinaryExpr(GT))
/// ```
///
Expand Down Expand Up @@ -784,6 +816,7 @@ impl Expr {
}
Expr::Wildcard => Ok(visitor),
Expr::GetIndexedField { ref expr, .. } => expr.accept(visitor),
Expr::Select { .. } => Ok(visitor),
}?;

visitor.post_visit(self)
Expand Down Expand Up @@ -948,6 +981,7 @@ impl Expr {
expr: rewrite_boxed(expr, rewriter)?,
key,
},
Expr::Select { logical_plan, name } => Expr::Select { name, logical_plan },
};

// now rewrite this expression itself
Expand Down Expand Up @@ -1852,6 +1886,7 @@ impl fmt::Debug for Expr {
Expr::GetIndexedField { ref expr, key } => {
write!(f, "({:?})[{}]", expr, key)
}
Expr::Select { name, .. } => write!(f, "subquery({})", name),
}
}
}
Expand Down Expand Up @@ -2013,6 +2048,9 @@ fn create_name(e: &Expr, input_schema: &DFSchema) -> Result<String> {
Expr::Wildcard => Err(DataFusionError::Internal(
"Create name does not support wildcard".to_string(),
)),
Expr::Select { logical_plan, .. } => {
Ok(format!("subquery {} in Select", logical_plan.display()))
}
}
}

Expand Down
70 changes: 47 additions & 23 deletions datafusion/src/logical_plan/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,15 @@ use crate::error::DataFusionError;
use crate::logical_plan::dfschema::DFSchemaRef;
use crate::sql::parser::FileType;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use std::cmp::Ordering;
use std::{
collections::HashSet,
fmt::{self, Display},
sync::Arc,
};

/// Join type
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd)]
pub enum JoinType {
/// Inner Join
Inner,
Expand All @@ -49,7 +50,7 @@ pub enum JoinType {
}

/// Join constraint
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)]
pub enum JoinConstraint {
/// Join ON
On,
Expand All @@ -59,7 +60,7 @@ pub enum JoinConstraint {

/// Evaluates an arbitrary list of expressions (essentially a
/// SELECT with an expression list) on its input.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Projection {
/// The list of expressions
pub expr: Vec<Expr>,
Expand All @@ -79,7 +80,7 @@ pub struct Projection {
/// If the value of `<predicate>` is true, the input row is passed to
/// the output. If the value of `<predicate>` is false, the row is
/// discarded.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Filter {
/// The predicate expression, which must have Boolean type.
pub predicate: Expr,
Expand All @@ -88,7 +89,7 @@ pub struct Filter {
}

/// Window its input based on a set of window spec and window function (e.g. SUM or RANK)
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Window {
/// The incoming logical plan
pub input: Arc<LogicalPlan>,
Expand All @@ -115,8 +116,20 @@ pub struct TableScan {
pub limit: Option<usize>,
}

impl PartialEq for TableScan {
fn eq(&self, other: &Self) -> bool {
todo!()
}
}

impl PartialOrd for TableScan {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
todo!()
}
}

/// Apply Cross Join to two logical plans
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct CrossJoin {
/// Left input
pub left: Arc<LogicalPlan>,
Expand All @@ -127,7 +140,7 @@ pub struct CrossJoin {
}

/// Repartition the plan based on a partitioning scheme.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Repartition {
/// The incoming logical plan
pub input: Arc<LogicalPlan>,
Expand All @@ -136,7 +149,7 @@ pub struct Repartition {
}

/// Union multiple inputs
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Union {
/// Inputs to merge
pub inputs: Vec<LogicalPlan>,
Expand All @@ -147,7 +160,7 @@ pub struct Union {
}

/// Creates an in memory table.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct CreateMemoryTable {
/// The table name
pub name: String,
Expand All @@ -156,7 +169,7 @@ pub struct CreateMemoryTable {
}

/// Creates an external table.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct CreateExternalTable {
/// The table schema
pub schema: DFSchemaRef,
Expand All @@ -171,7 +184,7 @@ pub struct CreateExternalTable {
}

/// Drops a table.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct DropTable {
/// The table name
pub name: String,
Expand All @@ -183,7 +196,7 @@ pub struct DropTable {

/// Produces a relation with string representations of
/// various parts of the plan
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Explain {
/// Should extra (detailed, intermediate plans) be included?
pub verbose: bool,
Expand All @@ -197,7 +210,7 @@ pub struct Explain {

/// Runs the actual plan, and then prints the physical plan with
/// with execution metrics.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Analyze {
/// Should extra detail be included?
pub verbose: bool,
Expand All @@ -214,8 +227,19 @@ pub struct Extension {
pub node: Arc<dyn UserDefinedLogicalNode + Send + Sync>,
}

impl PartialEq for Extension {
fn eq(&self, other: &Self) -> bool {
todo!()
}
}

impl PartialOrd for Extension {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
todo!()
}
}
/// Produces no rows: An empty relation with an empty schema
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct EmptyRelation {
/// Whether to produce a placeholder row
pub produce_one_row: bool,
Expand All @@ -224,7 +248,7 @@ pub struct EmptyRelation {
}

/// Produces the first `n` tuples from its input and discards the rest.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Limit {
/// The limit
pub n: usize,
Expand All @@ -235,7 +259,7 @@ pub struct Limit {
/// Values expression. See
/// [Postgres VALUES](https://www.postgresql.org/docs/current/queries-values.html)
/// documentation for more details.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Values {
/// The table schema
pub schema: DFSchemaRef,
Expand All @@ -245,7 +269,7 @@ pub struct Values {

/// Aggregates its input based on a set of grouping and aggregate
/// expressions (e.g. SUM).
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Aggregate {
/// The incoming logical plan
pub input: Arc<LogicalPlan>,
Expand All @@ -258,7 +282,7 @@ pub struct Aggregate {
}

/// Sorts its input according to a list of sort expressions.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Sort {
/// The sort expressions
pub expr: Vec<Expr>,
Expand All @@ -267,7 +291,7 @@ pub struct Sort {
}

/// Join two logical plans on one or more join columns
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub struct Join {
/// Left input
pub left: Arc<LogicalPlan>,
Expand All @@ -292,7 +316,7 @@ pub struct Join {
/// an output relation (table) with a (potentially) different
/// schema. A plan represents a dataflow tree where data flows
/// from leaves up to the root to produce the query result.
#[derive(Clone)]
#[derive(Clone, PartialEq, PartialOrd)]
pub enum LogicalPlan {
/// Evaluates an arbitrary list of expressions (essentially a
/// SELECT with an expression list) on its input.
Expand Down Expand Up @@ -547,7 +571,7 @@ impl LogicalPlan {
}

/// Logical partitioning schemes supported by the repartition operator.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq, PartialOrd)]
pub enum Partitioning {
/// Allocate batches using a round-robin algorithm and the specified number of partitions
RoundRobinBatch(usize),
Expand Down Expand Up @@ -1004,7 +1028,7 @@ impl fmt::Debug for LogicalPlan {

/// Represents which type of plan, when storing multiple
/// for use in EXPLAIN plans
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, PartialOrd)]
pub enum PlanType {
/// The initial LogicalPlan provided to DataFusion
InitialLogicalPlan,
Expand Down Expand Up @@ -1044,7 +1068,7 @@ impl fmt::Display for PlanType {
}

/// Represents some sort of execution plan, in String form
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, PartialOrd)]
#[allow(clippy::rc_buffer)]
pub struct StringifiedPlan {
/// An identifier of what type of plan this string represents
Expand Down
4 changes: 4 additions & 0 deletions datafusion/src/optimizer/common_subexpr_eliminate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,10 @@ impl ExprIdentifierVisitor<'_> {
desc.push_str("GetIndexedField-");
desc.push_str(&key.to_string());
}
Expr::Select { name, .. } => {
desc.push_str("Select-");
desc.push_str(name);
}
}

desc
Expand Down
1 change: 1 addition & 0 deletions datafusion/src/optimizer/simplify_expressions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,7 @@ impl ConstEvaluator {
Expr::WindowFunction { .. } => false,
Expr::Sort { .. } => false,
Expr::Wildcard => false,
Expr::Select { .. } => false,

Expr::Literal(_) => true,
Expr::BinaryExpr { .. } => true,
Expand Down
Loading