From 99d7ff474c2e09672e2ad8de8024df83ebb9d4a9 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 20 Jul 2021 09:22:19 -0400 Subject: [PATCH 1/6] Remove old optimize_for_explain plumbing --- datafusion/src/execution/context.rs | 64 ++++++++-- datafusion/src/optimizer/limit_push_down.rs | 20 ---- .../src/optimizer/projection_push_down.rs | 22 +--- .../src/optimizer/simplify_expressions.rs | 23 +--- datafusion/src/optimizer/utils.rs | 109 +----------------- datafusion/tests/sql.rs | 5 + 6 files changed, 65 insertions(+), 178 deletions(-) diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs index bd939cef7035b..372bfc94646d0 100644 --- a/datafusion/src/execution/context.rs +++ b/datafusion/src/execution/context.rs @@ -21,6 +21,7 @@ use crate::{ catalog::{CatalogList, MemoryCatalogList}, information_schema::CatalogWithInformationSchema, }, + logical_plan::{PlanType, StringifiedPlan}, optimizer::{ aggregate_statistics::AggregateStatistics, eliminate_limit::EliminateLimit, hash_build_probe_order::HashBuildProbeOrder, @@ -446,19 +447,32 @@ impl ExecutionContext { /// Optimizes the logical plan by applying optimizer rules. pub fn optimize(&self, plan: &LogicalPlan) -> Result { - let state = &mut self.state.lock().unwrap(); - let execution_props = &mut state.execution_props.clone(); - let optimizers = &state.config.optimizers; - - let execution_props = execution_props.start_execution(); - - let mut new_plan = plan.clone(); - debug!("Logical plan:\n {:?}", plan); - for optimizer in optimizers { - new_plan = optimizer.optimize(&new_plan, execution_props)?; + if let LogicalPlan::Explain { + verbose, + plan, + stringified_plans, + schema, + } = plan + { + let mut stringified_plans = stringified_plans.clone(); + + // optimize the child plan, capturing the output of each optimizer + let plan = self.optimize_internal(plan, |optimized_plan, optimizer| { + let optimizer_name = optimizer.name().to_string(); + let plan_type = PlanType::OptimizedLogicalPlan { optimizer_name }; + let plan_string = format!("{:#?}", optimized_plan); + stringified_plans.push(StringifiedPlan::new(plan_type, plan_string)); + })?; + + Ok(LogicalPlan::Explain { + verbose: *verbose, + plan: Arc::new(plan), + stringified_plans, + schema: schema.clone(), + }) + } else { + self.optimize_internal(plan, |_, _| {}) } - debug!("Optimized logical plan:\n {:?}", new_plan); - Ok(new_plan) } /// Creates a physical plan from a logical plan. @@ -556,6 +570,32 @@ impl ExecutionContext { ))), } } + + /// Optimizes the logical plan by applying optimizer rules, and + /// invoking observer function after each call + fn optimize_internal( + &self, + plan: &LogicalPlan, + mut observer: F, + ) -> Result + where + F: FnMut(&LogicalPlan, &dyn OptimizerRule), + { + let state = &mut self.state.lock().unwrap(); + let execution_props = &mut state.execution_props.clone(); + let optimizers = &state.config.optimizers; + + let execution_props = execution_props.start_execution(); + + let mut new_plan = plan.clone(); + debug!("Logical plan:\n {:?}", plan); + for optimizer in optimizers { + new_plan = optimizer.optimize(&new_plan, execution_props)?; + observer(&new_plan, optimizer.as_ref()); + } + debug!("Optimized logical plan:\n {:?}", new_plan); + Ok(new_plan) + } } impl From>> for ExecutionContext { diff --git a/datafusion/src/optimizer/limit_push_down.rs b/datafusion/src/optimizer/limit_push_down.rs index 37c95a4436922..46738c557c631 100644 --- a/datafusion/src/optimizer/limit_push_down.rs +++ b/datafusion/src/optimizer/limit_push_down.rs @@ -23,7 +23,6 @@ use crate::execution::context::ExecutionProps; use crate::logical_plan::LogicalPlan; use crate::optimizer::optimizer::OptimizerRule; use std::sync::Arc; -use utils::optimize_explain; /// Optimization rule that tries pushes down LIMIT n /// where applicable to reduce the amount of scanned / processed data @@ -43,25 +42,6 @@ fn limit_push_down( execution_props: &ExecutionProps, ) -> Result { match (plan, upper_limit) { - ( - LogicalPlan::Explain { - verbose, - schema, - plan, - stringified_plans, - }, - _, - ) => { - let schema = schema.as_ref().to_owned().into(); - optimize_explain( - optimizer, - *verbose, - plan, - stringified_plans, - &schema, - execution_props, - ) - } (LogicalPlan::Limit { n, input }, upper_limit) => { let smallest = upper_limit.map(|x| std::cmp::min(x, *n)).unwrap_or(*n); Ok(LogicalPlan::Limit { diff --git a/datafusion/src/optimizer/projection_push_down.rs b/datafusion/src/optimizer/projection_push_down.rs index 089dca2318c98..0de36f354206f 100644 --- a/datafusion/src/optimizer/projection_push_down.rs +++ b/datafusion/src/optimizer/projection_push_down.rs @@ -18,7 +18,7 @@ //! Projection Push Down optimizer rule ensures that only referenced columns are //! loaded into memory -use crate::error::Result; +use crate::error::{DataFusionError, Result}; use crate::execution::context::ExecutionProps; use crate::logical_plan::{ build_join_schema, Column, DFField, DFSchema, DFSchemaRef, LogicalPlan, @@ -33,7 +33,6 @@ use std::{ collections::{BTreeSet, HashSet}, sync::Arc, }; -use utils::optimize_explain; /// Optimizer that removes unused projections and aggregations from plans /// This reduces both scans and @@ -354,22 +353,9 @@ fn optimize_plan( limit: *limit, }) } - LogicalPlan::Explain { - verbose, - plan, - stringified_plans, - schema, - } => { - let schema = schema.as_ref().to_owned().into(); - optimize_explain( - optimizer, - *verbose, - &*plan, - stringified_plans, - &schema, - execution_props, - ) - } + LogicalPlan::Explain { .. } => Err(DataFusionError::Internal( + "Unsupported logical plan: Explain must be root of the plan".to_string(), + )), LogicalPlan::Union { inputs, schema, diff --git a/datafusion/src/optimizer/simplify_expressions.rs b/datafusion/src/optimizer/simplify_expressions.rs index 0e65de07305ff..f629eaf95b5f4 100644 --- a/datafusion/src/optimizer/simplify_expressions.rs +++ b/datafusion/src/optimizer/simplify_expressions.rs @@ -22,7 +22,6 @@ use crate::logical_plan::LogicalPlan; use crate::logical_plan::{lit, Expr}; use crate::optimizer::optimizer::OptimizerRule; use crate::optimizer::utils; -use crate::optimizer::utils::optimize_explain; use crate::scalar::ScalarValue; use crate::{error::Result, logical_plan::Operator}; @@ -278,27 +277,9 @@ impl OptimizerRule for SimplifyExpressions { fn optimize( &self, plan: &LogicalPlan, - execution_props: &ExecutionProps, + _execution_props: &ExecutionProps, ) -> Result { - match plan { - LogicalPlan::Explain { - verbose, - plan, - stringified_plans, - schema, - } => { - let schema = schema.as_ref().to_owned().into(); - optimize_explain( - self, - *verbose, - &*plan, - stringified_plans, - &schema, - execution_props, - ) - } - _ => optimize(plan), - } + optimize(plan) } } diff --git a/datafusion/src/optimizer/utils.rs b/datafusion/src/optimizer/utils.rs index 88380ea17c875..2530a1efd1338 100644 --- a/datafusion/src/optimizer/utils.rs +++ b/datafusion/src/optimizer/utils.rs @@ -17,15 +17,11 @@ //! Collection of utility functions that are leveraged by the query optimizer rules -use std::{collections::HashSet, sync::Arc}; - -use arrow::datatypes::Schema; - use super::optimizer::OptimizerRule; use crate::execution::context::ExecutionProps; use crate::logical_plan::{ build_join_schema, Column, DFSchemaRef, Expr, LogicalPlan, LogicalPlanBuilder, - Operator, Partitioning, PlanType, Recursion, StringifiedPlan, ToDFSchema, + Operator, Partitioning, Recursion, }; use crate::prelude::lit; use crate::scalar::ScalarValue; @@ -33,6 +29,7 @@ use crate::{ error::{DataFusionError, Result}, logical_plan::ExpressionVisitor, }; +use std::{collections::HashSet, sync::Arc}; const CASE_EXPR_MARKER: &str = "__DATAFUSION_CASE_EXPR__"; const CASE_ELSE_MARKER: &str = "__DATAFUSION_CASE_ELSE__"; @@ -94,34 +91,6 @@ pub fn expr_to_columns(expr: &Expr, accum: &mut HashSet) -> Result<()> { Ok(()) } -/// Create a `LogicalPlan::Explain` node by running `optimizer` on the -/// input plan and capturing the resulting plan string -pub fn optimize_explain( - optimizer: &impl OptimizerRule, - verbose: bool, - plan: &LogicalPlan, - stringified_plans: &[StringifiedPlan], - schema: &Schema, - execution_props: &ExecutionProps, -) -> Result { - // These are the fields of LogicalPlan::Explain It might be nice - // to transform that enum Variant into its own struct and avoid - // passing the fields individually - let plan = Arc::new(optimizer.optimize(plan, execution_props)?); - let mut stringified_plans = stringified_plans.to_vec(); - let optimizer_name = optimizer.name().into(); - stringified_plans.push(StringifiedPlan::new( - PlanType::OptimizedLogicalPlan { optimizer_name }, - format!("{:#?}", plan), - )); - Ok(LogicalPlan::Explain { - verbose, - plan, - stringified_plans, - schema: schema.clone().to_dfschema_ref()?, - }) -} - /// Convenience rule for writing optimizers: recursively invoke /// optimize on plan's children and then return a node of the same /// type. Useful for optimizer rules which want to leave the type @@ -132,23 +101,6 @@ pub fn optimize_children( plan: &LogicalPlan, execution_props: &ExecutionProps, ) -> Result { - if let LogicalPlan::Explain { - verbose, - plan, - stringified_plans, - schema, - } = plan - { - return optimize_explain( - optimizer, - *verbose, - &*plan, - stringified_plans, - &schema.as_ref().to_owned().into(), - execution_props, - ); - } - let new_exprs = plan.expressions(); let new_inputs = plan .inputs() @@ -514,61 +466,4 @@ mod tests { assert!(accum.contains(&Column::from_name("a"))); Ok(()) } - - struct TestOptimizer {} - - impl OptimizerRule for TestOptimizer { - fn optimize( - &self, - plan: &LogicalPlan, - _: &ExecutionProps, - ) -> Result { - Ok(plan.clone()) - } - - fn name(&self) -> &str { - "test_optimizer" - } - } - - #[test] - fn test_optimize_explain() -> Result<()> { - let optimizer = TestOptimizer {}; - - let empty_plan = LogicalPlanBuilder::empty(false).build()?; - let schema = LogicalPlan::explain_schema(); - - let optimized_explain = optimize_explain( - &optimizer, - true, - &empty_plan, - &[StringifiedPlan::new(PlanType::InitialLogicalPlan, "...")], - schema.as_ref(), - &ExecutionProps::new(), - )?; - - match &optimized_explain { - LogicalPlan::Explain { - verbose, - stringified_plans, - .. - } => { - assert!(*verbose); - - let expected_stringified_plans = vec![ - StringifiedPlan::new(PlanType::InitialLogicalPlan, "..."), - StringifiedPlan::new( - PlanType::OptimizedLogicalPlan { - optimizer_name: "test_optimizer".into(), - }, - "EmptyRelation", - ), - ]; - assert_eq!(*stringified_plans, expected_stringified_plans); - } - _ => panic!("Expected explain plan but got {:?}", optimized_explain), - } - - Ok(()) - } } diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 95b5596eb9f17..fb27199f66235 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -2507,6 +2507,11 @@ async fn execute_to_batches(ctx: &mut ExecutionContext, sql: &str) -> Vec Date: Tue, 20 Jul 2021 09:35:35 -0400 Subject: [PATCH 2/6] Show plans that are no different than the previous as "SAME" --- datafusion/src/physical_plan/explain.rs | 13 ++++++++++++- datafusion/tests/sql.rs | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/datafusion/src/physical_plan/explain.rs b/datafusion/src/physical_plan/explain.rs index 195a7a518370a..4fd3711d5d2e3 100644 --- a/datafusion/src/physical_plan/explain.rs +++ b/datafusion/src/physical_plan/explain.rs @@ -115,9 +115,20 @@ impl ExecutionPlan for ExplainExec { .iter() .filter(|s| s.should_display(self.verbose)); + // Identify plans that are not changed + let mut prev: Option<&StringifiedPlan> = None; + for p in plans_to_print { type_builder.append_value(p.plan_type.to_string())?; - plan_builder.append_value(&*p.plan)?; + match prev { + Some(prev) if p.plan == prev.plan => { + plan_builder.append_value("SAME")?; + } + Some(_) | None => { + plan_builder.append_value(&*p.plan)?; + } + } + prev = Some(p); } let record_batch = RecordBatch::try_new( diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index fb27199f66235..2c8c0dd6be332 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -2205,6 +2205,8 @@ async fn csv_explain_verbose() { "Actual: '{}'", actual ); + + assert!(false, "TODO: add test for NO CHANGE"); } #[tokio::test] From 266d1fae21fdae2ccc98f47b0b85b28eb7f51ef4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 20 Jul 2021 11:08:53 -0400 Subject: [PATCH 3/6] Add all physical optimizer passes --- datafusion/src/execution/context.rs | 5 +- datafusion/src/logical_plan/builder.rs | 13 ++-- datafusion/src/logical_plan/mod.rs | 2 +- datafusion/src/logical_plan/plan.rs | 20 +++++++ datafusion/src/physical_plan/display.rs | 11 ++++ datafusion/src/physical_plan/explain.rs | 15 ++++- datafusion/src/physical_plan/planner.rs | 79 ++++++++++++------------- datafusion/src/sql/planner.rs | 9 +-- 8 files changed, 93 insertions(+), 61 deletions(-) diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs index 372bfc94646d0..126493849845b 100644 --- a/datafusion/src/execution/context.rs +++ b/datafusion/src/execution/context.rs @@ -21,7 +21,7 @@ use crate::{ catalog::{CatalogList, MemoryCatalogList}, information_schema::CatalogWithInformationSchema, }, - logical_plan::{PlanType, StringifiedPlan}, + logical_plan::{PlanType, ToStringifiedPlan}, optimizer::{ aggregate_statistics::AggregateStatistics, eliminate_limit::EliminateLimit, hash_build_probe_order::HashBuildProbeOrder, @@ -460,8 +460,7 @@ impl ExecutionContext { let plan = self.optimize_internal(plan, |optimized_plan, optimizer| { let optimizer_name = optimizer.name().to_string(); let plan_type = PlanType::OptimizedLogicalPlan { optimizer_name }; - let plan_string = format!("{:#?}", optimized_plan); - stringified_plans.push(StringifiedPlan::new(plan_type, plan_string)); + stringified_plans.push(optimized_plan.to_stringified(plan_type)); })?; Ok(LogicalPlan::Explain { diff --git a/datafusion/src/logical_plan/builder.rs b/datafusion/src/logical_plan/builder.rs index 0335e29127ab9..7aedec7f1f2ee 100644 --- a/datafusion/src/logical_plan/builder.rs +++ b/datafusion/src/logical_plan/builder.rs @@ -27,18 +27,15 @@ use arrow::{ record_batch::RecordBatch, }; -use crate::datasource::TableProvider; use crate::error::{DataFusionError, Result}; +use crate::{datasource::TableProvider, logical_plan::plan::ToStringifiedPlan}; use crate::{ datasource::{empty::EmptyTable, parquet::ParquetTable, CsvFile, MemTable}, prelude::CsvReadOptions, }; use super::dfschema::ToDFSchema; -use super::{ - exprlist_to_fields, Expr, JoinConstraint, JoinType, LogicalPlan, PlanType, - StringifiedPlan, -}; +use super::{exprlist_to_fields, Expr, JoinConstraint, JoinType, LogicalPlan, PlanType}; use crate::logical_plan::{ columnize_expr, normalize_col, normalize_cols, Column, DFField, DFSchema, DFSchemaRef, Partitioning, @@ -398,10 +395,8 @@ impl LogicalPlanBuilder { /// Create an expression to represent the explanation of the plan pub fn explain(&self, verbose: bool) -> Result { - let stringified_plans = vec![StringifiedPlan::new( - PlanType::InitialLogicalPlan, - format!("{:#?}", self.plan.clone()), - )]; + let stringified_plans = + vec![self.plan.to_stringified(PlanType::InitialLogicalPlan)]; let schema = LogicalPlan::explain_schema(); diff --git a/datafusion/src/logical_plan/mod.rs b/datafusion/src/logical_plan/mod.rs index f381e316669e4..a021d06f09502 100644 --- a/datafusion/src/logical_plan/mod.rs +++ b/datafusion/src/logical_plan/mod.rs @@ -50,6 +50,6 @@ pub use extension::UserDefinedLogicalNode; pub use operators::Operator; pub use plan::{ JoinConstraint, JoinType, LogicalPlan, Partitioning, PlanType, PlanVisitor, - StringifiedPlan, }; +pub(crate) use plan::{StringifiedPlan, ToStringifiedPlan}; pub use registry::FunctionRegistry; diff --git a/datafusion/src/logical_plan/plan.rs b/datafusion/src/logical_plan/plan.rs index 9a4daae27ff56..28405fb6dfba0 100644 --- a/datafusion/src/logical_plan/plan.rs +++ b/datafusion/src/logical_plan/plan.rs @@ -820,6 +820,11 @@ pub enum PlanType { FinalLogicalPlan, /// The initial physical plan, prepared for execution InitialPhysicalPlan, + /// The ExecutionPlan which results from applying an optimizer pass + OptimizedPhysicalPlan { + /// The name of the optimizer which produced this plan + optimizer_name: String, + }, /// The final, fully optimized physical which would be executed FinalPhysicalPlan, } @@ -833,6 +838,9 @@ impl fmt::Display for PlanType { } PlanType::FinalLogicalPlan => write!(f, "logical_plan"), PlanType::InitialPhysicalPlan => write!(f, "initial_physical_plan"), + PlanType::OptimizedPhysicalPlan { optimizer_name } => { + write!(f, "physical_plan after {}", optimizer_name) + } PlanType::FinalPhysicalPlan => write!(f, "physical_plan"), } } @@ -868,6 +876,18 @@ impl StringifiedPlan { } } +/// Trait for something that can be formatted as a stringified plan +pub trait ToStringifiedPlan { + /// Create a stringified plan with the specified type + fn to_stringified(&self, plan_type: PlanType) -> StringifiedPlan; +} + +impl ToStringifiedPlan for LogicalPlan { + fn to_stringified(&self, plan_type: PlanType) -> StringifiedPlan { + StringifiedPlan::new(plan_type, self.display_indent().to_string()) + } +} + #[cfg(test)] mod tests { use super::super::{col, lit, LogicalPlanBuilder}; diff --git a/datafusion/src/physical_plan/display.rs b/datafusion/src/physical_plan/display.rs index 8498e02d50c88..e251e4ea53db1 100644 --- a/datafusion/src/physical_plan/display.rs +++ b/datafusion/src/physical_plan/display.rs @@ -21,6 +21,8 @@ use std::fmt; +use crate::logical_plan::{StringifiedPlan, ToStringifiedPlan}; + use super::{accept, ExecutionPlan, ExecutionPlanVisitor}; /// Options for controlling how each [`ExecutionPlan`] should format itself @@ -131,3 +133,12 @@ impl<'a, 'b> ExecutionPlanVisitor for IndentVisitor<'a, 'b> { Ok(true) } } + +impl<'a> ToStringifiedPlan for DisplayableExecutionPlan<'a> { + fn to_stringified( + &self, + plan_type: crate::logical_plan::PlanType, + ) -> StringifiedPlan { + StringifiedPlan::new(plan_type, self.indent().to_string()) + } +} diff --git a/datafusion/src/physical_plan/explain.rs b/datafusion/src/physical_plan/explain.rs index 4fd3711d5d2e3..8edc9a5f22650 100644 --- a/datafusion/src/physical_plan/explain.rs +++ b/datafusion/src/physical_plan/explain.rs @@ -121,8 +121,8 @@ impl ExecutionPlan for ExplainExec { for p in plans_to_print { type_builder.append_value(p.plan_type.to_string())?; match prev { - Some(prev) if p.plan == prev.plan => { - plan_builder.append_value("SAME")?; + Some(prev) if !should_show(prev, p) => { + plan_builder.append_value("SAME TEXT AS ABOVE")?; } Some(_) | None => { plan_builder.append_value(&*p.plan)?; @@ -157,3 +157,14 @@ impl ExecutionPlan for ExplainExec { } } } + +/// If this plan should be shown, given the previous plan that was +/// displayed. +/// +/// This is meant to avoid repeating the same plan over and over again +/// to make clear what is changing +fn should_show(previous_plan: &StringifiedPlan, this_plan: &StringifiedPlan) -> bool { + // if the plans are different, or if they would have been + // displayed in the normal explain (aka non verbose) plan + (previous_plan.plan != this_plan.plan) || this_plan.should_display(false) +} diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/src/physical_plan/planner.rs index 5163e4b425b4f..6bd898de8da14 100644 --- a/datafusion/src/physical_plan/planner.rs +++ b/datafusion/src/physical_plan/planner.rs @@ -24,9 +24,10 @@ use super::{ use crate::execution::context::ExecutionContextState; use crate::logical_plan::{ unnormalize_cols, DFSchema, Expr, LogicalPlan, Operator, - Partitioning as LogicalPartitioning, PlanType, StringifiedPlan, + Partitioning as LogicalPartitioning, PlanType, ToStringifiedPlan, UserDefinedLogicalNode, }; +use crate::physical_optimizer::optimizer::PhysicalOptimizerRule; use crate::physical_plan::explain::ExplainExec; use crate::physical_plan::expressions; use crate::physical_plan::expressions::{CaseExpr, Column, Literal, PhysicalSortExpr}; @@ -244,7 +245,7 @@ impl PhysicalPlanner for DefaultPhysicalPlanner { Some(plan) => Ok(plan), None => { let plan = self.create_initial_plan(logical_plan, ctx_state)?; - self.optimize_plan(plan, ctx_state) + self.optimize_internal(plan, ctx_state, |_, _| {}) } } } @@ -285,23 +286,6 @@ impl DefaultPhysicalPlanner { Self { extension_planners } } - /// Optimize a physical plan by applying each physical optimizer - fn optimize_plan( - &self, - plan: Arc, - ctx_state: &ExecutionContextState, - ) -> Result> { - let optimizers = &ctx_state.config.physical_optimizers; - debug!("Physical plan:\n{:?}", plan); - - let mut new_plan = plan; - for optimizer in optimizers { - new_plan = optimizer.optimize(new_plan, &ctx_state.config)?; - } - debug!("Optimized physical plan:\n{:?}", new_plan); - Ok(new_plan) - } - /// Create a physical plan from a logical plan fn create_initial_plan( &self, @@ -1315,32 +1299,24 @@ impl DefaultPhysicalPlanner { schema, } = logical_plan { - let final_logical_plan = StringifiedPlan::new( - PlanType::FinalLogicalPlan, - plan.display_indent().to_string(), - ); + use PlanType::*; + let mut stringified_plans = stringified_plans.clone(); - let input = self.create_initial_plan(plan, ctx_state)?; + stringified_plans.push(plan.to_stringified(FinalLogicalPlan)); - let initial_physical_plan = StringifiedPlan::new( - PlanType::InitialPhysicalPlan, - displayable(input.as_ref()).indent().to_string(), - ); + let input = self.create_initial_plan(plan, ctx_state)?; - let input = self.optimize_plan(input, ctx_state)?; + stringified_plans + .push(displayable(input.as_ref()).to_stringified(InitialPhysicalPlan)); - let final_physical_plan = StringifiedPlan::new( - PlanType::FinalPhysicalPlan, - displayable(input.as_ref()).indent().to_string(), - ); + let input = self.optimize_internal(input, ctx_state, |plan, optimizer| { + let optimizer_name = optimizer.name().to_string(); + let plan_type = OptimizedPhysicalPlan { optimizer_name }; + stringified_plans.push(displayable(plan).to_stringified(plan_type)); + })?; - let stringified_plans = stringified_plans - .iter() - .cloned() - .chain(std::iter::once(final_logical_plan)) - .chain(std::iter::once(initial_physical_plan)) - .chain(std::iter::once(final_physical_plan)) - .collect::>(); + stringified_plans + .push(displayable(input.as_ref()).to_stringified(FinalPhysicalPlan)); Ok(Some(Arc::new(ExplainExec::new( SchemaRef::new(schema.as_ref().to_owned().into()), @@ -1351,6 +1327,29 @@ impl DefaultPhysicalPlanner { Ok(None) } } + + /// Optimize a physical plan by applying each physical optimizer, + /// calling observer(plan, optimizer after each one) + fn optimize_internal( + &self, + plan: Arc, + ctx_state: &ExecutionContextState, + mut observer: F, + ) -> Result> + where + F: FnMut(&dyn ExecutionPlan, &dyn PhysicalOptimizerRule), + { + let optimizers = &ctx_state.config.physical_optimizers; + debug!("Physical plan:\n{:?}", plan); + + let mut new_plan = plan; + for optimizer in optimizers { + new_plan = optimizer.optimize(new_plan, &ctx_state.config)?; + observer(new_plan.as_ref(), optimizer.as_ref()) + } + debug!("Optimized physical plan:\n{:?}", new_plan); + Ok(new_plan) + } } fn tuple_err(value: (Result, Result)) -> Result<(T, R)> { diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index a4bb02cf0f9a9..fa2b035162a60 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -28,8 +28,8 @@ use crate::logical_plan::window_frames::{WindowFrame, WindowFrameUnits}; use crate::logical_plan::Expr::Alias; use crate::logical_plan::{ and, builder::expand_wildcard, col, lit, normalize_col, union_with_alias, Column, - DFSchema, Expr, LogicalPlan, LogicalPlanBuilder, Operator, PlanType, StringifiedPlan, - ToDFSchema, + DFSchema, Expr, LogicalPlan, LogicalPlanBuilder, Operator, PlanType, ToDFSchema, + ToStringifiedPlan, }; use crate::prelude::JoinType; use crate::scalar::ScalarValue; @@ -233,10 +233,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { ) -> Result { let plan = self.sql_statement_to_plan(statement)?; - let stringified_plans = vec![StringifiedPlan::new( - PlanType::InitialLogicalPlan, - plan.display_indent().to_string(), - )]; + let stringified_plans = vec![plan.to_stringified(PlanType::InitialLogicalPlan)]; let schema = LogicalPlan::explain_schema(); let plan = Arc::new(plan); From a1d5a4ac518371184eb78f8cc56e7a89f01dc62a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 20 Jul 2021 11:57:21 -0400 Subject: [PATCH 4/6] tests --- datafusion/src/execution/context.rs | 42 +++++++++++++++++++++++++ datafusion/src/logical_plan/builder.rs | 2 ++ datafusion/src/optimizer/utils.rs | 2 +- datafusion/src/physical_plan/planner.rs | 32 +++++++++++++++++++ datafusion/tests/sql.rs | 13 ++++---- 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs index 126493849845b..fc3e25e3022b6 100644 --- a/datafusion/src/execution/context.rs +++ b/datafusion/src/execution/context.rs @@ -980,6 +980,48 @@ mod tests { use tempfile::TempDir; use test::*; + #[test] + fn optimize_explain() { + let schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false), + ]); + + let plan = LogicalPlanBuilder::scan_empty( + Some("employee"), + &schema, + None, + ) + .unwrap() + .explain(true) + .unwrap() + .build() + .unwrap(); + + if let LogicalPlan::Explain{stringified_plans, ..} = &plan { + assert_eq!(stringified_plans.len(), 1); + } + else { + panic!("plan was not an explain: {:?}", plan); + } + + // now optimize the plan and expect to see more plans + let optimized_plan = ExecutionContext::new().optimize(&plan).unwrap(); + if let LogicalPlan::Explain{stringified_plans, ..} = &optimized_plan { + // should have more than one plan + assert!(stringified_plans.len() > 1, "plans: {:#?}", stringified_plans); + // should have at least one optimized plan + let opt = stringified_plans + .iter() + .any(|p| matches!(p.plan_type, PlanType::OptimizedLogicalPlan{..})); + + assert!(opt, "plans: {:#?}", stringified_plans); + } + else { + panic!("plan was not an explain: {:?}", plan); + } + } + + #[tokio::test] async fn parallel_projection() -> Result<()> { let partition_count = 4; diff --git a/datafusion/src/logical_plan/builder.rs b/datafusion/src/logical_plan/builder.rs index 7aedec7f1f2ee..60e0ed3c09883 100644 --- a/datafusion/src/logical_plan/builder.rs +++ b/datafusion/src/logical_plan/builder.rs @@ -548,6 +548,8 @@ pub(crate) fn expand_wildcard( mod tests { use arrow::datatypes::{DataType, Field}; + use crate::logical_plan::StringifiedPlan; + use super::super::{col, lit, sum}; use super::*; diff --git a/datafusion/src/optimizer/utils.rs b/datafusion/src/optimizer/utils.rs index 2530a1efd1338..615f0ccfceaf5 100644 --- a/datafusion/src/optimizer/utils.rs +++ b/datafusion/src/optimizer/utils.rs @@ -441,7 +441,7 @@ pub fn rewrite_expression(expr: &Expr, expressions: &[Expr]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::logical_plan::{col, LogicalPlanBuilder}; + use crate::logical_plan::col; use arrow::datatypes::DataType; use std::collections::HashSet; diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/src/physical_plan/planner.rs index 6bd898de8da14..2c0c7ed493505 100644 --- a/datafusion/src/physical_plan/planner.rs +++ b/datafusion/src/physical_plan/planner.rs @@ -1644,6 +1644,38 @@ mod tests { Ok(()) } + #[test] + fn test_explain() { + let schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false), + ]); + + let logical_plan = LogicalPlanBuilder::scan_empty( + Some("employee"), + &schema, + None, + ) + .unwrap() + .explain(true) + .unwrap() + .build() + .unwrap(); + + let plan = plan(&logical_plan).unwrap(); + if let Some(plan) = plan.as_any().downcast_ref::() { + let stringified_plans = plan.stringified_plans(); + assert!(stringified_plans.len() >= 4); + assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::FinalLogicalPlan))); + assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::InitialPhysicalPlan))); + assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::OptimizedPhysicalPlan{..}))); + assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::FinalPhysicalPlan))); + } + else { + panic!("Plan was not an explain plan: {}", displayable(plan.as_ref()).indent()); + } + } + + /// An example extension node that doesn't do anything struct NoOpExtensionNode { schema: DFSchemaRef, diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 2c8c0dd6be332..0fc0e2bb429bd 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -2206,7 +2206,13 @@ async fn csv_explain_verbose() { actual ); - assert!(false, "TODO: add test for NO CHANGE"); + // ensure the "same text as above" optimization is working + assert!( + actual.contains("logical_plan after limit_push_down | SAME TEXT AS ABOVE"), + "Actual: '{}'", + actual + ); + } #[tokio::test] @@ -2509,11 +2515,6 @@ async fn execute_to_batches(ctx: &mut ExecutionContext, sql: &str) -> Vec Date: Tue, 20 Jul 2021 12:19:58 -0400 Subject: [PATCH 5/6] cleanup --- datafusion/src/execution/context.rs | 37 +++++++++---------- datafusion/src/physical_plan/explain.rs | 2 +- datafusion/src/physical_plan/planner.rs | 48 +++++++++++++------------ datafusion/tests/sql.rs | 5 ++- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs index fc3e25e3022b6..0cf8b3b6c2765 100644 --- a/datafusion/src/execution/context.rs +++ b/datafusion/src/execution/context.rs @@ -982,46 +982,47 @@ mod tests { #[test] fn optimize_explain() { - let schema = Schema::new(vec![ - Field::new("id", DataType::Int32, false), - ]); + let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]); - let plan = LogicalPlanBuilder::scan_empty( - Some("employee"), - &schema, - None, - ) + let plan = LogicalPlanBuilder::scan_empty(Some("employee"), &schema, None) .unwrap() .explain(true) .unwrap() .build() .unwrap(); - if let LogicalPlan::Explain{stringified_plans, ..} = &plan { + if let LogicalPlan::Explain { + stringified_plans, .. + } = &plan + { assert_eq!(stringified_plans.len(), 1); - } - else { + } else { panic!("plan was not an explain: {:?}", plan); } // now optimize the plan and expect to see more plans let optimized_plan = ExecutionContext::new().optimize(&plan).unwrap(); - if let LogicalPlan::Explain{stringified_plans, ..} = &optimized_plan { + if let LogicalPlan::Explain { + stringified_plans, .. + } = &optimized_plan + { // should have more than one plan - assert!(stringified_plans.len() > 1, "plans: {:#?}", stringified_plans); + assert!( + stringified_plans.len() > 1, + "plans: {:#?}", + stringified_plans + ); // should have at least one optimized plan - let opt = stringified_plans + let opt = stringified_plans .iter() - .any(|p| matches!(p.plan_type, PlanType::OptimizedLogicalPlan{..})); + .any(|p| matches!(p.plan_type, PlanType::OptimizedLogicalPlan { .. })); assert!(opt, "plans: {:#?}", stringified_plans); - } - else { + } else { panic!("plan was not an explain: {:?}", plan); } } - #[tokio::test] async fn parallel_projection() -> Result<()> { let partition_count = 4; diff --git a/datafusion/src/physical_plan/explain.rs b/datafusion/src/physical_plan/explain.rs index 8edc9a5f22650..a6a34f5d0b0cf 100644 --- a/datafusion/src/physical_plan/explain.rs +++ b/datafusion/src/physical_plan/explain.rs @@ -162,7 +162,7 @@ impl ExecutionPlan for ExplainExec { /// displayed. /// /// This is meant to avoid repeating the same plan over and over again -/// to make clear what is changing +/// in explain plans to make clear what is changing fn should_show(previous_plan: &StringifiedPlan, this_plan: &StringifiedPlan) -> bool { // if the plans are different, or if they would have been // displayed in the normal explain (aka non verbose) plan diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/src/physical_plan/planner.rs index 2c0c7ed493505..e662821e4539f 100644 --- a/datafusion/src/physical_plan/planner.rs +++ b/datafusion/src/physical_plan/planner.rs @@ -1646,36 +1646,40 @@ mod tests { #[test] fn test_explain() { - let schema = Schema::new(vec![ - Field::new("id", DataType::Int32, false), - ]); - - let logical_plan = LogicalPlanBuilder::scan_empty( - Some("employee"), - &schema, - None, - ) - .unwrap() - .explain(true) - .unwrap() - .build() - .unwrap(); + let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]); + + let logical_plan = + LogicalPlanBuilder::scan_empty(Some("employee"), &schema, None) + .unwrap() + .explain(true) + .unwrap() + .build() + .unwrap(); let plan = plan(&logical_plan).unwrap(); if let Some(plan) = plan.as_any().downcast_ref::() { let stringified_plans = plan.stringified_plans(); assert!(stringified_plans.len() >= 4); - assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::FinalLogicalPlan))); - assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::InitialPhysicalPlan))); - assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::OptimizedPhysicalPlan{..}))); - assert!(stringified_plans.iter().any(|p| matches!(p.plan_type, PlanType::FinalPhysicalPlan))); - } - else { - panic!("Plan was not an explain plan: {}", displayable(plan.as_ref()).indent()); + assert!(stringified_plans + .iter() + .any(|p| matches!(p.plan_type, PlanType::FinalLogicalPlan))); + assert!(stringified_plans + .iter() + .any(|p| matches!(p.plan_type, PlanType::InitialPhysicalPlan))); + assert!(stringified_plans + .iter() + .any(|p| matches!(p.plan_type, PlanType::OptimizedPhysicalPlan { .. }))); + assert!(stringified_plans + .iter() + .any(|p| matches!(p.plan_type, PlanType::FinalPhysicalPlan))); + } else { + panic!( + "Plan was not an explain plan: {}", + displayable(plan.as_ref()).indent() + ); } } - /// An example extension node that doesn't do anything struct NoOpExtensionNode { schema: DFSchemaRef, diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 0fc0e2bb429bd..0ef8b4ca30243 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -2208,11 +2208,10 @@ async fn csv_explain_verbose() { // ensure the "same text as above" optimization is working assert!( - actual.contains("logical_plan after limit_push_down | SAME TEXT AS ABOVE"), - "Actual: '{}'", + actual.contains("SAME TEXT AS ABOVE"), + "Actual 2: '{}'", actual ); - } #[tokio::test] From 8a97e07a434a223627d7336b538d2177846d817b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 20 Jul 2021 12:39:54 -0400 Subject: [PATCH 6/6] fix user_defined_plan test --- datafusion/tests/user_defined_plan.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/tests/user_defined_plan.rs b/datafusion/tests/user_defined_plan.rs index e1f8c767bd8d5..c1269d9a217ff 100644 --- a/datafusion/tests/user_defined_plan.rs +++ b/datafusion/tests/user_defined_plan.rs @@ -163,9 +163,9 @@ async fn topk_plan() -> Result<()> { let mut ctx = setup_table(make_topk_context()).await?; let expected = vec![ - "| logical_plan after topk | TopK: k=3 |", - "| | Projection: #sales.customer_id, #sales.revenue |", - "| | TableScan: sales projection=Some([0, 1]) |", + "| logical_plan after topk | TopK: k=3 |", + "| | Projection: #sales.customer_id, #sales.revenue |", + "| | TableScan: sales projection=Some([0, 1]) |", ].join("\n"); let explain_query = format!("EXPLAIN VERBOSE {}", QUERY);