From cc49b60dadc59af30fc870a7869eb745ae602da1 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Mon, 2 Nov 2026 14:46:42 -0500 Subject: [PATCH 1/2] decline datafusion projections the vortex scan does not evaluate Signed-off-by: Matt Katz --- vortex-datafusion/src/persistent/format.rs | 7 +- vortex-datafusion/src/persistent/opener.rs | 31 +++++++-- vortex-datafusion/src/persistent/source.rs | 79 +++++++++++++++++++--- 3 files changed, 98 insertions(+), 19 deletions(-) diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index fb50ed26d17..3ad3623928a 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -215,9 +215,10 @@ extensions_options! { pub footer_initial_read_size_bytes: usize, default = DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES /// Whether to enable projection pushdown into the underlying Vortex scan. /// - /// When enabled, projection expressions may be partially evaluated during - /// the scan. When disabled, Vortex reads only the referenced columns and - /// all expressions are evaluated after the scan. + /// When enabled, the scan evaluates projections whose expressions all convert + /// to Vortex, and DataFusion evaluates any other projection after the scan. + /// When disabled, Vortex reads only the referenced columns and all expressions + /// are evaluated after the scan. /// /// Disabled by default. pub projection_pushdown: bool, default = false diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 208691f0b0e..cd34e0139fa 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -9,6 +9,7 @@ use std::sync::Weak; use arrow_array::RecordBatchOptions; use arrow_schema::DataType; use arrow_schema::Field; +use arrow_schema::Schema; use arrow_schema::SchemaRef; use datafusion_common::DataFusionError; use datafusion_common::Result as DFResult; @@ -315,9 +316,6 @@ impl FileOpener for VortexOpener { let projection = projection.try_map_exprs(|p| simplifier.simplify(expr_adapter.rewrite(p)?))?; - // TODO: support lambda pushdown. - let projection_pushdown = projection_pushdown && !contains_lambda(&projection)?; - // Split the adapted filter into conjuncts Vortex evaluates natively and residual // conjuncts DataFusion evaluates on the raw scan output. let mut native_filters = Vec::new(); @@ -373,14 +371,14 @@ impl FileOpener for VortexOpener { scan_reference_schema, leftover_projection: projection, } - } else if projection_pushdown { - expr_converter.split_projection( + } else { + process_projection( + expr_converter.as_ref(), projection, &this_file_schema, output_schema.as_ref(), + projection_pushdown, )? - } else { - expr_converter.no_pushdown_projection(projection, &this_file_schema)? }; // The schema of the stream returned from the vortex scan. @@ -680,6 +678,25 @@ impl NaturalSplits { } } +/// Splits `projection` into the Vortex scan projection and the DataFusion leftover. +/// +/// [`VortexSource`](super::VortexSource) makes the same split against the table schema when +/// planning, and accepts only projections the scan evaluates entirely. +pub(super) fn process_projection( + expr_converter: &dyn ExpressionConverter, + projection: ProjectionExprs, + input_schema: &Schema, + output_schema: &Schema, + projection_pushdown: bool, +) -> DFResult { + // TODO: support lambda pushdown. + if projection_pushdown && !contains_lambda(&projection)? { + expr_converter.split_projection(projection, input_schema, output_schema) + } else { + expr_converter.no_pushdown_projection(projection, input_schema) + } +} + /// Whether any expression in `projection` contains a lambda. fn contains_lambda(projection: &ProjectionExprs) -> DFResult { for projection_expr in projection.iter() { diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index 644fea4e71c..91f5a84b930 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -20,6 +20,7 @@ use datafusion_physical_expr::LexOrdering; use datafusion_physical_expr::PhysicalExprRef; use datafusion_physical_expr::PhysicalSortExpr; use datafusion_physical_expr::conjunction; +use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::projection::ProjectionExprs; use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory; use datafusion_physical_expr_common::physical_expr::fmt_sql; @@ -43,9 +44,11 @@ use vortex_utils::aliases::dash_map::DashMap; use super::opener::NaturalSplits; use super::opener::VortexOpener; +use super::opener::process_projection; use crate::VortexTableOptions; use crate::convert::exprs::DefaultExpressionConverter; use crate::convert::exprs::ExpressionConverter; +use crate::convert::exprs::ProcessedProjection; use crate::persistent::reader::DefaultVortexReaderFactory; use crate::persistent::reader::VortexReaderFactory; @@ -137,13 +140,15 @@ use crate::persistent::reader::VortexReaderFactory; /// - `vortex_predicate`, which contains accepted exact filters. After per-file /// adaptation, these run either natively or as DataFusion residual filters. /// -/// Projection handling depends on -/// [`VortexTableOptions::projection_pushdown`]: +/// `VortexSource` accepts a projection only when the scan evaluates all of it, so +/// DataFusion keeps any other projection in a `ProjectionExec` above the scan. What +/// the scan evaluates depends on [`VortexTableOptions::projection_pushdown`]: /// -/// - when disabled, `VortexSource` still prunes unreferenced top-level columns, -/// but DataFusion applies the full projection after the scan, -/// - when enabled, the default converter evaluates fully supported projections -/// natively. Otherwise, DataFusion evaluates the full projection over raw columns. +/// - when disabled, the scan only selects top-level columns, +/// - when enabled, the scan also evaluates projections the converter fully supports. +/// +/// A file whose schema differs from the table schema can still need DataFusion to +/// evaluate part of an accepted projection, which then happens inside the scan. /// /// Predicate handling depends on [`VortexTableOptions::predicate_pushdown`]: /// @@ -563,8 +568,31 @@ impl FileSource for VortexSource { &self, projection: &ProjectionExprs, ) -> DFResult>> { + let projection = self.projection.try_merge(projection)?; + + // Accept only projections the scan evaluates entirely, so DataFusion keeps anything else + // in its own `ProjectionExec`. Files whose schema differs from the table's are split + // again when opened, and may still evaluate part of the projection inside the scan. + let table_schema = self.table_schema.table_schema(); + let ProcessedProjection { + leftover_projection, + .. + } = process_projection( + self.expression_converter.as_ref(), + projection.clone(), + table_schema, + &projection.project_schema(table_schema)?, + self.options.projection_pushdown, + )?; + if !leftover_projection + .iter() + .all(|leftover| leftover.expr.is::()) + { + return Ok(None); + } + let mut source = self.clone(); - source.projection = self.projection.try_merge(projection)?; + source.projection = projection; Ok(Some(Arc::new(source))) } @@ -604,12 +632,12 @@ mod tests { use datafusion_functions::string::octet_length::OctetLengthFunc; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions as df_expr; - use datafusion_physical_expr::expressions::Column; + use datafusion_physical_expr::projection::ProjectionExpr; use object_store::memory::InMemory; + use rstest::rstest; use vortex::VortexSessionDefault; use super::*; - use crate::convert::exprs::ProcessedProjection; struct TrackingExpressionConverter { inner: DefaultExpressionConverter, @@ -774,4 +802,37 @@ mod tests { assert!(updated_source.vortex_predicate.is_none()); Ok(()) } + + /// `None` projects column `b`; `Some(op)` projects `a op 2`. Vortex converts `+` but not `%`. + #[rstest] + #[case::columns(&[None], false, true)] + #[case::columns_with_pushdown(&[None], true, true)] + #[case::convertible_without_pushdown(&[Some(Operator::Plus)], false, false)] + #[case::convertible(&[None, Some(Operator::Plus)], true, true)] + #[case::unconvertible(&[Some(Operator::Modulo)], true, false)] + #[case::mixed(&[Some(Operator::Plus), Some(Operator::Modulo)], true, false)] + fn try_pushdown_projection_accepts_only_what_the_scan_evaluates( + #[case] exprs: &[Option], + #[case] projection_pushdown: bool, + #[case] accepted: bool, + ) -> anyhow::Result<()> { + let source = + sort_test_source(sort_test_schema()).with_projection_pushdown(projection_pushdown); + let projection = ProjectionExprs::new(exprs.iter().enumerate().map(|(i, op)| { + let expr: PhysicalExprRef = match op { + None => Arc::new(Column::new("b", 1)), + Some(op) => Arc::new(df_expr::BinaryExpr::new( + Arc::new(Column::new("a", 0)), + *op, + Arc::new(df_expr::Literal::new(ScalarValue::Int32(Some(2)))), + )), + }; + ProjectionExpr::new(expr, format!("e{i}")) + })); + + let pushed = source.try_pushdown_projection(&projection)?; + + assert_eq!(pushed.is_some(), accepted); + Ok(()) + } } From debe9ecd91fcbd984e831be0b7a915f2048131ca Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Mon, 2 Nov 2026 15:32:53 -0500 Subject: [PATCH 2/2] update datafusion slt plans Signed-off-by: Matt Katz --- .../clickbench/datafusion/plans/q29.slt.no | 3 +- .../slt/datafusion/decimal_to_integer.slt | 7 +- .../slt/tpcds/datafusion/plans/q23.slt.no | 102 +++++++++--------- .../slt/tpcds/datafusion/plans/q5.slt.no | 71 ++++++------ .../slt/tpcds/datafusion/plans/q62.slt.no | 5 +- .../slt/tpcds/datafusion/plans/q99.slt.no | 5 +- .../slt/tpch/datafusion/plans/q1.slt.no | 3 +- 7 files changed, 105 insertions(+), 91 deletions(-) diff --git a/vortex-sqllogictest/slt/clickbench/datafusion/plans/q29.slt.no b/vortex-sqllogictest/slt/clickbench/datafusion/plans/q29.slt.no index eeacb256682..d5a845d4383 100644 --- a/vortex-sqllogictest/slt/clickbench/datafusion/plans/q29.slt.no +++ b/vortex-sqllogictest/slt/clickbench/datafusion/plans/q29.slt.no @@ -16,4 +16,5 @@ physical_plan 02)--AggregateExec: mode=Final, gby=[], aggr=[sum(hits.ResolutionWidth), count(hits.ResolutionWidth)] 03)----CoalescePartitionsExec 04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.ResolutionWidth), count(hits.ResolutionWidth)] -05)--------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/hits.vortex:], [${WORK_DIR}/hits.vortex:], [${WORK_DIR}/hits.vortex:], [${WORK_DIR}/hits.vortex:]]}, projection=[CAST(ResolutionWidth@20 AS Int64) as __common_expr_2], file_type=vortex +05)--------ProjectionExec: expr=[CAST(ResolutionWidth@0 AS Int64) as __common_expr_2] +06)----------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/hits.vortex:], [${WORK_DIR}/hits.vortex:], [${WORK_DIR}/hits.vortex:], [${WORK_DIR}/hits.vortex:]]}, projection=[ResolutionWidth], file_type=vortex diff --git a/vortex-sqllogictest/slt/datafusion/decimal_to_integer.slt b/vortex-sqllogictest/slt/datafusion/decimal_to_integer.slt index a9d0dbcf575..f7934048488 100644 --- a/vortex-sqllogictest/slt/datafusion/decimal_to_integer.slt +++ b/vortex-sqllogictest/slt/datafusion/decimal_to_integer.slt @@ -21,7 +21,8 @@ COPY (SELECT id, CAST(value AS DECIMAL(21,1)) AS value FROM (VALUES statement ok CREATE VIEW decimals AS SELECT * FROM '${WORK_DIR}/decimal_to_integer.vortex'; -# EXPLAIN verifies that the cast executes inside the Vortex scan. +# Vortex does not convert decimal-to-integer casts, so EXPLAIN shows the cast in a +# DataFusion ProjectionExec above the scan. query TT EXPLAIN SELECT CAST(value AS BIGINT) AS integer_value FROM decimals; ---- @@ -29,7 +30,9 @@ logical_plan 01)Projection: CAST(decimals.value AS Int64) AS integer_value 02)--SubqueryAlias: decimals 03)----TableScan: ${WORK_DIR}/decimal_to_integer.vortex projection=[value] -physical_plan DataSourceExec: file_groups={1 group: [[${WORK_DIR}/decimal_to_integer.vortex]]}, projection=[CAST(value@1 AS Int64) as integer_value], file_type=vortex +physical_plan +01)ProjectionExec: expr=[CAST(value@0 AS Int64) as integer_value] +02)--DataSourceExec: file_groups={1 group: [[${WORK_DIR}/decimal_to_integer.vortex]]}, projection=[value], file_type=vortex # Fractions truncate toward zero; large integers remain exact and nulls survive. query I diff --git a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q23.slt.no b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q23.slt.no index 7ca75863192..e0af9316e4f 100644 --- a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q23.slt.no +++ b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q23.slt.no @@ -248,53 +248,55 @@ physical_plan 43)--------------------------RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 4), input_partitions=4 44)----------------------------AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] 45)------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@1, ss_item_sk@0)], projection=[d_date@3, itemdesc@0, i_item_sk@1] -46)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/item.vortex]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=vortex -47)--------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, d_date@1] -48)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk, d_date], file_type=vortex, predicate: d_year@6 IN (SET) ([2000, 2001, 2002, 2003]) -49)----------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=vortex -50)------ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(web_sales.ws_quantity * web_sales.ws_list_price)@2 as sales] -51)--------AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] -52)----------RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 4), input_partitions=4 -53)------------AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] -54)--------------HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(c_customer_sk@0, ws_bill_customer_sk@0)], projection=[ws_quantity@1, ws_list_price@2, c_first_name@3, c_last_name@4] -55)----------------CoalescePartitionsExec -56)------------------FilterExec: CAST(sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 AS Decimal128(38, 15)) > CAST(0.5 * CAST(max(max_store_sales.tpcds_cmax)@2 AS Float64) AS Decimal128(38, 15)), projection=[c_customer_sk@0] -57)--------------------AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] -58)----------------------RepartitionExec: partitioning=Hash([c_customer_sk@0], 4), input_partitions=4 -59)------------------------AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] -60)--------------------------ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] -61)----------------------------CrossJoinExec -62)------------------------------ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] -63)--------------------------------AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] -64)----------------------------------CoalescePartitionsExec -65)------------------------------------AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] -66)--------------------------------------ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] -67)----------------------------------------AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] -68)------------------------------------------RepartitionExec: partitioning=Hash([c_customer_sk@0], 4), input_partitions=4 -69)--------------------------------------------AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] -70)----------------------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@4] -71)------------------------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_year@6 IN (SET) ([2000, 2001, 2002, 2003]) -72)------------------------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_sold_date_sk@1, ss_quantity@3, ss_sales_price@4, c_customer_sk@0] -73)--------------------------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/customer.vortex]]}, projection=[c_customer_sk], file_type=vortex -74)--------------------------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=vortex -75)------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@0] -76)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/customer.vortex]]}, projection=[c_customer_sk], file_type=vortex -77)--------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=vortex -78)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, item_sk@0)], projection=[ws_bill_customer_sk@1, ws_quantity@2, ws_list_price@3, c_first_name@4, c_last_name@5] -79)------------------CoalescePartitionsExec -80)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_bill_customer_sk@3, ws_quantity@4, ws_list_price@5, c_first_name@6, c_last_name@7] -81)----------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_year@6 = 2000 AND d_moy@8 = 2 -82)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@2)], projection=[ws_sold_date_sk@3, ws_item_sk@4, ws_bill_customer_sk@5, ws_quantity@6, ws_list_price@7, c_first_name@1, c_last_name@2] -83)------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/customer.vortex]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=vortex -84)------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=vortex -85)------------------ProjectionExec: expr=[i_item_sk@0 as item_sk] -86)--------------------FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] -87)----------------------ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] -88)------------------------AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] -89)--------------------------RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 4), input_partitions=4 -90)----------------------------AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] -91)------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@1, ss_item_sk@0)], projection=[d_date@3, itemdesc@0, i_item_sk@1] -92)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/item.vortex]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=vortex -93)--------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, d_date@1] -94)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk, d_date], file_type=vortex, predicate: d_year@6 IN (SET) ([2000, 2001, 2002, 2003]) -95)----------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=vortex +46)--------------------------------ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] +47)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/item.vortex]]}, projection=[i_item_sk, i_item_desc], file_type=vortex +48)--------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, d_date@1] +49)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk, d_date], file_type=vortex, predicate: d_year@6 IN (SET) ([2000, 2001, 2002, 2003]) +50)----------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=vortex +51)------ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(web_sales.ws_quantity * web_sales.ws_list_price)@2 as sales] +52)--------AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] +53)----------RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 4), input_partitions=4 +54)------------AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] +55)--------------HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(c_customer_sk@0, ws_bill_customer_sk@0)], projection=[ws_quantity@1, ws_list_price@2, c_first_name@3, c_last_name@4] +56)----------------CoalescePartitionsExec +57)------------------FilterExec: CAST(sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 AS Decimal128(38, 15)) > CAST(0.5 * CAST(max(max_store_sales.tpcds_cmax)@2 AS Float64) AS Decimal128(38, 15)), projection=[c_customer_sk@0] +58)--------------------AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] +59)----------------------RepartitionExec: partitioning=Hash([c_customer_sk@0], 4), input_partitions=4 +60)------------------------AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] +61)--------------------------ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] +62)----------------------------CrossJoinExec +63)------------------------------ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] +64)--------------------------------AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] +65)----------------------------------CoalescePartitionsExec +66)------------------------------------AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] +67)--------------------------------------ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] +68)----------------------------------------AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] +69)------------------------------------------RepartitionExec: partitioning=Hash([c_customer_sk@0], 4), input_partitions=4 +70)--------------------------------------------AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] +71)----------------------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@4] +72)------------------------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_year@6 IN (SET) ([2000, 2001, 2002, 2003]) +73)------------------------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_sold_date_sk@1, ss_quantity@3, ss_sales_price@4, c_customer_sk@0] +74)--------------------------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/customer.vortex]]}, projection=[c_customer_sk], file_type=vortex +75)--------------------------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=vortex +76)------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@0] +77)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/customer.vortex]]}, projection=[c_customer_sk], file_type=vortex +78)--------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=vortex +79)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, item_sk@0)], projection=[ws_bill_customer_sk@1, ws_quantity@2, ws_list_price@3, c_first_name@4, c_last_name@5] +80)------------------CoalescePartitionsExec +81)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_bill_customer_sk@3, ws_quantity@4, ws_list_price@5, c_first_name@6, c_last_name@7] +82)----------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_year@6 = 2000 AND d_moy@8 = 2 +83)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@2)], projection=[ws_sold_date_sk@3, ws_item_sk@4, ws_bill_customer_sk@5, ws_quantity@6, ws_list_price@7, c_first_name@1, c_last_name@2] +84)------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/customer.vortex]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=vortex +85)------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=vortex +86)------------------ProjectionExec: expr=[i_item_sk@0 as item_sk] +87)--------------------FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] +88)----------------------ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] +89)------------------------AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] +90)--------------------------RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 4), input_partitions=4 +91)----------------------------AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] +92)------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@1, ss_item_sk@0)], projection=[d_date@3, itemdesc@0, i_item_sk@1] +93)--------------------------------ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] +94)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/item.vortex]]}, projection=[i_item_sk, i_item_desc], file_type=vortex +95)--------------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, d_date@1] +96)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk, d_date], file_type=vortex, predicate: d_year@6 IN (SET) ([2000, 2001, 2002, 2003]) +97)----------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=vortex diff --git a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q5.slt.no b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q5.slt.no index c8dd02ad643..13cd739550b 100644 --- a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q5.slt.no +++ b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q5.slt.no @@ -195,36 +195,41 @@ physical_plan 14)------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, date_sk@1)], projection=[store_sk@1, sales_price@3, profit@4, return_amt@5, net_loss@6] 15)--------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06 16)--------------------------UnionExec -17)----------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_store_sk@7 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@15 as sales_price, ss_net_profit@22 as profit, CAST(0.00 AS Decimal128(7, 2)) as return_amt, CAST(0.00 AS Decimal128(7, 2)) as net_loss], file_type=vortex -18)----------------------------ProjectionExec: expr=[store_sk@0 as store_sk, date_sk@1 as date_sk, CAST(sales_price@2 AS Decimal128(7, 2)) as sales_price, CAST(profit@3 AS Decimal128(7, 2)) as profit, return_amt@4 as return_amt, net_loss@5 as net_loss] -19)------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -20)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/store_returns.vortex]]}, projection=[sr_store_sk@7 as store_sk, sr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, sr_return_amt@11 as return_amt, sr_net_loss@19 as net_loss], file_type=vortex -21)--------------ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] -22)----------------AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] -23)------------------RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 4), input_partitions=4 -24)--------------------AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] -25)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(page_sk@0, cp_catalog_page_sk@0)], projection=[sales_price@1, profit@2, return_amt@3, net_loss@4, cp_catalog_page_id@6] -26)------------------------CoalescePartitionsExec -27)--------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, date_sk@1)], projection=[page_sk@1, sales_price@3, profit@4, return_amt@5, net_loss@6] -28)----------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06 -29)----------------------------UnionExec -30)------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:]]}, projection=[cs_catalog_page_sk@12 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@23 as sales_price, cs_net_profit@33 as profit, CAST(0.00 AS Decimal128(7, 2)) as return_amt, CAST(0.00 AS Decimal128(7, 2)) as net_loss], file_type=vortex -31)------------------------------ProjectionExec: expr=[page_sk@0 as page_sk, date_sk@1 as date_sk, CAST(sales_price@2 AS Decimal128(7, 2)) as sales_price, CAST(profit@3 AS Decimal128(7, 2)) as profit, return_amt@4 as return_amt, net_loss@5 as net_loss] -32)--------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -33)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/catalog_returns.vortex]]}, projection=[cr_catalog_page_sk@12 as page_sk, cr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, cr_return_amount@18 as return_amt, cr_net_loss@26 as net_loss], file_type=vortex -34)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -35)--------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/catalog_page.vortex]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=vortex -36)--------------ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] -37)----------------AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] -38)------------------RepartitionExec: partitioning=Hash([web_site_id@0], 4), input_partitions=8 -39)--------------------AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] -40)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(web_site_sk@0, wsr_web_site_sk@0)], projection=[sales_price@3, profit@4, return_amt@5, net_loss@6, web_site_id@1] -41)------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/web_site.vortex]]}, projection=[web_site_sk, web_site_id], file_type=vortex -42)------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, date_sk@1)], projection=[wsr_web_site_sk@1, sales_price@3, profit@4, return_amt@5, net_loss@6] -43)--------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06 -44)--------------------------UnionExec -45)----------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_web_site_sk@13 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@23 as sales_price, ws_net_profit@33 as profit, CAST(0.00 AS Decimal128(7, 2)) as return_amt, CAST(0.00 AS Decimal128(7, 2)) as net_loss], file_type=vortex -46)----------------------------ProjectionExec: expr=[ws_web_site_sk@0 as wsr_web_site_sk, wr_returned_date_sk@1 as date_sk, CAST(0.00 AS Decimal128(7, 2)) as sales_price, CAST(0.00 AS Decimal128(7, 2)) as profit, wr_return_amt@2 as return_amt, wr_net_loss@3 as net_loss] -47)------------------------------HashJoinExec: mode=CollectLeft, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)], projection=[ws_web_site_sk@6, wr_returned_date_sk@0, wr_return_amt@3, wr_net_loss@4] -48)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/web_returns.vortex]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=vortex -49)--------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=vortex +17)----------------------------ProjectionExec: expr=[ss_store_sk@1 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@2 as sales_price, ss_net_profit@3 as profit, CAST(0.00 AS Decimal128(7, 2)) as return_amt, CAST(0.00 AS Decimal128(7, 2)) as net_loss] +18)------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:], [${WORK_DIR}/store_sales.vortex:]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=vortex +19)----------------------------ProjectionExec: expr=[store_sk@0 as store_sk, date_sk@1 as date_sk, CAST(sales_price@2 AS Decimal128(7, 2)) as sales_price, CAST(profit@3 AS Decimal128(7, 2)) as profit, return_amt@4 as return_amt, net_loss@5 as net_loss] +20)------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +21)--------------------------------ProjectionExec: expr=[sr_store_sk@1 as store_sk, sr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, sr_return_amt@2 as return_amt, sr_net_loss@3 as net_loss] +22)----------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/store_returns.vortex]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=vortex +23)--------------ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] +24)----------------AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] +25)------------------RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 4), input_partitions=4 +26)--------------------AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] +27)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(page_sk@0, cp_catalog_page_sk@0)], projection=[sales_price@1, profit@2, return_amt@3, net_loss@4, cp_catalog_page_id@6] +28)------------------------CoalescePartitionsExec +29)--------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, date_sk@1)], projection=[page_sk@1, sales_price@3, profit@4, return_amt@5, net_loss@6] +30)----------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06 +31)----------------------------UnionExec +32)------------------------------ProjectionExec: expr=[cs_catalog_page_sk@1 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@2 as sales_price, cs_net_profit@3 as profit, CAST(0.00 AS Decimal128(7, 2)) as return_amt, CAST(0.00 AS Decimal128(7, 2)) as net_loss] +33)--------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_ext_sales_price, cs_net_profit], file_type=vortex +34)------------------------------ProjectionExec: expr=[page_sk@0 as page_sk, date_sk@1 as date_sk, CAST(sales_price@2 AS Decimal128(7, 2)) as sales_price, CAST(profit@3 AS Decimal128(7, 2)) as profit, return_amt@4 as return_amt, net_loss@5 as net_loss] +35)--------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +36)----------------------------------ProjectionExec: expr=[cr_catalog_page_sk@1 as page_sk, cr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, cr_return_amount@2 as return_amt, cr_net_loss@3 as net_loss] +37)------------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/catalog_returns.vortex]]}, projection=[cr_returned_date_sk, cr_catalog_page_sk, cr_return_amount, cr_net_loss], file_type=vortex +38)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +39)--------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/catalog_page.vortex]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=vortex +40)--------------ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] +41)----------------AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] +42)------------------RepartitionExec: partitioning=Hash([web_site_id@0], 4), input_partitions=8 +43)--------------------AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] +44)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(web_site_sk@0, wsr_web_site_sk@0)], projection=[sales_price@3, profit@4, return_amt@5, net_loss@6, web_site_id@1] +45)------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/web_site.vortex]]}, projection=[web_site_sk, web_site_id], file_type=vortex +46)------------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, date_sk@1)], projection=[wsr_web_site_sk@1, sales_price@3, profit@4, return_amt@5, net_loss@6] +47)--------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/date_dim.vortex]]}, projection=[d_date_sk], file_type=vortex, predicate: d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06 +48)--------------------------UnionExec +49)----------------------------ProjectionExec: expr=[ws_web_site_sk@1 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@2 as sales_price, ws_net_profit@3 as profit, CAST(0.00 AS Decimal128(7, 2)) as return_amt, CAST(0.00 AS Decimal128(7, 2)) as net_loss] +50)------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_sold_date_sk, ws_web_site_sk, ws_ext_sales_price, ws_net_profit], file_type=vortex +51)----------------------------ProjectionExec: expr=[ws_web_site_sk@0 as wsr_web_site_sk, wr_returned_date_sk@1 as date_sk, CAST(0.00 AS Decimal128(7, 2)) as sales_price, CAST(0.00 AS Decimal128(7, 2)) as profit, wr_return_amt@2 as return_amt, wr_net_loss@3 as net_loss] +52)------------------------------HashJoinExec: mode=CollectLeft, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)], projection=[ws_web_site_sk@6, wr_returned_date_sk@0, wr_return_amt@3, wr_net_loss@4] +53)--------------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/web_returns.vortex]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=vortex +54)--------------------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=vortex diff --git a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q62.slt.no b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q62.slt.no index f2b8bc7eeb6..6f3d047ee5c 100644 --- a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q62.slt.no +++ b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q62.slt.no @@ -85,5 +85,6 @@ physical_plan 12)------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sm_ship_mode_sk@0, ws_ship_mode_sk@3)], projection=[ws_sold_date_sk@2, ws_ship_date_sk@3, ws_web_site_sk@4, w_substr@6, sm_type@1] 13)--------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/ship_mode.vortex]]}, projection=[sm_ship_mode_sk, sm_type], file_type=vortex 14)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@1, ws_warehouse_sk@4)], projection=[ws_sold_date_sk@2, ws_ship_date_sk@3, ws_web_site_sk@4, ws_ship_mode_sk@5, w_substr@0] -15)----------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/warehouse.vortex]]}, projection=[substr(w_warehouse_name@2, 1, 20) as w_substr, w_warehouse_sk], file_type=vortex -16)----------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=vortex +15)----------------------ProjectionExec: expr=[substr(w_warehouse_name@1, 1, 20) as w_substr, w_warehouse_sk@0 as w_warehouse_sk] +16)------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/warehouse.vortex]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=vortex +17)----------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:], [${WORK_DIR}/web_sales.vortex:]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=vortex diff --git a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q99.slt.no b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q99.slt.no index 31003b9df3e..de1541ef8e1 100644 --- a/vortex-sqllogictest/slt/tpcds/datafusion/plans/q99.slt.no +++ b/vortex-sqllogictest/slt/tpcds/datafusion/plans/q99.slt.no @@ -84,5 +84,6 @@ physical_plan 12)------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sm_ship_mode_sk@0, cs_ship_mode_sk@3)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_call_center_sk@4, w_substr@6, sm_type@1] 13)--------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/ship_mode.vortex]]}, projection=[sm_ship_mode_sk, sm_type], file_type=vortex 14)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@1, cs_warehouse_sk@4)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_call_center_sk@4, cs_ship_mode_sk@5, w_substr@0] -15)----------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/warehouse.vortex]]}, projection=[substr(w_warehouse_name@2, 1, 20) as w_substr, w_warehouse_sk], file_type=vortex -16)----------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=vortex +15)----------------------ProjectionExec: expr=[substr(w_warehouse_name@1, 1, 20) as w_substr, w_warehouse_sk@0 as w_warehouse_sk] +16)------------------------DataSourceExec: file_groups={1 group: [[${WORK_DIR}/warehouse.vortex]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=vortex +17)----------------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:], [${WORK_DIR}/catalog_sales.vortex:]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=vortex diff --git a/vortex-sqllogictest/slt/tpch/datafusion/plans/q1.slt.no b/vortex-sqllogictest/slt/tpch/datafusion/plans/q1.slt.no index 538137fa19e..3c54d256875 100644 --- a/vortex-sqllogictest/slt/tpch/datafusion/plans/q1.slt.no +++ b/vortex-sqllogictest/slt/tpch/datafusion/plans/q1.slt.no @@ -39,4 +39,5 @@ physical_plan 04)------AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] 05)--------RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 4), input_partitions=4 06)----------AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] -07)------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/lineitem.vortex:], [${WORK_DIR}/lineitem.vortex:], [${WORK_DIR}/lineitem.vortex:], [${WORK_DIR}/lineitem.vortex:]]}, projection=[l_extendedprice@5 * (1 - l_discount@6) as __common_expr_1, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus], file_type=vortex, predicate: l_shipdate@10 <= 1998-09-02 +07)------------ProjectionExec: expr=[l_extendedprice@1 * (1 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] +08)--------------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/lineitem.vortex:], [${WORK_DIR}/lineitem.vortex:], [${WORK_DIR}/lineitem.vortex:], [${WORK_DIR}/lineitem.vortex:]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=vortex, predicate: l_shipdate@10 <= 1998-09-02