From 370b6157d33890183ae95ff934d52b9a46caf8ac Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 6 Sep 2022 21:52:38 +0200 Subject: [PATCH 1/2] Don't scan first column on empty projection --- datafusion/core/src/physical_plan/memory.rs | 13 +++++++++-- .../optimizer/src/projection_push_down.rs | 23 +++++++------------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/datafusion/core/src/physical_plan/memory.rs b/datafusion/core/src/physical_plan/memory.rs index d2dbe0d738c39..b0134c4cdbc51 100644 --- a/datafusion/core/src/physical_plan/memory.rs +++ b/datafusion/core/src/physical_plan/memory.rs @@ -28,9 +28,9 @@ use super::{ RecordBatchStream, SendableRecordBatchStream, Statistics, }; use crate::error::Result; -use arrow::datatypes::SchemaRef; +use arrow::datatypes::{Schema, SchemaRef}; use arrow::error::Result as ArrowResult; -use arrow::record_batch::RecordBatch; +use arrow::record_batch::{RecordBatch, RecordBatchOptions}; use crate::execution::context::TaskContext; use datafusion_common::DataFusionError; @@ -195,6 +195,15 @@ impl Stream for MemoryStream { // return just the columns requested let batch = match self.projection.as_ref() { + Some(columns) if columns.is_empty() => { + let mut options = RecordBatchOptions::default(); + options.row_count = Some(batch.num_rows()); + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &options, + )? + } Some(columns) => batch.project(columns)?, None => batch.clone(), }; diff --git a/datafusion/optimizer/src/projection_push_down.rs b/datafusion/optimizer/src/projection_push_down.rs index 79364eb1f109b..167e6a61d5d4c 100644 --- a/datafusion/optimizer/src/projection_push_down.rs +++ b/datafusion/optimizer/src/projection_push_down.rs @@ -93,21 +93,14 @@ fn get_projected_schema( .filter_map(ArrowResult::ok) .collect(); - if projection.is_empty() { - if has_projection && !schema.fields().is_empty() { - // Ensure that we are reading at least one column from the table in case the query - // does not reference any columns directly such as "SELECT COUNT(1) FROM table", - // except when the table is empty (no column) - projection.insert(0); - } else { - // for table scan without projection, we default to return all columns - projection = schema - .fields() - .iter() - .enumerate() - .map(|(i, _)| i) - .collect::>(); - } + if projection.is_empty() && !has_projection { + // for table scan without projection, we default to return all columns + projection = schema + .fields() + .iter() + .enumerate() + .map(|(i, _)| i) + .collect::>(); } // create the projected schema From 6f7edca51d0a3ac2ae1766edee6302d3c3d952a0 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 6 Sep 2022 21:57:39 +0200 Subject: [PATCH 2/2] Fix test --- datafusion/core/tests/sql/json.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/tests/sql/json.rs b/datafusion/core/tests/sql/json.rs index 3293ecf43f5a5..fd3e81beb3562 100644 --- a/datafusion/core/tests/sql/json.rs +++ b/datafusion/core/tests/sql/json.rs @@ -88,7 +88,7 @@ async fn json_explain() { "logical_plan", "Projection: #COUNT(UInt8(1))\ \n Aggregate: groupBy=[[]], aggr=[[COUNT(UInt8(1))]]\ - \n TableScan: t1 projection=[a]", + \n TableScan: t1 projection=[]", ], vec![ "physical_plan",