From b3ce07cb70b32d49e99dd4973d49f0ca86c92210 Mon Sep 17 00:00:00 2001 From: patrick Date: Sat, 12 Mar 2022 15:11:55 -0700 Subject: [PATCH 01/12] Fixed parquet path partitioning when only selecting partitioned columns --- .../src/physical_plan/file_format/mod.rs | 28 ++++ .../src/physical_plan/file_format/parquet.rs | 146 +++++++++++++++--- datafusion/tests/path_partition.rs | 37 +++++ 3 files changed, 187 insertions(+), 24 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/mod.rs b/datafusion/src/physical_plan/file_format/mod.rs index 0e1e8596c7cb9..50af3d8cc3f9a 100644 --- a/datafusion/src/physical_plan/file_format/mod.rs +++ b/datafusion/src/physical_plan/file_format/mod.rs @@ -293,6 +293,34 @@ impl PartitionColumnProjector { } } + // Creates a RecordBatch with values from the partition_values. Used when no non-partition values are read + fn project_empty( + &mut self, + batch_size: usize, + partition_values: &[ScalarValue], + ) -> ArrowResult { + let expected_cols = self.projected_schema.fields().len(); + if expected_cols != self.projected_partition_indexes.len() { + return Err(ArrowError::SchemaError(format!( + "Unexepected number of partition values, expected {} but got {}", + expected_cols, + partition_values.len() + ))); + } + let mut cols = Vec::with_capacity(partition_values.len()); + for &(pidx, sidx) in &self.projected_partition_indexes { + cols.insert( + sidx, + create_dict_array( + &mut self.key_buffer_cache, + &partition_values[pidx], + batch_size, + ), + ) + } + RecordBatch::try_new(Arc::clone(&self.projected_schema), cols) + } + // Transform the batch read from the file by inserting the partitioning columns // to the right positions as deduced from `projected_schema` // - file_batch: batch read from the file, with internal projection applied diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index 2d23ca1c3ada5..f7f59003f18c2 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -236,32 +236,59 @@ impl ExecutionPlan for ParquetExec { let adapter = SchemaAdapter::new(self.base_config.file_schema.clone()); - let join_handle = task::spawn_blocking(move || { - if let Err(e) = read_partition( - object_store.as_ref(), - adapter, - partition_index, - &partition, - metrics, - &projection, - &pruning_predicate, - batch_size, - response_tx.clone(), - limit, - partition_col_proj, - ) { - println!( + let join_handle = if projection.is_empty() { + task::spawn_blocking(move || { + if let Err(e) = read_partition_no_file_columns( + object_store.as_ref(), + partition_index, + &partition, + metrics, + &pruning_predicate, + batch_size, + response_tx.clone(), + limit, + partition_col_proj, + ) { + println!( + "Parquet reader thread terminated due to error: {:?} for files: {:?}", + e, partition + ); + // Send the error back to the main thread. + // + // Ignore error sending (via `.ok()`) because that + // means the receiver has been torn down (and nothing + // cares about the errors anymore) + send_result(&response_tx, Err(e.into())).ok(); + } + }) + } else { + task::spawn_blocking(move || { + if let Err(e) = read_partition( + object_store.as_ref(), + adapter, + partition_index, + &partition, + metrics, + &projection, + &pruning_predicate, + batch_size, + response_tx.clone(), + limit, + partition_col_proj, + ) { + println!( "Parquet reader thread terminated due to error: {:?} for files: {:?}", e, partition ); - // Send the error back to the main thread. - // - // Ignore error sending (via `.ok()`) because that - // means the receiver has been torn down (and nothing - // cares about the errors anymore) - send_result(&response_tx, Err(e.into())).ok(); - } - }); + // Send the error back to the main thread. + // + // Ignore error sending (via `.ok()`) because that + // means the receiver has been torn down (and nothing + // cares about the errors anymore) + send_result(&response_tx, Err(e.into())).ok(); + } + }) + }; Ok(RecordBatchReceiverStream::create( &self.projected_schema, @@ -446,6 +473,78 @@ fn build_row_group_predicate( } } +#[allow(clippy::too_many_arguments)] +fn read_partition_no_file_columns( + object_store: &dyn ObjectStore, + partition_index: usize, + partition: &[PartitionedFile], + metrics: ExecutionPlanMetricsSet, + predicate_builder: &Option, + batch_size: usize, + response_tx: Sender>, + limit: Option, + mut partition_column_projector: PartitionColumnProjector, +) -> Result<()> { + let mut remaining_rows = limit.unwrap_or(usize::MAX); + for partitioned_file in partition { + let mut file_row_count = 0; + let file_metrics = ParquetFileMetrics::new( + partition_index, + &*partitioned_file.file_meta.path(), + &metrics, + ); + let object_reader = + object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; + let mut file_reader = + SerializedFileReader::new(ChunkObjectReader(object_reader))?; + if let Some(predicate_builder) = predicate_builder { + let row_group_predicate = build_row_group_predicate( + predicate_builder, + file_metrics, + file_reader.metadata().row_groups(), + ); + file_reader.filter_row_groups(&row_group_predicate); + } + + for i in 0..file_reader.num_row_groups() { + let row_group = file_reader.get_row_group(i)?; + let num_rows = row_group.metadata().num_rows(); + let num_rows = usize::try_from(num_rows).map_err(|e| { + DataFusionError::Execution(format!( + "Could not convert number of rows from i64 to usize: {}", + e + )) + })?; + file_row_count += num_rows; + if file_row_count >= remaining_rows { + file_row_count = remaining_rows; + remaining_rows = 0; + break; + } else { + remaining_rows -= file_row_count; + } + } + + while file_row_count > batch_size { + send_result( + &response_tx, + partition_column_projector + .project_empty(batch_size, &partitioned_file.partition_values), + )?; + file_row_count -= batch_size; + } + send_result( + &response_tx, + partition_column_projector + .project_empty(batch_size, &partitioned_file.partition_values), + )?; + if remaining_rows == 0 { + break; + } + } + Ok(()) +} + #[allow(clippy::too_many_arguments)] fn read_partition( object_store: &dyn ObjectStore, @@ -481,7 +580,6 @@ fn read_partition( ); file_reader.filter_row_groups(&row_group_predicate); } - let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(file_reader)); let adapted_projections = schema_adapter.map_projections(&arrow_reader.get_schema()?, projection)?; diff --git a/datafusion/tests/path_partition.rs b/datafusion/tests/path_partition.rs index 178e318775c9f..8028b824d77ef 100644 --- a/datafusion/tests/path_partition.rs +++ b/datafusion/tests/path_partition.rs @@ -37,6 +37,43 @@ use datafusion::{ }; use futures::{stream, StreamExt}; +#[tokio::test] +async fn parquet_distinct_partition_col() -> Result<()> { + let mut ctx = ExecutionContext::new(); + + register_partitioned_alltypes_parquet( + &mut ctx, + &[ + "year=2021/month=09/day=09/file.parquet", + "year=2021/month=10/day=09/file.parquet", + "year=2021/month=10/day=28/file.parquet", + ], + &["year", "month", "day"], + "", + "alltypes_plain.parquet", + ) + .await; + + let result = ctx + .sql("SELECT distinct year,month,day FROM t") + .await? + .collect() + .await?; + + let expected = vec![ + "+------+-------+-----+", + "| year | month | day |", + "+------+-------+-----+", + "| 2021 | 09 | 09 |", + "| 2021 | 10 | 09 |", + "| 2021 | 10 | 28 |", + "+------+-------+-----+", + ]; + assert_batches_sorted_eq!(expected, &result); + + Ok(()) +} + #[tokio::test] async fn csv_filter_with_file_col() -> Result<()> { let mut ctx = ExecutionContext::new(); From d61a1aa88061524765f0dbccf9a15c5f106afa99 Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 13 Mar 2022 20:39:25 -0600 Subject: [PATCH 02/12] Removed unnecesary row group pruning and file metrics --- .../src/physical_plan/file_format/parquet.rs | 54 +++++++------------ 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index f7f59003f18c2..b35a93aceafa0 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -18,6 +18,7 @@ //! Execution plan for reading Parquet files use futures::{StreamExt, TryStreamExt}; +use log::error; use std::fmt; use std::fs; use std::path::Path; @@ -240,16 +241,13 @@ impl ExecutionPlan for ParquetExec { task::spawn_blocking(move || { if let Err(e) = read_partition_no_file_columns( object_store.as_ref(), - partition_index, &partition, - metrics, - &pruning_predicate, batch_size, response_tx.clone(), limit, partition_col_proj, ) { - println!( + error!( "Parquet reader thread terminated due to error: {:?} for files: {:?}", e, partition ); @@ -276,7 +274,7 @@ impl ExecutionPlan for ParquetExec { limit, partition_col_proj, ) { - println!( + error!( "Parquet reader thread terminated due to error: {:?} for files: {:?}", e, partition ); @@ -473,39 +471,21 @@ fn build_row_group_predicate( } } -#[allow(clippy::too_many_arguments)] fn read_partition_no_file_columns( object_store: &dyn ObjectStore, - partition_index: usize, partition: &[PartitionedFile], - metrics: ExecutionPlanMetricsSet, - predicate_builder: &Option, batch_size: usize, response_tx: Sender>, - limit: Option, + mut limit: Option, mut partition_column_projector: PartitionColumnProjector, ) -> Result<()> { - let mut remaining_rows = limit.unwrap_or(usize::MAX); + for partitioned_file in partition { let mut file_row_count = 0; - let file_metrics = ParquetFileMetrics::new( - partition_index, - &*partitioned_file.file_meta.path(), - &metrics, - ); let object_reader = object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; - let mut file_reader = + let file_reader = SerializedFileReader::new(ChunkObjectReader(object_reader))?; - if let Some(predicate_builder) = predicate_builder { - let row_group_predicate = build_row_group_predicate( - predicate_builder, - file_metrics, - file_reader.metadata().row_groups(), - ); - file_reader.filter_row_groups(&row_group_predicate); - } - for i in 0..file_reader.num_row_groups() { let row_group = file_reader.get_row_group(i)?; let num_rows = row_group.metadata().num_rows(); @@ -516,12 +496,15 @@ fn read_partition_no_file_columns( )) })?; file_row_count += num_rows; + let remaining_rows = limit.unwrap_or(usize::MAX); if file_row_count >= remaining_rows { file_row_count = remaining_rows; - remaining_rows = 0; + limit = Some(0); break; } else { - remaining_rows -= file_row_count; + if let Some(remaining_limt) = &mut limit{ + *remaining_limt -= file_row_count; + } } } @@ -533,12 +516,15 @@ fn read_partition_no_file_columns( )?; file_row_count -= batch_size; } - send_result( - &response_tx, - partition_column_projector - .project_empty(batch_size, &partitioned_file.partition_values), - )?; - if remaining_rows == 0 { + if file_row_count != 0{ + send_result( + &response_tx, + partition_column_projector + .project_empty(batch_size, &partitioned_file.partition_values), + )?; + } + + if limit == Some(0) { break; } } From dcc2cd07b4e07bc7dbb4248427ddb0c702343827 Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 13 Mar 2022 20:44:08 -0600 Subject: [PATCH 03/12] Ran cargo fmt --- .../src/physical_plan/file_format/parquet.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index b35a93aceafa0..b46625a9b67fc 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -479,13 +479,11 @@ fn read_partition_no_file_columns( mut limit: Option, mut partition_column_projector: PartitionColumnProjector, ) -> Result<()> { - for partitioned_file in partition { let mut file_row_count = 0; let object_reader = object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; - let file_reader = - SerializedFileReader::new(ChunkObjectReader(object_reader))?; + let file_reader = SerializedFileReader::new(ChunkObjectReader(object_reader))?; for i in 0..file_reader.num_row_groups() { let row_group = file_reader.get_row_group(i)?; let num_rows = row_group.metadata().num_rows(); @@ -501,10 +499,8 @@ fn read_partition_no_file_columns( file_row_count = remaining_rows; limit = Some(0); break; - } else { - if let Some(remaining_limt) = &mut limit{ - *remaining_limt -= file_row_count; - } + } else if let Some(remaining_limt) = &mut limit { + *remaining_limt -= file_row_count; } } @@ -516,14 +512,14 @@ fn read_partition_no_file_columns( )?; file_row_count -= batch_size; } - if file_row_count != 0{ + if file_row_count != 0 { send_result( &response_tx, partition_column_projector .project_empty(batch_size, &partitioned_file.partition_values), )?; } - + if limit == Some(0) { break; } From e3aa7dd5c9ae1ea16e61bedaddbb031fe5d4a1a0 Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Mar 2022 20:52:35 -0600 Subject: [PATCH 04/12] Switched from row group level metadata to file level metadata to determine number of rows to emit --- .../src/physical_plan/file_format/parquet.rs | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index b46625a9b67fc..6da581978ad9e 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -238,6 +238,7 @@ impl ExecutionPlan for ParquetExec { let adapter = SchemaAdapter::new(self.base_config.file_schema.clone()); let join_handle = if projection.is_empty() { + println!("Empty projection. using read_partition_no_file_columns"); task::spawn_blocking(move || { if let Err(e) = read_partition_no_file_columns( object_store.as_ref(), @@ -260,6 +261,7 @@ impl ExecutionPlan for ParquetExec { } }) } else { + println!("Non-empty projection. using read_partition"); task::spawn_blocking(move || { if let Err(e) = read_partition( object_store.as_ref(), @@ -480,39 +482,27 @@ fn read_partition_no_file_columns( mut partition_column_projector: PartitionColumnProjector, ) -> Result<()> { for partitioned_file in partition { - let mut file_row_count = 0; - let object_reader = - object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; + let object_reader = object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; let file_reader = SerializedFileReader::new(ChunkObjectReader(object_reader))?; - for i in 0..file_reader.num_row_groups() { - let row_group = file_reader.get_row_group(i)?; - let num_rows = row_group.metadata().num_rows(); - let num_rows = usize::try_from(num_rows).map_err(|e| { - DataFusionError::Execution(format!( - "Could not convert number of rows from i64 to usize: {}", - e - )) - })?; - file_row_count += num_rows; - let remaining_rows = limit.unwrap_or(usize::MAX); - if file_row_count >= remaining_rows { - file_row_count = remaining_rows; - limit = Some(0); - break; - } else if let Some(remaining_limt) = &mut limit { - *remaining_limt -= file_row_count; - } + let mut file_rows: usize = file_reader.metadata().file_metadata().num_rows().try_into().expect("Row count should always be greater than or equal to 0"); + let remaining_rows = limit.unwrap_or(usize::MAX); + if file_rows >= remaining_rows{ + file_rows = remaining_rows; + limit = Some(0); + }else if let Some(remaining_limit) = &mut limit{ + *remaining_limit -= file_rows; } - - while file_row_count > batch_size { + + while file_rows > batch_size { send_result( &response_tx, partition_column_projector .project_empty(batch_size, &partitioned_file.partition_values), )?; - file_row_count -= batch_size; + file_rows -= batch_size; } - if file_row_count != 0 { + + if file_rows != 0 { send_result( &response_tx, partition_column_projector From 5bec5e7ef50bcb29c61ad1a7ec43d5c0c508d88e Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Mar 2022 20:57:57 -0600 Subject: [PATCH 05/12] Reworked task spawning in ParquetExec::execute --- .../src/physical_plan/file_format/parquet.rs | 68 ++++++++----------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index 6da581978ad9e..349c194645aff 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -237,33 +237,18 @@ impl ExecutionPlan for ParquetExec { let adapter = SchemaAdapter::new(self.base_config.file_schema.clone()); - let join_handle = if projection.is_empty() { - println!("Empty projection. using read_partition_no_file_columns"); - task::spawn_blocking(move || { - if let Err(e) = read_partition_no_file_columns( + let join_handle = task::spawn_blocking(move || { + let res = if projection.is_empty() { + read_partition_no_file_columns( object_store.as_ref(), &partition, batch_size, response_tx.clone(), limit, partition_col_proj, - ) { - error!( - "Parquet reader thread terminated due to error: {:?} for files: {:?}", - e, partition - ); - // Send the error back to the main thread. - // - // Ignore error sending (via `.ok()`) because that - // means the receiver has been torn down (and nothing - // cares about the errors anymore) - send_result(&response_tx, Err(e.into())).ok(); - } - }) - } else { - println!("Non-empty projection. using read_partition"); - task::spawn_blocking(move || { - if let Err(e) = read_partition( + ) + } else { + read_partition( object_store.as_ref(), adapter, partition_index, @@ -275,20 +260,22 @@ impl ExecutionPlan for ParquetExec { response_tx.clone(), limit, partition_col_proj, - ) { - error!( + ) + }; + + if let Err(e) = res { + error!( "Parquet reader thread terminated due to error: {:?} for files: {:?}", e, partition ); - // Send the error back to the main thread. - // - // Ignore error sending (via `.ok()`) because that - // means the receiver has been torn down (and nothing - // cares about the errors anymore) - send_result(&response_tx, Err(e.into())).ok(); - } - }) - }; + // Send the error back to the main thread. + // + // Ignore error sending (via `.ok()`) because that + // means the receiver has been torn down (and nothing + // cares about the errors anymore) + send_result(&response_tx, Err(e.into())).ok(); + } + }); Ok(RecordBatchReceiverStream::create( &self.projected_schema, @@ -482,17 +469,23 @@ fn read_partition_no_file_columns( mut partition_column_projector: PartitionColumnProjector, ) -> Result<()> { for partitioned_file in partition { - let object_reader = object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; + let object_reader = + object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; let file_reader = SerializedFileReader::new(ChunkObjectReader(object_reader))?; - let mut file_rows: usize = file_reader.metadata().file_metadata().num_rows().try_into().expect("Row count should always be greater than or equal to 0"); + let mut file_rows: usize = file_reader + .metadata() + .file_metadata() + .num_rows() + .try_into() + .expect("Row count should always be greater than or equal to 0"); let remaining_rows = limit.unwrap_or(usize::MAX); - if file_rows >= remaining_rows{ + if file_rows >= remaining_rows { file_rows = remaining_rows; limit = Some(0); - }else if let Some(remaining_limit) = &mut limit{ + } else if let Some(remaining_limit) = &mut limit { *remaining_limit -= file_rows; } - + while file_rows > batch_size { send_result( &response_tx, @@ -501,7 +494,6 @@ fn read_partition_no_file_columns( )?; file_rows -= batch_size; } - if file_rows != 0 { send_result( &response_tx, From 48a3627f5cfd0c0d53028978fec541350ee88f08 Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Mar 2022 21:21:18 -0600 Subject: [PATCH 06/12] Changed index based partition column generating to iterator version --- .../src/physical_plan/file_format/mod.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/mod.rs b/datafusion/src/physical_plan/file_format/mod.rs index 50af3d8cc3f9a..de0b62c2cfa63 100644 --- a/datafusion/src/physical_plan/file_format/mod.rs +++ b/datafusion/src/physical_plan/file_format/mod.rs @@ -307,17 +307,19 @@ impl PartitionColumnProjector { partition_values.len() ))); } - let mut cols = Vec::with_capacity(partition_values.len()); - for &(pidx, sidx) in &self.projected_partition_indexes { - cols.insert( - sidx, + //The destination index is not needed. Since there are no non-partition columns it will simply be equivalent to + //the index that would be provided by .enumerate() + let cols = self + .projected_partition_indexes + .iter() + .map(|(pidx, _)| { create_dict_array( &mut self.key_buffer_cache, - &partition_values[pidx], + &partition_values[*pidx], batch_size, - ), - ) - } + ) + }) + .collect(); RecordBatch::try_new(Arc::clone(&self.projected_schema), cols) } @@ -340,7 +342,6 @@ impl PartitionColumnProjector { file_batch.columns().len() ))); } - let mut cols = file_batch.columns().to_vec(); for &(pidx, sidx) in &self.projected_partition_indexes { cols.insert( From a5f0404103e7930db5688a99379b51b8ede09557 Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 24 Mar 2022 20:19:04 -0600 Subject: [PATCH 07/12] Moved limit unwrap outside of loop --- .../src/physical_plan/file_format/parquet.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index 5163879075dca..3bc07f1dfd643 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -453,10 +453,11 @@ fn read_partition_no_file_columns( partition: &[PartitionedFile], batch_size: usize, response_tx: Sender>, - mut limit: Option, + limit: Option, mut partition_column_projector: PartitionColumnProjector, ) -> Result<()> { use parquet::file::reader::FileReader; + let mut limit = limit.unwrap_or(usize::MAX); for partitioned_file in partition { let object_reader = object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; @@ -467,15 +468,9 @@ fn read_partition_no_file_columns( .num_rows() .try_into() .expect("Row count should always be greater than or equal to 0"); - let remaining_rows = limit.unwrap_or(usize::MAX); - if file_rows >= remaining_rows { - file_rows = remaining_rows; - limit = Some(0); - } else if let Some(remaining_limit) = &mut limit { - *remaining_limit -= file_rows; - } - - while file_rows > batch_size { + file_rows = limit.min(file_rows); + limit -= file_rows; + while file_rows >= batch_size { send_result( &response_tx, partition_column_projector @@ -490,8 +485,7 @@ fn read_partition_no_file_columns( .project_empty(batch_size, &partitioned_file.partition_values), )?; } - - if limit == Some(0) { + if limit == 0 { break; } } From d331f8f83cd5bb27675d9a43d6c0edbc74c63eab Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 27 Mar 2022 20:57:53 -0600 Subject: [PATCH 08/12] Fixed bug about number of rows emitted when querying only partition columns and reuse partition record batch --- .../src/physical_plan/file_format/mod.rs | 2 +- .../src/physical_plan/file_format/parquet.rs | 56 ++++++++++--------- datafusion/tests/path_partition.rs | 20 +++++++ 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/datafusion/src/physical_plan/file_format/mod.rs b/datafusion/src/physical_plan/file_format/mod.rs index c6b97e6f2d6f0..733d06af2bb80 100644 --- a/datafusion/src/physical_plan/file_format/mod.rs +++ b/datafusion/src/physical_plan/file_format/mod.rs @@ -294,7 +294,7 @@ impl PartitionColumnProjector { } // Creates a RecordBatch with values from the partition_values. Used when no non-partition values are read - fn project_empty( + fn project_from_size( &mut self, batch_size: usize, partition_values: &[ScalarValue], diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs index 3bc07f1dfd643..1eb26e838e657 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/src/physical_plan/file_format/parquet.rs @@ -458,36 +458,42 @@ fn read_partition_no_file_columns( ) -> Result<()> { use parquet::file::reader::FileReader; let mut limit = limit.unwrap_or(usize::MAX); + for partitioned_file in partition { + if limit == 0 { + break; + } let object_reader = object_store.file_reader(partitioned_file.file_meta.sized_file.clone())?; let file_reader = SerializedFileReader::new(ChunkObjectReader(object_reader))?; - let mut file_rows: usize = file_reader - .metadata() - .file_metadata() - .num_rows() - .try_into() - .expect("Row count should always be greater than or equal to 0"); - file_rows = limit.min(file_rows); - limit -= file_rows; - while file_rows >= batch_size { - send_result( - &response_tx, - partition_column_projector - .project_empty(batch_size, &partitioned_file.partition_values), - )?; - file_rows -= batch_size; - } - if file_rows != 0 { - send_result( - &response_tx, - partition_column_projector - .project_empty(batch_size, &partitioned_file.partition_values), - )?; - } - if limit == 0 { - break; + let mut num_rows = usize::min(limit, file_reader + .metadata() + .file_metadata() + .num_rows() + .try_into() + .expect("Row count should always be greater than or equal to 0 and less than usize::MAX")); + limit -= num_rows; + + let partition_batch = partition_column_projector + .project_from_size(batch_size, &partitioned_file.partition_values) + .map_err(|e| { + let err_msg = + format!("Error reading batch from {}: {}", partitioned_file, e); + if let Err(send_err) = send_result( + &response_tx, + Err(ArrowError::ParquetError(err_msg.clone())), + ) { + return send_err; + } + DataFusionError::Execution(err_msg) + })?; + + while num_rows > batch_size { + send_result(&response_tx, Ok(partition_batch.clone()))?; + num_rows -= batch_size; } + let residual_batch = partition_batch.slice(0, num_rows); + send_result(&response_tx, Ok(residual_batch))?; } Ok(()) } diff --git a/datafusion/tests/path_partition.rs b/datafusion/tests/path_partition.rs index 7f8b5e167fe5d..1dcee862952a7 100644 --- a/datafusion/tests/path_partition.rs +++ b/datafusion/tests/path_partition.rs @@ -74,6 +74,26 @@ async fn parquet_distinct_partition_col() -> Result<()> { ]; assert_batches_sorted_eq!(expected, &result); + let actual_row_count: usize = ctx + .sql("SELECT id from t") + .await? + .collect() + .await? + .into_iter() + .map(|batch| batch.num_rows()) + .sum(); + + let partition_row_count: usize = ctx + .sql("SELECT year from t") + .await? + .collect() + .await? + .into_iter() + .map(|batch| batch.num_rows()) + .sum(); + + assert_eq!(actual_row_count, partition_row_count); + Ok(()) } From ffac6ea8f87590ba20cfb757e6d23f1acb31141b Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 3 Apr 2022 12:15:34 -0600 Subject: [PATCH 09/12] Added limit logic tests for partitioned hive partitioned parquet file --- datafusion/tests/path_partition.rs | 65 ++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/datafusion/tests/path_partition.rs b/datafusion/tests/path_partition.rs index 1dcee862952a7..1ee8d75389164 100644 --- a/datafusion/tests/path_partition.rs +++ b/datafusion/tests/path_partition.rs @@ -38,6 +38,7 @@ use datafusion::{ prelude::SessionContext, test_util::{self, arrow_test_data, parquet_test_data}, }; +use datafusion_common::ScalarValue; use futures::{stream, StreamExt}; #[tokio::test] @@ -56,7 +57,7 @@ async fn parquet_distinct_partition_col() -> Result<()> { "alltypes_plain.parquet", ) .await; - + //Test that only selecting partition columns is possible let result = ctx .sql("SELECT distinct year,month,day FROM t") .await? @@ -73,7 +74,7 @@ async fn parquet_distinct_partition_col() -> Result<()> { "+------+-------+-----+", ]; assert_batches_sorted_eq!(expected, &result); - + //Test that the number of rows returned by partition column scan and actually reading the parquet file are the same let actual_row_count: usize = ctx .sql("SELECT id from t") .await? @@ -91,9 +92,67 @@ async fn parquet_distinct_partition_col() -> Result<()> { .into_iter() .map(|batch| batch.num_rows()) .sum(); - assert_eq!(actual_row_count, partition_row_count); + //Test limit logic. 3 test cases + //1. limit is contained within a single partition with leftover rows + //2. limit is contained within a single partition without leftover rows + //3. limit is not contained within a single partition + //The id column is included to ensure that the parquet file is actually scanned. + let results = ctx + .sql("SELECT COUNT(*) as num_rows_per_month, month, MAX(id) from t group by month order by num_rows_per_month desc") + .await? + .collect() + .await?; + + let mut max_limit =match ScalarValue::try_from_array(results[0].column(0), 0)?{ + ScalarValue::UInt64(Some(count)) => count, + s => panic!("Expected count as Int64 found {}", s), + }; + + max_limit += 1; + let last_batch = results.last().expect("There shouled be at least one record batch returned"); + let last_row_idx = last_batch.num_rows() - 1; + let mut min_limit = match ScalarValue::try_from_array(last_batch.column(0), last_row_idx)?{ + ScalarValue::UInt64(Some(count)) => count, + s => panic!("Expected count as Int64 found {}", s), + }; + + min_limit -= 1; + + let sql_cross_partition_boundary = format!("SELECT month FROM t limit {}", max_limit); + let resulting_limit:u64 = ctx + .sql(sql_cross_partition_boundary.as_str()) + .await? + .collect() + .await?.into_iter().map(|r| r.num_rows() as u64) + .sum(); + + assert_eq!(max_limit, resulting_limit); + + let sql_within_partition_boundary = format!("SELECT month from t limit {}", min_limit); + let resulting_limit: u64 = ctx.sql(sql_within_partition_boundary.as_str()) + .await? + .collect() + .await?.into_iter().map(|r| r.num_rows() as u64) + .sum(); + + assert_eq!(min_limit, resulting_limit); + + let month = match ScalarValue::try_from_array(results[0].column(1), 0)?{ + ScalarValue::Utf8(Some(month)) => month, + s => panic!("Expected count as Int64 found {}", s), + }; + + let sql_on_partition_boundary = format!("SELECT month from t where month = '{}' LIMIT {}", month, max_limit - 1); + let resulting_limit: u64 = ctx.sql(sql_on_partition_boundary.as_str()) + .await? + .collect() + .await? + .into_iter().map(|r| r.num_rows() as u64) + .sum(); + let partition_row_count = max_limit - 1; + assert_eq!(partition_row_count, resulting_limit); Ok(()) } From c420885fc86ddf6636b173fdae8817413655983c Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 3 Apr 2022 12:20:33 -0600 Subject: [PATCH 10/12] Formatted code --- datafusion/core/tests/path_partition.rs | 49 ++++++++++++++++--------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs index 033d1fb596192..c67a7d80c5f42 100644 --- a/datafusion/core/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -105,51 +105,66 @@ async fn parquet_distinct_partition_col() -> Result<()> { .collect() .await?; - let mut max_limit =match ScalarValue::try_from_array(results[0].column(0), 0)?{ + let mut max_limit = match ScalarValue::try_from_array(results[0].column(0), 0)? { ScalarValue::UInt64(Some(count)) => count, s => panic!("Expected count as Int64 found {}", s), }; max_limit += 1; - let last_batch = results.last().expect("There shouled be at least one record batch returned"); + let last_batch = results + .last() + .expect("There shouled be at least one record batch returned"); let last_row_idx = last_batch.num_rows() - 1; - let mut min_limit = match ScalarValue::try_from_array(last_batch.column(0), last_row_idx)?{ - ScalarValue::UInt64(Some(count)) => count, - s => panic!("Expected count as Int64 found {}", s), - }; + let mut min_limit = + match ScalarValue::try_from_array(last_batch.column(0), last_row_idx)? { + ScalarValue::UInt64(Some(count)) => count, + s => panic!("Expected count as Int64 found {}", s), + }; min_limit -= 1; - + let sql_cross_partition_boundary = format!("SELECT month FROM t limit {}", max_limit); - let resulting_limit:u64 = ctx + let resulting_limit: u64 = ctx .sql(sql_cross_partition_boundary.as_str()) .await? .collect() - .await?.into_iter().map(|r| r.num_rows() as u64) + .await? + .into_iter() + .map(|r| r.num_rows() as u64) .sum(); - + assert_eq!(max_limit, resulting_limit); - let sql_within_partition_boundary = format!("SELECT month from t limit {}", min_limit); - let resulting_limit: u64 = ctx.sql(sql_within_partition_boundary.as_str()) + let sql_within_partition_boundary = + format!("SELECT month from t limit {}", min_limit); + let resulting_limit: u64 = ctx + .sql(sql_within_partition_boundary.as_str()) .await? .collect() - .await?.into_iter().map(|r| r.num_rows() as u64) + .await? + .into_iter() + .map(|r| r.num_rows() as u64) .sum(); assert_eq!(min_limit, resulting_limit); - let month = match ScalarValue::try_from_array(results[0].column(1), 0)?{ + let month = match ScalarValue::try_from_array(results[0].column(1), 0)? { ScalarValue::Utf8(Some(month)) => month, s => panic!("Expected count as Int64 found {}", s), }; - let sql_on_partition_boundary = format!("SELECT month from t where month = '{}' LIMIT {}", month, max_limit - 1); - let resulting_limit: u64 = ctx.sql(sql_on_partition_boundary.as_str()) + let sql_on_partition_boundary = format!( + "SELECT month from t where month = '{}' LIMIT {}", + month, + max_limit - 1 + ); + let resulting_limit: u64 = ctx + .sql(sql_on_partition_boundary.as_str()) .await? .collect() .await? - .into_iter().map(|r| r.num_rows() as u64) + .into_iter() + .map(|r| r.num_rows() as u64) .sum(); let partition_row_count = max_limit - 1; assert_eq!(partition_row_count, resulting_limit); From 2fb3596cb6efbee759614294d50ab33000d74244 Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 3 Apr 2022 15:18:10 -0600 Subject: [PATCH 11/12] Fixed clippy lint --- datafusion/core/tests/path_partition.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs index c67a7d80c5f42..d7abf8b76a1c3 100644 --- a/datafusion/core/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -46,7 +46,7 @@ async fn parquet_distinct_partition_col() -> Result<()> { let mut ctx = SessionContext::new(); register_partitioned_alltypes_parquet( - &mut ctx, + &ctx, &[ "year=2021/month=09/day=09/file.parquet", "year=2021/month=10/day=09/file.parquet", From 6c4edd0b7a61bff6743a7cb1210128f29f612502 Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 3 Apr 2022 15:46:04 -0600 Subject: [PATCH 12/12] Fixed other clippy lint --- datafusion/core/tests/path_partition.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs index d7abf8b76a1c3..2e40005528577 100644 --- a/datafusion/core/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -43,7 +43,7 @@ use futures::{stream, StreamExt}; #[tokio::test] async fn parquet_distinct_partition_col() -> Result<()> { - let mut ctx = SessionContext::new(); + let ctx = SessionContext::new(); register_partitioned_alltypes_parquet( &ctx,