From 9e5c51d22b4018ce7307a5dd595531e9b3ec2f10 Mon Sep 17 00:00:00 2001 From: Dan Harris Date: Wed, 6 Apr 2022 11:00:52 +0100 Subject: [PATCH 1/8] Handle merged schemas in parquet pruning --- .../src/physical_plan/file_format/parquet.rs | 154 +++++++++++++++--- 1 file changed, 129 insertions(+), 25 deletions(-) diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index d6800af9f6011..c82a686b31e2e 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -366,7 +366,7 @@ macro_rules! get_statistic { // Extract the min or max value calling `func` or `bytes_func` on the ParquetStatistics as appropriate macro_rules! get_min_max_values { ($self:expr, $column:expr, $func:ident, $bytes_func:ident) => {{ - let (column_index, field) = + let (_column_index, field) = if let Some((v, f)) = $self.parquet_schema.column_with_name(&$column.name) { (v, f) } else { @@ -379,8 +379,10 @@ macro_rules! get_min_max_values { let null_scalar: ScalarValue = data_type.try_into().ok()?; $self.row_group_metadata - .column(column_index) - .statistics() + .columns() + .iter() + .find(|c| c.column_descr().name() == &$column.name) + .and_then(|c| c.statistics()) .map(|stats| get_statistic!(stats, $func, $bytes_func)) .flatten() // column either didn't have statistics at all or didn't have min/max values @@ -392,21 +394,19 @@ macro_rules! get_min_max_values { // Extract the null count value on the ParquetStatistics macro_rules! get_null_count_values { ($self:expr, $column:expr) => {{ - let column_index = - if let Some((v, _)) = $self.parquet_schema.column_with_name(&$column.name) { - v - } else { - // Named column was not present - return None; - }; - let value = ScalarValue::UInt64( - $self + if let Some(col) = $self .row_group_metadata - .column(column_index) - .statistics() - .map(|s| s.null_count()), + .columns() + .iter() + .find(|c| c.column_descr().name() == &$column.name) + { + col.statistics().map(|s| s.null_count()) + } else { + Some($self.row_group_metadata.num_rows() as u64) + }, ); + Some(value.to_array()) }}; } @@ -664,6 +664,7 @@ mod tests { array::{Int64Array, Int8Array, StringArray}, datatypes::{DataType, Field}, }; + use datafusion_expr::{col, lit}; use futures::StreamExt; use parquet::{ arrow::ArrowWriter, @@ -684,6 +685,7 @@ mod tests { batches: Vec, projection: Option>, schema: Option, + predicate: Option, ) -> Result> { // When vec is dropped, temp files are deleted let files: Vec<_> = batches @@ -735,7 +737,7 @@ mod tests { limit: None, table_partition_cols: vec![], }, - None, + predicate, ); let session_ctx = SessionContext::new(); @@ -785,7 +787,7 @@ mod tests { let batch3 = add_to_batch(&batch1, "c3", c3); // read/write them files: - let read = round_trip_to_parquet(vec![batch1, batch2, batch3], None, None) + let read = round_trip_to_parquet(vec![batch1, batch2, batch3], None, None, None) .await .unwrap(); let expected = vec![ @@ -826,7 +828,7 @@ mod tests { let batch2 = create_batch(vec![("c3", c3), ("c2", c2), ("c1", c1)]); // read/write them files: - let read = round_trip_to_parquet(vec![batch1, batch2], None, None) + let read = round_trip_to_parquet(vec![batch1, batch2], None, None, None) .await .unwrap(); let expected = vec![ @@ -860,7 +862,7 @@ mod tests { let batch2 = create_batch(vec![("c3", c3), ("c2", c2)]); // read/write them files: - let read = round_trip_to_parquet(vec![batch1, batch2], None, None) + let read = round_trip_to_parquet(vec![batch1, batch2], None, None, None) .await .unwrap(); let expected = vec![ @@ -878,6 +880,39 @@ mod tests { assert_batches_sorted_eq!(expected, &read); } + #[tokio::test] + async fn evolved_schema_intersection_filter() { + let c1: ArrayRef = + Arc::new(StringArray::from(vec![Some("Foo"), None, Some("bar")])); + + let c2: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), Some(2), None])); + + let c3: ArrayRef = Arc::new(Int8Array::from(vec![Some(10), Some(20), None])); + + // batch1: c1(string), c2(int64), c3(int8) + let batch1 = create_batch(vec![("c1", c1), ("c3", c3.clone())]); + + // batch2: c3(int8), c2(int64), c1(string) + let batch2 = create_batch(vec![("c3", c3), ("c2", c2)]); + + let filter = col("c2").eq(lit(0_i64)); + + // read/write them files: + let read = round_trip_to_parquet(vec![batch1, batch2], None, None, Some(filter)) + .await + .unwrap(); + let expected = vec![ + "+-----+----+----+", + "| c1 | c3 | c2 |", + "+-----+----+----+", + "| Foo | 10 | |", + "| | 20 | |", + "| bar | | |", + "+-----+----+----+", + ]; + assert_batches_sorted_eq!(expected, &read); + } + #[tokio::test] async fn evolved_schema_projection() { let c1: ArrayRef = @@ -901,9 +936,10 @@ mod tests { let batch2 = create_batch(vec![("c3", c3), ("c2", c2), ("c1", c1), ("c4", c4)]); // read/write them files: - let read = round_trip_to_parquet(vec![batch1, batch2], Some(vec![0, 3]), None) - .await - .unwrap(); + let read = + round_trip_to_parquet(vec![batch1, batch2], Some(vec![0, 3]), None, None) + .await + .unwrap(); let expected = vec![ "+-----+-----+", "| c1 | c4 |", @@ -919,6 +955,70 @@ mod tests { assert_batches_sorted_eq!(expected, &read); } + #[tokio::test] + async fn evolved_schema_filter() { + let c1: ArrayRef = + Arc::new(StringArray::from(vec![Some("Foo"), None, Some("bar")])); + + let c2: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), Some(2), None])); + + let c3: ArrayRef = Arc::new(Int8Array::from(vec![Some(10), Some(20), None])); + + // batch1: c1(string), c2(int64), c3(int8) + let batch1 = create_batch(vec![ + ("c1", c1.clone()), + ("c2", c2.clone()), + ("c3", c3.clone()), + ]); + + // batch2: c3(int8), c2(int64), c1(string) + let batch2 = create_batch(vec![("c3", c3), ("c2", c2), ("c1", c1)]); + + let filter = col("c3").eq(lit(0_i8)); + + // read/write them files: + let read = round_trip_to_parquet(vec![batch1, batch2], None, None, Some(filter)) + .await + .unwrap(); + + // Predicate should prune all row groups + assert_eq!(read.len(), 0); + } + + #[tokio::test] + async fn evolved_schema_disjoint_schema_filter() { + let c1: ArrayRef = + Arc::new(StringArray::from(vec![Some("Foo"), None, Some("bar")])); + + let c2: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), Some(2), None])); + + // let c3: ArrayRef = Arc::new(Int8Array::from(vec![Some(10), Some(20), None])); + + // batch1: c1(string), c2(int64), c3(int8) + let batch1 = create_batch(vec![("c1", c1.clone())]); + + // batch2: c3(int8), c2(int64), c1(string) + let batch2 = create_batch(vec![("c2", c2)]); + + let filter = col("c2").eq(lit(0_i64)); + + // read/write them files: + let read = round_trip_to_parquet(vec![batch1, batch2], None, None, Some(filter)) + .await + .unwrap(); + + let expected = vec![ + "+-----+----+", + "| c1 | c2 |", + "+-----+----+", + "| Foo | |", + "| | |", + "| bar | |", + "+-----+----+", + ]; + assert_batches_sorted_eq!(expected, &read); + } + #[tokio::test] async fn evolved_schema_incompatible_types() { let c1: ArrayRef = @@ -948,9 +1048,13 @@ mod tests { ]); // read/write them files: - let read = - round_trip_to_parquet(vec![batch1, batch2], None, Some(Arc::new(schema))) - .await; + let read = round_trip_to_parquet( + vec![batch1, batch2], + None, + Some(Arc::new(schema)), + None, + ) + .await; assert_contains!(read.unwrap_err().to_string(), "Execution error: Failed to map column projection for field c3. Incompatible data types Float32 and Int8"); } From 04d9deccabbafce013710269088e2e413669b05e Mon Sep 17 00:00:00 2001 From: Dan Harris Date: Wed, 6 Apr 2022 15:55:32 +0100 Subject: [PATCH 2/8] Handle merged schemas in ListingTable stats collection --- .../core/src/datasource/file_format/avro.rs | 8 +- .../core/src/datasource/file_format/csv.rs | 8 +- .../core/src/datasource/file_format/json.rs | 11 +- .../core/src/datasource/file_format/mod.rs | 6 +- .../src/datasource/file_format/parquet.rs | 158 +++++++++++++++--- .../core/src/datasource/listing/table.rs | 5 +- .../core/src/physical_plan/file_format/mod.rs | 12 ++ datafusion/core/src/row/mod.rs | 2 +- 8 files changed, 180 insertions(+), 30 deletions(-) diff --git a/datafusion/core/src/datasource/file_format/avro.rs b/datafusion/core/src/datasource/file_format/avro.rs index 59c7b2e153454..954c28a773265 100644 --- a/datafusion/core/src/datasource/file_format/avro.rs +++ b/datafusion/core/src/datasource/file_format/avro.rs @@ -57,7 +57,11 @@ impl FileFormat for AvroFormat { Ok(Arc::new(merged_schema)) } - async fn infer_stats(&self, _reader: Arc) -> Result { + async fn infer_stats( + &self, + _reader: Arc, + _table_schema: SchemaRef, + ) -> Result { Ok(Statistics::default()) } @@ -367,7 +371,7 @@ mod tests { .await .expect("Schema inference"); let statistics = format - .infer_stats(local_object_reader(filename.clone())) + .infer_stats(local_object_reader(filename.clone()), file_schema.clone()) .await .expect("Stats inference"); let file_groups = vec![vec![local_unpartitioned_file(filename.to_owned())]]; diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index efcf4bde9d2a0..87bc6be1a5c27 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -120,7 +120,11 @@ impl FileFormat for CsvFormat { Ok(Arc::new(merged_schema)) } - async fn infer_stats(&self, _reader: Arc) -> Result { + async fn infer_stats( + &self, + _reader: Arc, + _table_schema: SchemaRef, + ) -> Result { Ok(Statistics::default()) } @@ -265,7 +269,7 @@ mod tests { .await .expect("Schema inference"); let statistics = format - .infer_stats(local_object_reader(filename.clone())) + .infer_stats(local_object_reader(filename.clone()), file_schema.clone()) .await .expect("Stats inference"); let file_groups = vec![vec![local_unpartitioned_file(filename.to_owned())]]; diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rs index 6d5961a6df9b5..c56b843530c15 100644 --- a/datafusion/core/src/datasource/file_format/json.rs +++ b/datafusion/core/src/datasource/file_format/json.rs @@ -92,7 +92,11 @@ impl FileFormat for JsonFormat { Ok(Arc::new(schema)) } - async fn infer_stats(&self, _reader: Arc) -> Result { + async fn infer_stats( + &self, + _reader: Arc, + _table_schema: SchemaRef, + ) -> Result { Ok(Statistics::default()) } @@ -219,7 +223,10 @@ mod tests { .await .expect("Schema inference"); let statistics = format - .infer_stats(local_object_reader(filename.to_owned())) + .infer_stats( + local_object_reader(filename.to_owned()), + file_schema.clone(), + ) .await .expect("Stats inference"); let file_groups = vec![vec![local_unpartitioned_file(filename.to_owned())]]; diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index d0334b7a96d89..b07cfb38a772d 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -56,7 +56,11 @@ pub trait FileFormat: Send + Sync + fmt::Debug { /// Infer the statistics for the provided object. The cost and accuracy of the /// estimated statistics might vary greatly between file formats. - async fn infer_stats(&self, reader: Arc) -> Result; + async fn infer_stats( + &self, + reader: Arc, + table_schema: SchemaRef, + ) -> Result; /// Take a list of files and convert it to the appropriate executor /// according to this file format. diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs index b1272780c08a9..18168d95fe70b 100644 --- a/datafusion/core/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -25,6 +25,7 @@ use arrow::datatypes::Schema; use arrow::datatypes::SchemaRef; use async_trait::async_trait; use futures::TryStreamExt; +use hashbrown::HashMap; use parquet::arrow::ArrowReader; use parquet::arrow::ParquetFileArrowReader; use parquet::errors::ParquetError; @@ -46,7 +47,7 @@ use crate::error::Result; use crate::logical_plan::combine_filters; use crate::logical_plan::Expr; use crate::physical_plan::expressions::{MaxAccumulator, MinAccumulator}; -use crate::physical_plan::file_format::ParquetExec; +use crate::physical_plan::file_format::{ParquetExec, SchemaAdapter}; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::{Accumulator, Statistics}; use datafusion_data_access::object_store::{ObjectReader, ObjectReaderStream}; @@ -99,8 +100,12 @@ impl FileFormat for ParquetFormat { Ok(Arc::new(merged_schema)) } - async fn infer_stats(&self, reader: Arc) -> Result { - let stats = fetch_statistics(reader)?; + async fn infer_stats( + &self, + reader: Arc, + table_schema: SchemaRef, + ) -> Result { + let stats = fetch_statistics(reader, table_schema)?; Ok(stats) } @@ -279,13 +284,16 @@ fn fetch_schema(object_reader: Arc) -> Result { } /// Read and parse the statistics of the Parquet file at location `path` -fn fetch_statistics(object_reader: Arc) -> Result { +fn fetch_statistics( + object_reader: Arc, + table_schema: SchemaRef, +) -> Result { let obj_reader = ChunkObjectReader(object_reader); let file_reader = Arc::new(SerializedFileReader::new(obj_reader)?); let mut arrow_reader = ParquetFileArrowReader::new(file_reader); - let schema = arrow_reader.get_schema()?; - let num_fields = schema.fields().len(); - let fields = schema.fields().to_vec(); + let file_schema = arrow_reader.get_schema()?; + let num_fields = table_schema.fields().len(); + let fields = table_schema.fields().to_vec(); let meta_data = arrow_reader.get_metadata(); let mut num_rows = 0; @@ -293,32 +301,48 @@ fn fetch_statistics(object_reader: Arc) -> Result let mut null_counts = vec![0; num_fields]; let mut has_statistics = false; - let (mut max_values, mut min_values) = create_max_min_accs(&schema); + let schema_adapter = SchemaAdapter::new(table_schema.clone()); + + let (mut max_values, mut min_values) = create_max_min_accs(&table_schema); for row_group_meta in meta_data.row_groups() { num_rows += row_group_meta.num_rows(); total_byte_size += row_group_meta.total_byte_size(); - let columns_null_counts = row_group_meta - .columns() - .iter() - .flat_map(|c| c.statistics().map(|stats| stats.null_count())); - - for (i, cnt) in columns_null_counts.enumerate() { - null_counts[i] += cnt as usize - } + let mut column_stats: HashMap = HashMap::new(); for (i, column) in row_group_meta.columns().iter().enumerate() { if let Some(stat) = column.statistics() { has_statistics = true; - summarize_min_max(&mut max_values, &mut min_values, &fields, i, stat) + column_stats.insert(i, (stat.null_count(), stat)); + } + } + + if has_statistics { + for (table_idx, null_cnt) in null_counts.iter_mut().enumerate() { + if let Some(file_idx) = + schema_adapter.map_column_index(table_idx, &file_schema) + { + if let Some((null_count, stats)) = column_stats.get(&file_idx) { + *null_cnt += *null_count as usize; + summarize_min_max( + &mut max_values, + &mut min_values, + &fields, + table_idx, + stats, + ) + } + } else { + *null_cnt += num_rows as usize; + } } } } let column_stats = if has_statistics { Some(get_col_stats( - &schema, + &table_schema, null_counts, &mut max_values, &mut min_values, @@ -369,10 +393,102 @@ mod tests { use crate::prelude::{SessionConfig, SessionContext}; use arrow::array::{ - BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, - TimestampNanosecondArray, + ArrayRef, BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, + StringArray, TimestampNanosecondArray, }; + use arrow::record_batch::RecordBatch; + use datafusion_common::ScalarValue; use futures::StreamExt; + use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; + use tempfile::NamedTempFile; + + // Add a new column with the specified field name to the RecordBatch + fn add_to_batch( + batch: &RecordBatch, + field_name: &str, + array: ArrayRef, + ) -> RecordBatch { + let mut fields = batch.schema().fields().clone(); + fields.push(Field::new(field_name, array.data_type().clone(), true)); + let schema = Arc::new(Schema::new(fields)); + + let mut columns = batch.columns().to_vec(); + columns.push(array); + RecordBatch::try_new(schema, columns).expect("error; creating record batch") + } + + fn create_batch(columns: Vec<(&str, ArrayRef)>) -> RecordBatch { + columns.into_iter().fold( + RecordBatch::new_empty(Arc::new(Schema::new(vec![]))), + |batch, (field_name, arr)| add_to_batch(&batch, field_name, arr.clone()), + ) + } + + async fn create_table( + batches: Vec, + ) -> Result<(Vec, Schema)> { + let merged_schema = + Schema::try_merge(batches.iter().map(|b| b.schema().as_ref().clone()))?; + + let files: Vec<_> = batches + .into_iter() + .map(|batch| { + let output = tempfile::NamedTempFile::new().expect("creating temp file"); + + let props = WriterProperties::builder().build(); + let file: std::fs::File = (*output.as_file()) + .try_clone() + .expect("cloning file descriptor"); + let mut writer = ArrowWriter::try_new(file, batch.schema(), Some(props)) + .expect("creating writer"); + + writer.write(&batch).expect("Writing batch"); + writer.close().unwrap(); + output + }) + .collect(); + + Ok((files, merged_schema)) + } + + #[tokio::test] + async fn read_merged_batches() -> Result<()> { + let c1: ArrayRef = + Arc::new(StringArray::from(vec![Some("Foo"), None, Some("bar")])); + + let c2: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), Some(2), None])); + + let batch1 = create_batch(vec![("c1", c1.clone())]); + + let batch2 = create_batch(vec![("c2", c2)]); + + let (files, schema) = create_table(vec![batch1, batch2]).await?; + let table_schema = Arc::new(schema); + + let reader = local_object_reader(files[0].path().to_string_lossy().to_string()); + + let stats = fetch_statistics(reader, table_schema.clone())?; + + assert_eq!(stats.num_rows, Some(3)); + let c1_stats = &stats.column_statistics.as_ref().expect("missing c1 stats")[0]; + let c2_stats = &stats.column_statistics.as_ref().expect("missing c2 stats")[1]; + assert_eq!(c1_stats.null_count, Some(1)); + assert_eq!(c2_stats.null_count, Some(3)); + + let reader = local_object_reader(files[1].path().to_string_lossy().to_string()); + + let stats = fetch_statistics(reader, table_schema)?; + assert_eq!(stats.num_rows, Some(3)); + let c1_stats = &stats.column_statistics.as_ref().expect("missing c1 stats")[0]; + let c2_stats = &stats.column_statistics.as_ref().expect("missing c2 stats")[1]; + assert_eq!(c1_stats.null_count, Some(3)); + assert_eq!(c2_stats.null_count, Some(1)); + assert_eq!(c2_stats.max_value, Some(ScalarValue::Int64(Some(2)))); + assert_eq!(c2_stats.min_value, Some(ScalarValue::Int64(Some(1)))); + + Ok(()) + } #[tokio::test] async fn read_small_batches() -> Result<()> { @@ -645,7 +761,7 @@ mod tests { .await .expect("Schema inference"); let statistics = format - .infer_stats(local_object_reader(filename.clone())) + .infer_stats(local_object_reader(filename.clone()), file_schema.clone()) .await .expect("Stats inference"); let file_groups = vec![vec![local_unpartitioned_file(filename.clone())]]; diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index a470e32c6693c..18f12d03a494a 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -376,7 +376,10 @@ impl ListingTable { let statistics = if self.options.collect_stat { let object_reader = object_store .file_reader(part_file.file_meta.sized_file.clone())?; - self.options.format.infer_stats(object_reader).await? + self.options + .format + .infer_stats(object_reader, self.file_schema.clone()) + .await? } else { Statistics::default() }; diff --git a/datafusion/core/src/physical_plan/file_format/mod.rs b/datafusion/core/src/physical_plan/file_format/mod.rs index cd2bb7fc1344b..72e2f9b87200f 100644 --- a/datafusion/core/src/physical_plan/file_format/mod.rs +++ b/datafusion/core/src/physical_plan/file_format/mod.rs @@ -212,6 +212,18 @@ impl SchemaAdapter { Self { table_schema } } + /// Map a column index in the table schema to a column index in a particular + /// file schema + /// Panics if index is not in range for the table schema + pub(crate) fn map_column_index( + &self, + index: usize, + file_schema: &Schema, + ) -> Option { + let field = self.table_schema.field(index); + file_schema.index_of(field.name()).ok() + } + /// Map projected column indexes to the file schema. This will fail if the table schema /// and the file schema contain a field with the same name and different types. pub fn map_projections( diff --git a/datafusion/core/src/row/mod.rs b/datafusion/core/src/row/mod.rs index 531dbfe3e41ee..d92da7ae5620c 100644 --- a/datafusion/core/src/row/mod.rs +++ b/datafusion/core/src/row/mod.rs @@ -610,7 +610,7 @@ mod tests { .await .expect("Schema inference"); let statistics = format - .infer_stats(local_object_reader(filename.clone())) + .infer_stats(local_object_reader(filename.clone()), file_schema.clone()) .await .expect("Stats inference"); let file_groups = vec![vec![local_unpartitioned_file(filename.clone())]]; From 067c10d9fc839ad9ab72ea8e0cccacb044223ec1 Mon Sep 17 00:00:00 2001 From: Dan Harris <1327726+thinkharderdev@users.noreply.github.com> Date: Wed, 6 Apr 2022 20:17:50 +0100 Subject: [PATCH 3/8] Update datafusion/core/src/physical_plan/file_format/parquet.rs Co-authored-by: Andrew Lamb --- datafusion/core/src/physical_plan/file_format/parquet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index c82a686b31e2e..77dff99cc605a 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -889,7 +889,7 @@ mod tests { let c3: ArrayRef = Arc::new(Int8Array::from(vec![Some(10), Some(20), None])); - // batch1: c1(string), c2(int64), c3(int8) + // batch1: c1(string), c3(int8) let batch1 = create_batch(vec![("c1", c1), ("c3", c3.clone())]); // batch2: c3(int8), c2(int64), c1(string) From adc0a28fc0e7c3fa5d33dbc177fc5a7988b490e1 Mon Sep 17 00:00:00 2001 From: Dan Harris <1327726+thinkharderdev@users.noreply.github.com> Date: Wed, 6 Apr 2022 20:18:25 +0100 Subject: [PATCH 4/8] Update datafusion/core/src/datasource/file_format/mod.rs Co-authored-by: Andrew Lamb --- datafusion/core/src/datasource/file_format/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index b07cfb38a772d..2583d3eba8081 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -56,6 +56,11 @@ pub trait FileFormat: Send + Sync + fmt::Debug { /// Infer the statistics for the provided object. The cost and accuracy of the /// estimated statistics might vary greatly between file formats. + /// + /// `table_schema` is the (combined) schema of the overall table + /// and may be a superset of the schema contained in this file. + /// + /// TODO: should the file source return statistics for only columns referred to in the table schema? async fn infer_stats( &self, reader: Arc, From a0809b568ec2a581e7dc11246ce9c35c1a6b364f Mon Sep 17 00:00:00 2001 From: Dan Harris <1327726+thinkharderdev@users.noreply.github.com> Date: Wed, 6 Apr 2022 20:18:43 +0100 Subject: [PATCH 5/8] Update datafusion/core/src/physical_plan/file_format/parquet.rs Co-authored-by: Andrew Lamb --- datafusion/core/src/physical_plan/file_format/parquet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index 77dff99cc605a..3ebb5764859de 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -997,7 +997,7 @@ mod tests { // batch1: c1(string), c2(int64), c3(int8) let batch1 = create_batch(vec![("c1", c1.clone())]); - // batch2: c3(int8), c2(int64), c1(string) + // batch2: c2(int64) let batch2 = create_batch(vec![("c2", c2)]); let filter = col("c2").eq(lit(0_i64)); From 869fe27fcb3db3e70657f3062da3ff9d31e78496 Mon Sep 17 00:00:00 2001 From: Dan Harris <1327726+thinkharderdev@users.noreply.github.com> Date: Wed, 6 Apr 2022 20:22:52 +0100 Subject: [PATCH 6/8] Update datafusion/core/src/physical_plan/file_format/parquet.rs Co-authored-by: Andrew Lamb --- datafusion/core/src/physical_plan/file_format/parquet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index 3ebb5764859de..c90327b0b17a0 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -892,7 +892,7 @@ mod tests { // batch1: c1(string), c3(int8) let batch1 = create_batch(vec![("c1", c1), ("c3", c3.clone())]); - // batch2: c3(int8), c2(int64), c1(string) + // batch2: c3(int8), c2(int64) let batch2 = create_batch(vec![("c3", c3), ("c2", c2)]); let filter = col("c2").eq(lit(0_i64)); From 5e6f77da2cf628f57e2e3fce2cc20e6fc5f050bf Mon Sep 17 00:00:00 2001 From: Dan Harris <1327726+thinkharderdev@users.noreply.github.com> Date: Wed, 6 Apr 2022 20:23:56 +0100 Subject: [PATCH 7/8] Update datafusion/core/src/physical_plan/file_format/parquet.rs Co-authored-by: Andrew Lamb --- datafusion/core/src/physical_plan/file_format/parquet.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index c90327b0b17a0..4e2f90ed99e00 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -992,9 +992,7 @@ mod tests { let c2: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), Some(2), None])); - // let c3: ArrayRef = Arc::new(Int8Array::from(vec![Some(10), Some(20), None])); - - // batch1: c1(string), c2(int64), c3(int8) + // batch1: c1(string) let batch1 = create_batch(vec![("c1", c1.clone())]); // batch2: c2(int64) From b1aaaa8ecdd44579cc998de3b67ed30227d13971 Mon Sep 17 00:00:00 2001 From: Dan Harris Date: Thu, 7 Apr 2022 11:13:31 +0100 Subject: [PATCH 8/8] Add comments and cargo fmt --- datafusion/core/src/datasource/file_format/mod.rs | 4 ++-- datafusion/core/src/physical_plan/file_format/parquet.rs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 2583d3eba8081..bb37b755745a7 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -56,10 +56,10 @@ pub trait FileFormat: Send + Sync + fmt::Debug { /// Infer the statistics for the provided object. The cost and accuracy of the /// estimated statistics might vary greatly between file formats. - /// + /// /// `table_schema` is the (combined) schema of the overall table /// and may be a superset of the schema contained in this file. - /// + /// /// TODO: should the file source return statistics for only columns referred to in the table schema? async fn infer_stats( &self, diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index 4e2f90ed99e00..1f5c9d0e72b41 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -1005,6 +1005,11 @@ mod tests { .await .unwrap(); + // This does not look correct since the "c2" values in the result do not in fact match the predicate `c2 == 0` + // but parquet pruning is not exact. If the min/max values are not defined (which they are not in this case since the it is + // a null array, then the pruning predicate (currently) can not be applied. + // In a real query where this predicate was pushed down from a filter stage instead of created directly in the `ParquetExec`, + // the filter stage would be preserved as a separate execution plan stage so the actual query results would be as expected. let expected = vec![ "+-----+----+", "| c1 | c2 |",