diff --git a/datafusion/core/tests/parquet/io/mod.rs b/datafusion/core/tests/parquet/io/mod.rs index 0a9e9a47ea289..93e160fc054ca 100644 --- a/datafusion/core/tests/parquet/io/mod.rs +++ b/datafusion/core/tests/parquet/io/mod.rs @@ -36,6 +36,7 @@ use url::Url; // Tests are in submodules mod bloom_filter_io; mod filter_pushdown_io; +mod page_index_cache_io; mod page_pruning_io; mod row_group_pruning_io; mod topk_io; @@ -530,6 +531,12 @@ impl ParquetIoTest { .cache_manager .get_file_metadata_cache() .clear(); + self.query_with_cached_metadata(sql).await + } + + /// Like [`Self::query`], but keeps the file metadata cached by earlier + /// queries, so the output shows the I/O of a warm read. + pub(crate) async fn query_with_cached_metadata(&self, sql: &str) -> String { self.events.lock().unwrap().clear(); self.requests.store(0, Ordering::SeqCst); diff --git a/datafusion/core/tests/parquet/io/page_index_cache_io.rs b/datafusion/core/tests/parquet/io/page_index_cache_io.rs new file mode 100644 index 0000000000000..38b26dee0bfad --- /dev/null +++ b/datafusion/core/tests/parquet/io/page_index_cache_io.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Tests tracking I/O events when the page index is loaded after the footer +//! and the file metadata cache is reused by later queries. +//! +//! Statistics collection is disabled so that planning does not load (and +//! cache) the page index. Each scan first reads the footer without the page +//! index, and loads the page index only if row group statistics can not prove +//! it is unneeded. +//! +//! Most tests use a file with one row group with three data pages of three +//! rows per column (see [`three_page_file`]), so the page index of `id` can +//! prune pages: +//! +//! ```text +//! id page min/max: page 0: 1/3 page 1: 4/6 page 2: 7/9 +//! id = 5 -> page 1 of each column +//! id > 0 -> the row group is fully matched, the page index is not needed +//! ``` + +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int32Array, RecordBatch}; +use bytes::Bytes; +use datafusion::assert_batches_eq; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::prelude::SessionConfig; +use insta::assert_snapshot; +use object_store::path::Path; +use object_store::{PutOptions, PutPayload}; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; + +use super::ParquetIoTest; + +const VAL: [i32; 9] = [0, 5, 10, 0, 7, 10, 0, 5, 10]; + +/// A page index loaded after the footer is cached with the footer, so a +/// later query of the same file reads neither again. +#[tokio::test] +async fn test_page_index_cache_io_repeated_query() { + let test = page_index_cache_test(three_page_file(VAL, true)).await; + let sql = "SELECT * FROM t WHERE id = 5"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0) (offsets 178..323) + Request 4 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + Produce batch (1 rows) + "); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + Produce batch (1 rows) + "); + let batches = test.ctx.sql(sql).await.unwrap().collect().await.unwrap(); + assert_batches_eq!( + [ + "+----+-----+", + "| id | val |", + "+----+-----+", + "| 5 | 7 |", + "+----+-----+", + ], + &batches + ); +} + +/// Each range of a file split across partitions opens the file separately. +/// Once the page index is cached, the opens of all ranges reuse it. +#[tokio::test] +async fn test_page_index_cache_io_file_ranges() { + let mut config = SessionConfig::new() + .with_target_partitions(2) + .with_collect_statistics(false); + config.options_mut().optimizer.repartition_file_min_size = 0; + let test = ParquetIoTest::new(three_row_group_file(), config).await; + let sql = "SELECT * FROM t WHERE val = 7 ORDER BY id"; + + let plan = test.ctx.sql(&format!("EXPLAIN {sql}")).await.unwrap(); + let plan = arrow::util::pretty::pretty_format_batches(&plan.collect().await.unwrap()) + .unwrap() + .to_string(); + assert_eq!(plan.matches("table.parquet:").count(), 2, "{plan}"); + + // Partitions run concurrently, so the order of requests (and whether + // both ranges read the page index on the first query) varies + test.query(sql).await; + let warm = test.query_with_cached_metadata(sql).await; + assert!(!warm.contains("Parquet footer"), "{warm}"); + assert!(!warm.contains("ColumnIndex"), "{warm}"); + assert!(!warm.contains("OffsetIndex"), "{warm}"); + assert!(warm.contains("(rg 0)") && warm.contains("(rg 2)"), "{warm}"); + + let batches = test.ctx.sql(sql).await.unwrap().collect().await.unwrap(); + assert_batches_eq!( + [ + "+------+-----+", + "| id | val |", + "+------+-----+", + "| 500 | 7 |", + "| 1500 | 7 |", + "| 2500 | 7 |", + "| 3500 | 7 |", + "| 4500 | 7 |", + "| 5500 | 7 |", + "+------+-----+", + ], + &batches + ); +} + +/// When row group statistics prove the page index is not needed, it is not +/// read by the first query, nor by a later query that reuses the footer. +#[tokio::test] +async fn test_page_index_cache_io_fully_matched() { + let test = page_index_cache_test(three_page_file(VAL, true)).await; + let sql = "SELECT * FROM t WHERE id > 0"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id > 0 (9 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (9 rows) + "); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id > 0 (9 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (9 rows) + "); +} + +/// Without a column index for `id`, the page index is never read. +#[tokio::test] +async fn test_page_index_cache_io_no_column_index() { + let test = page_index_cache_test(three_page_file(VAL, false)).await; + let sql = "SELECT * FROM t WHERE id = 5"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 229..734) + Request 3 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (1 rows) + "); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (1 rows) + "); +} + +/// A cached footer and page index are not reused after the file changes. +/// +/// The new file has the same layout (only the values of `val` differ), so +/// the reported byte ranges still describe it. +#[tokio::test] +async fn test_page_index_cache_io_changed_file() { + let test = page_index_cache_test(three_page_file(VAL, true)).await; + let sql = "SELECT * FROM t WHERE id = 5 AND val = 8"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 AND val = 8 (0 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0) (offsets 178..323) + Request 4 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + "); + + test.ctx + .runtime_env() + .object_store(ObjectStoreUrl::parse("mem://").unwrap()) + .unwrap() + .put_opts( + &Path::from("table.parquet"), + PutPayload::from(three_page_file(VAL.map(|v| v + 1), true)), + PutOptions::default(), + ) + .await + .unwrap(); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 AND val = 8 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0) (offsets 178..323) + Request 4 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + Produce batch (1 rows) + "); +} + +/// Returns a [`ParquetIoTest`] for `file` that does not collect statistics +/// during planning +async fn page_index_cache_test(file: Bytes) -> ParquetIoTest { + let config = SessionConfig::new() + .with_target_partitions(1) + .with_collect_statistics(false); + ParquetIoTest::new(file, config).await +} + +/// Returns a Parquet file with three row groups of 2000 rows, `id` `0..6000` +/// and `val`, written as pages of 1000 rows. `val` is 7 when `id % 1000` is +/// 500, and `id % 5` otherwise. The column chunks are much larger than the +/// footer, so splitting the file in two puts row groups 0 and 1 in the first +/// range and row group 2 in the second. +fn three_row_group_file() -> Bytes { + let id: ArrayRef = Arc::new(Int32Array::from_iter_values(0..6000)); + let val: ArrayRef = Arc::new(Int32Array::from_iter_values( + (0..6000).map(|i| if i % 1000 == 500 { 7 } else { i % 5 }), + )); + let batch = RecordBatch::try_from_iter([("id", id), ("val", val)]).unwrap(); + let props = WriterProperties::builder() + .set_dictionary_enabled(false) + .set_max_row_group_row_count(Some(2000)) + .set_data_page_row_count_limit(1000) + .set_write_batch_size(1000) + .build(); + + let mut buffer = Vec::new(); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + Bytes::from(buffer) +} + +/// Returns a Parquet file with one row group of nine rows, `id` `1..=9` and +/// `val`, where each column chunk is written as three data pages of three +/// rows each. The column index is written only if `page_statistics` is true. +fn three_page_file(val: [i32; 9], page_statistics: bool) -> Bytes { + let id: ArrayRef = Arc::new(Int32Array::from_iter_values(1..=9)); + let val: ArrayRef = Arc::new(Int32Array::from(val.to_vec())); + let batch = RecordBatch::try_from_iter([("id", id), ("val", val)]).unwrap(); + + let statistics = if page_statistics { + EnabledStatistics::Page + } else { + EnabledStatistics::Chunk + }; + let props = WriterProperties::builder() + .set_dictionary_enabled(false) + .set_statistics_enabled(statistics) + .set_data_page_row_count_limit(3) + .set_write_batch_size(3) + .build(); + + let mut buffer = Vec::new(); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + Bytes::from(buffer) +} diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index cee1618379fb3..0b734ea14d28f 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -21,8 +21,9 @@ use crate::push_decoder::{ }; use crate::row_group_filter::{RowGroupAccessPlanFilter, row_group_in_range}; use crate::{ - BloomFilterStatistics, Int96Coercer, ParquetAccessPlan, ParquetFileMetrics, - ParquetFileReaderFactory, ParquetRowSelection, ParquetVirtualColumn, RowGroupAccess, + BloomFilterStatistics, CachedParquetFileReaderFactory, Int96Coercer, + ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory, ParquetRowSelection, + ParquetVirtualColumn, RowGroupAccess, schema_coercion::apply_file_schema_type_coercions_with_rle, }; use arrow::array::RecordBatch; @@ -76,7 +77,9 @@ use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::parquet_column; use parquet::basic::Type; use parquet::bloom_filter::Sbbf; -use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMetaData}; +use parquet::file::metadata::{ + PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData, +}; /// Morselizer-level state for virtual columns, precomputed once per scan /// partition so each file skips the validator walks, `null_replacements` @@ -1111,6 +1114,24 @@ impl PreparedParquetOpen { options, }) } + + /// Stores `metadata`, read after the initial metadata load, in the cache + /// of a [`CachedParquetFileReaderFactory`] so later opens of the file + /// reuse it. + fn cache_metadata(&self, metadata: &Arc) { + // Like the initial metadata load, never share decrypted metadata + #[cfg(feature = "parquet_encryption")] + if self.file_decryption_properties.is_some() { + return; + } + if let Some(factory) = self + .parquet_file_reader_factory + .downcast_ref::() + { + factory + .cache_metadata(&self.partitioned_file.object_meta, Arc::clone(metadata)); + } + } } impl MetadataLoadedParquetOpen { @@ -1494,7 +1515,7 @@ impl RowGroupsPrunedParquetOpen { /// Load the page index if pruning requires it and metadata did not include it. async fn load_page_index(mut self) -> Result { - self.prepared.loaded.reader_metadata = load_page_index( + let reader_metadata = load_page_index( self.prepared.loaded.reader_metadata.clone(), &mut self.prepared.loaded.prepared.async_file_reader, self.prepared @@ -1511,6 +1532,14 @@ impl RowGroupsPrunedParquetOpen { ), ) .await?; + let loaded = &mut self.prepared.loaded; + if !Arc::ptr_eq( + reader_metadata.metadata(), + loaded.reader_metadata.metadata(), + ) { + loaded.prepared.cache_metadata(reader_metadata.metadata()); + } + loaded.reader_metadata = reader_metadata; Ok(self) } diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rs index f9a5e11fddffd..e9c497d3a0128 100644 --- a/datafusion/datasource-parquet/src/reader.rs +++ b/datafusion/datasource-parquet/src/reader.rs @@ -11,13 +11,14 @@ use arrow::datatypes::SchemaRef; use bytes::Bytes; use datafusion_common::HashMap; use datafusion_datasource::PartitionedFile; +use datafusion_execution::cache::cache_manager::CachedFileMetadataEntry; use datafusion_execution::cache::cache_manager::FileMetadata; use datafusion_execution::cache::cache_manager::FileMetadataCache; use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use futures::FutureExt; use futures::TryFutureExt; use futures::future::BoxFuture; -use object_store::{ObjectStore, ObjectStoreExt}; +use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::arrow::async_reader::AsyncFileReader; use parquet::errors::ParquetError; @@ -155,6 +156,22 @@ impl CachedParquetFileReaderFactory { metadata_cache, } } + + /// Replaces the cached metadata of `object_meta` with `metadata`, which + /// was read after [`ParquetFileReader::get_metadata`] (for example a page + /// index loaded only once pruning needed it). + pub(crate) fn cache_metadata( + &self, + object_meta: &ObjectMeta, + metadata: Arc, + ) { + // `DFParquetMetadata::fetch_metadata` only reuses entries of this type + let file_metadata = crate::metadata::CachedParquetMetaData::new(metadata); + self.metadata_cache.put( + &object_meta.location, + CachedFileMetadataEntry::new(object_meta.clone(), Arc::new(file_metadata)), + ); + } } impl ParquetFileReaderFactory for CachedParquetFileReaderFactory {