From f0b1923bc05749d631163c4ebc9c8b357389d8d2 Mon Sep 17 00:00:00 2001 From: Varun Srinivas <5206947+nookcreed@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:51:16 -0800 Subject: [PATCH] fix: cache Parquet page index loaded after the footer The Parquet opener reads the footer without the page index and loads the page index later, only when pruning needs it. That deferred load bypassed the file metadata cache, so every later open of the same file (each query and each file range) read the page index from storage again. After a successful deferred load returns changed metadata, store it in the cache of `CachedParquetFileReaderFactory` under the same key and object metadata. Decrypted metadata is still never cached, and custom reader factories are unchanged. Closes #23978. --- datafusion/core/tests/parquet/io/mod.rs | 7 + .../tests/parquet/io/page_index_cache_io.rs | 288 ++++++++++++++++++ .../datasource-parquet/src/opener/mod.rs | 37 ++- datafusion/datasource-parquet/src/reader.rs | 19 +- 4 files changed, 346 insertions(+), 5 deletions(-) create mode 100644 datafusion/core/tests/parquet/io/page_index_cache_io.rs diff --git a/datafusion/core/tests/parquet/io/mod.rs b/datafusion/core/tests/parquet/io/mod.rs index 0a9e9a47ea289..93e160fc054ca 100644 --- a/datafusion/core/tests/parquet/io/mod.rs +++ b/datafusion/core/tests/parquet/io/mod.rs @@ -36,6 +36,7 @@ use url::Url; // Tests are in submodules mod bloom_filter_io; mod filter_pushdown_io; +mod page_index_cache_io; mod page_pruning_io; mod row_group_pruning_io; mod topk_io; @@ -530,6 +531,12 @@ impl ParquetIoTest { .cache_manager .get_file_metadata_cache() .clear(); + self.query_with_cached_metadata(sql).await + } + + /// Like [`Self::query`], but keeps the file metadata cached by earlier + /// queries, so the output shows the I/O of a warm read. + pub(crate) async fn query_with_cached_metadata(&self, sql: &str) -> String { self.events.lock().unwrap().clear(); self.requests.store(0, Ordering::SeqCst); diff --git a/datafusion/core/tests/parquet/io/page_index_cache_io.rs b/datafusion/core/tests/parquet/io/page_index_cache_io.rs new file mode 100644 index 0000000000000..38b26dee0bfad --- /dev/null +++ b/datafusion/core/tests/parquet/io/page_index_cache_io.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Tests tracking I/O events when the page index is loaded after the footer +//! and the file metadata cache is reused by later queries. +//! +//! Statistics collection is disabled so that planning does not load (and +//! cache) the page index. Each scan first reads the footer without the page +//! index, and loads the page index only if row group statistics can not prove +//! it is unneeded. +//! +//! Most tests use a file with one row group with three data pages of three +//! rows per column (see [`three_page_file`]), so the page index of `id` can +//! prune pages: +//! +//! ```text +//! id page min/max: page 0: 1/3 page 1: 4/6 page 2: 7/9 +//! id = 5 -> page 1 of each column +//! id > 0 -> the row group is fully matched, the page index is not needed +//! ``` + +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int32Array, RecordBatch}; +use bytes::Bytes; +use datafusion::assert_batches_eq; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::prelude::SessionConfig; +use insta::assert_snapshot; +use object_store::path::Path; +use object_store::{PutOptions, PutPayload}; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; + +use super::ParquetIoTest; + +const VAL: [i32; 9] = [0, 5, 10, 0, 7, 10, 0, 5, 10]; + +/// A page index loaded after the footer is cached with the footer, so a +/// later query of the same file reads neither again. +#[tokio::test] +async fn test_page_index_cache_io_repeated_query() { + let test = page_index_cache_test(three_page_file(VAL, true)).await; + let sql = "SELECT * FROM t WHERE id = 5"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0) (offsets 178..323) + Request 4 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + Produce batch (1 rows) + "); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + Produce batch (1 rows) + "); + let batches = test.ctx.sql(sql).await.unwrap().collect().await.unwrap(); + assert_batches_eq!( + [ + "+----+-----+", + "| id | val |", + "+----+-----+", + "| 5 | 7 |", + "+----+-----+", + ], + &batches + ); +} + +/// Each range of a file split across partitions opens the file separately. +/// Once the page index is cached, the opens of all ranges reuse it. +#[tokio::test] +async fn test_page_index_cache_io_file_ranges() { + let mut config = SessionConfig::new() + .with_target_partitions(2) + .with_collect_statistics(false); + config.options_mut().optimizer.repartition_file_min_size = 0; + let test = ParquetIoTest::new(three_row_group_file(), config).await; + let sql = "SELECT * FROM t WHERE val = 7 ORDER BY id"; + + let plan = test.ctx.sql(&format!("EXPLAIN {sql}")).await.unwrap(); + let plan = arrow::util::pretty::pretty_format_batches(&plan.collect().await.unwrap()) + .unwrap() + .to_string(); + assert_eq!(plan.matches("table.parquet:").count(), 2, "{plan}"); + + // Partitions run concurrently, so the order of requests (and whether + // both ranges read the page index on the first query) varies + test.query(sql).await; + let warm = test.query_with_cached_metadata(sql).await; + assert!(!warm.contains("Parquet footer"), "{warm}"); + assert!(!warm.contains("ColumnIndex"), "{warm}"); + assert!(!warm.contains("OffsetIndex"), "{warm}"); + assert!(warm.contains("(rg 0)") && warm.contains("(rg 2)"), "{warm}"); + + let batches = test.ctx.sql(sql).await.unwrap().collect().await.unwrap(); + assert_batches_eq!( + [ + "+------+-----+", + "| id | val |", + "+------+-----+", + "| 500 | 7 |", + "| 1500 | 7 |", + "| 2500 | 7 |", + "| 3500 | 7 |", + "| 4500 | 7 |", + "| 5500 | 7 |", + "+------+-----+", + ], + &batches + ); +} + +/// When row group statistics prove the page index is not needed, it is not +/// read by the first query, nor by a later query that reuses the footer. +#[tokio::test] +async fn test_page_index_cache_io_fully_matched() { + let test = page_index_cache_test(three_page_file(VAL, true)).await; + let sql = "SELECT * FROM t WHERE id > 0"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id > 0 (9 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (9 rows) + "); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id > 0 (9 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (9 rows) + "); +} + +/// Without a column index for `id`, the page index is never read. +#[tokio::test] +async fn test_page_index_cache_io_no_column_index() { + let test = page_index_cache_test(three_page_file(VAL, false)).await; + let sql = "SELECT * FROM t WHERE id = 5"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 229..734) + Request 3 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (1 rows) + "); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read data pages 0..=2 for column 'id' (rg 0) (offsets 4..91) + Read data pages 0..=2 for column 'val' (rg 0) (offsets 91..178) + Produce batch (1 rows) + "); +} + +/// A cached footer and page index are not reused after the file changes. +/// +/// The new file has the same layout (only the values of `val` differ), so +/// the reported byte ranges still describe it. +#[tokio::test] +async fn test_page_index_cache_io_changed_file() { + let test = page_index_cache_test(three_page_file(VAL, true)).await; + let sql = "SELECT * FROM t WHERE id = 5 AND val = 8"; + assert_snapshot!(test.query(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 AND val = 8 (0 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0) (offsets 178..323) + Request 4 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + "); + + test.ctx + .runtime_env() + .object_store(ObjectStoreUrl::parse("mem://").unwrap()) + .unwrap() + .put_opts( + &Path::from("table.parquet"), + PutPayload::from(three_page_file(VAL.map(|v| v + 1), true)), + PutOptions::default(), + ) + .await + .unwrap(); + assert_snapshot!(test.query_with_cached_metadata(sql).await, @r" + --- Query: SELECT * FROM t WHERE id = 5 AND val = 8 (1 rows) --- + Request 1 + Head (object size and metadata) + Request 2 + Read Parquet footer (offsets 323..838) + Request 3 + Read ColumnIndex and OffsetIndex for columns 'id', 'val' (rg 0) (offsets 178..323) + Request 4 + Read data page 1 for column 'id' (rg 0) (offsets 33..62) + Read data page 1 for column 'val' (rg 0) (offsets 120..149) + Produce batch (1 rows) + "); +} + +/// Returns a [`ParquetIoTest`] for `file` that does not collect statistics +/// during planning +async fn page_index_cache_test(file: Bytes) -> ParquetIoTest { + let config = SessionConfig::new() + .with_target_partitions(1) + .with_collect_statistics(false); + ParquetIoTest::new(file, config).await +} + +/// Returns a Parquet file with three row groups of 2000 rows, `id` `0..6000` +/// and `val`, written as pages of 1000 rows. `val` is 7 when `id % 1000` is +/// 500, and `id % 5` otherwise. The column chunks are much larger than the +/// footer, so splitting the file in two puts row groups 0 and 1 in the first +/// range and row group 2 in the second. +fn three_row_group_file() -> Bytes { + let id: ArrayRef = Arc::new(Int32Array::from_iter_values(0..6000)); + let val: ArrayRef = Arc::new(Int32Array::from_iter_values( + (0..6000).map(|i| if i % 1000 == 500 { 7 } else { i % 5 }), + )); + let batch = RecordBatch::try_from_iter([("id", id), ("val", val)]).unwrap(); + let props = WriterProperties::builder() + .set_dictionary_enabled(false) + .set_max_row_group_row_count(Some(2000)) + .set_data_page_row_count_limit(1000) + .set_write_batch_size(1000) + .build(); + + let mut buffer = Vec::new(); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + Bytes::from(buffer) +} + +/// Returns a Parquet file with one row group of nine rows, `id` `1..=9` and +/// `val`, where each column chunk is written as three data pages of three +/// rows each. The column index is written only if `page_statistics` is true. +fn three_page_file(val: [i32; 9], page_statistics: bool) -> Bytes { + let id: ArrayRef = Arc::new(Int32Array::from_iter_values(1..=9)); + let val: ArrayRef = Arc::new(Int32Array::from(val.to_vec())); + let batch = RecordBatch::try_from_iter([("id", id), ("val", val)]).unwrap(); + + let statistics = if page_statistics { + EnabledStatistics::Page + } else { + EnabledStatistics::Chunk + }; + let props = WriterProperties::builder() + .set_dictionary_enabled(false) + .set_statistics_enabled(statistics) + .set_data_page_row_count_limit(3) + .set_write_batch_size(3) + .build(); + + let mut buffer = Vec::new(); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + Bytes::from(buffer) +} diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index cee1618379fb3..0b734ea14d28f 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -21,8 +21,9 @@ use crate::push_decoder::{ }; use crate::row_group_filter::{RowGroupAccessPlanFilter, row_group_in_range}; use crate::{ - BloomFilterStatistics, Int96Coercer, ParquetAccessPlan, ParquetFileMetrics, - ParquetFileReaderFactory, ParquetRowSelection, ParquetVirtualColumn, RowGroupAccess, + BloomFilterStatistics, CachedParquetFileReaderFactory, Int96Coercer, + ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory, ParquetRowSelection, + ParquetVirtualColumn, RowGroupAccess, schema_coercion::apply_file_schema_type_coercions_with_rle, }; use arrow::array::RecordBatch; @@ -76,7 +77,9 @@ use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::parquet_column; use parquet::basic::Type; use parquet::bloom_filter::Sbbf; -use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMetaData}; +use parquet::file::metadata::{ + PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData, +}; /// Morselizer-level state for virtual columns, precomputed once per scan /// partition so each file skips the validator walks, `null_replacements` @@ -1111,6 +1114,24 @@ impl PreparedParquetOpen { options, }) } + + /// Stores `metadata`, read after the initial metadata load, in the cache + /// of a [`CachedParquetFileReaderFactory`] so later opens of the file + /// reuse it. + fn cache_metadata(&self, metadata: &Arc) { + // Like the initial metadata load, never share decrypted metadata + #[cfg(feature = "parquet_encryption")] + if self.file_decryption_properties.is_some() { + return; + } + if let Some(factory) = self + .parquet_file_reader_factory + .downcast_ref::() + { + factory + .cache_metadata(&self.partitioned_file.object_meta, Arc::clone(metadata)); + } + } } impl MetadataLoadedParquetOpen { @@ -1494,7 +1515,7 @@ impl RowGroupsPrunedParquetOpen { /// Load the page index if pruning requires it and metadata did not include it. async fn load_page_index(mut self) -> Result { - self.prepared.loaded.reader_metadata = load_page_index( + let reader_metadata = load_page_index( self.prepared.loaded.reader_metadata.clone(), &mut self.prepared.loaded.prepared.async_file_reader, self.prepared @@ -1511,6 +1532,14 @@ impl RowGroupsPrunedParquetOpen { ), ) .await?; + let loaded = &mut self.prepared.loaded; + if !Arc::ptr_eq( + reader_metadata.metadata(), + loaded.reader_metadata.metadata(), + ) { + loaded.prepared.cache_metadata(reader_metadata.metadata()); + } + loaded.reader_metadata = reader_metadata; Ok(self) } diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rs index f9a5e11fddffd..e9c497d3a0128 100644 --- a/datafusion/datasource-parquet/src/reader.rs +++ b/datafusion/datasource-parquet/src/reader.rs @@ -11,13 +11,14 @@ use arrow::datatypes::SchemaRef; use bytes::Bytes; use datafusion_common::HashMap; use datafusion_datasource::PartitionedFile; +use datafusion_execution::cache::cache_manager::CachedFileMetadataEntry; use datafusion_execution::cache::cache_manager::FileMetadata; use datafusion_execution::cache::cache_manager::FileMetadataCache; use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use futures::FutureExt; use futures::TryFutureExt; use futures::future::BoxFuture; -use object_store::{ObjectStore, ObjectStoreExt}; +use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::arrow::async_reader::AsyncFileReader; use parquet::errors::ParquetError; @@ -155,6 +156,22 @@ impl CachedParquetFileReaderFactory { metadata_cache, } } + + /// Replaces the cached metadata of `object_meta` with `metadata`, which + /// was read after [`ParquetFileReader::get_metadata`] (for example a page + /// index loaded only once pruning needed it). + pub(crate) fn cache_metadata( + &self, + object_meta: &ObjectMeta, + metadata: Arc, + ) { + // `DFParquetMetadata::fetch_metadata` only reuses entries of this type + let file_metadata = crate::metadata::CachedParquetMetaData::new(metadata); + self.metadata_cache.put( + &object_meta.location, + CachedFileMetadataEntry::new(object_meta.clone(), Arc::new(file_metadata)), + ); + } } impl ParquetFileReaderFactory for CachedParquetFileReaderFactory {