diff --git a/src/core/src/cache/budget.rs b/src/core/src/cache/budget.rs index 2b7bc5d0..8e15dfdd 100644 --- a/src/core/src/cache/budget.rs +++ b/src/core/src/cache/budget.rs @@ -37,7 +37,7 @@ impl BudgetAccounting { /// Returns ok if the memory was reserved, err if the memory budget is full. pub(super) fn try_reserve_memory(&self, request_bytes: usize) -> Result<(), ()> { let used = self.used_memory_bytes.load(Ordering::Relaxed); - if used + request_bytes > self.max_memory_bytes { + if used.saturating_add(request_bytes) > self.max_memory_bytes { return Err(()); } @@ -52,6 +52,12 @@ impl BudgetAccounting { } } + /// Charge `bytes` of memory whatever the budget reads: for memory an + /// entry already holds, which the budget missed. + pub(super) fn charge_memory(&self, bytes: usize) { + self.used_memory_bytes.fetch_add(bytes, Ordering::Relaxed); + } + /// Adjust memory usage after transcoding. /// Returns ok if the usage was adjusted, err if the memory budget is full when new_size is larger than old_size. pub(super) fn try_update_memory_usage( diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 03350b31..16739ae8 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -220,6 +220,16 @@ pub enum PrefetchResult { /// Length of the identity trailer appended to every object written to disk. const IDENTITY_TRAILER_LEN: usize = std::mem::size_of::(); +/// Whether `slot` is the memory entry `observed`: the same array, not only an +/// equal one, so an entry written since is not mistaken for it. +fn same_memory_entry(slot: &CacheEntry, observed: &CacheEntry) -> bool { + match (slot, observed) { + (CacheEntry::MemoryArrow(a), CacheEntry::MemoryArrow(b)) => std::sync::Arc::ptr_eq(a, b), + (CacheEntry::MemoryLiquid(a), CacheEntry::MemoryLiquid(b)) => std::sync::Arc::ptr_eq(a, b), + _ => false, + } +} + /// Builder returned by [`LiquidCache::insert`] for configuring cache writes. impl LiquidCache { /// Return current cache statistics: counts and resource usage. @@ -515,6 +525,11 @@ impl LiquidCache { Ok((new_batch, write)) } CacheEntry::MemoryLiquid(liquid_array) => { + // With no disk tier the write below can only fail; it is + // refused before the array is serialised for it. + if self.config.max_disk_bytes() == 0 { + return Err(CacheFull); + } let liquid_bytes = Bytes::from(liquid_array.to_bytes()); let disk_bytes = liquid_bytes.len(); let write = self @@ -554,7 +569,14 @@ impl LiquidCache { kind: CachedBatchType::from(¬_inserted), }); - let victims = self.cache_policy.find_memory_victim(8); + // No eviction makes room for an entry larger than the whole memory + // tier; evicting for it would only empty the cache before refusing + // it. It goes the way of an insert into an empty cache instead. + let victims = if not_inserted.memory_usage_bytes() > self.config.max_memory_bytes() { + Vec::new() + } else { + self.cache_policy.find_memory_victim(8) + }; if victims.is_empty() { // no advice, because the cache is already empty // this can happen if the entry to be inserted is too large, in that case, @@ -631,24 +653,43 @@ impl LiquidCache { "a disk record is installed only by the write that charged it" ); let new_memory_size = to_insert.memory_usage_bytes(); - let reserved = match self.index.get(&entry_id) { - Some(slot) => self - .budget - .try_update_memory_usage(slot.entry.memory_usage_bytes(), new_memory_size), - // A disk record holds no memory, so it installs however full the - // memory budget reads. Its write cannot then be dropped, taking - // its charge, while the caller retries the record. - None if new_memory_size == 0 => Ok(()), - None => self.budget.try_reserve_memory(new_memory_size), - }; - if reserved.is_err() { + // Only growth over the slot read here is reserved up front, against + // the budget. The rest is settled once the insert has displaced + // whatever slot is there by then, which another write can have + // replaced or removed since: the budget then counts exactly the + // entries the index holds, never fewer. When the slot read here was + // removed meanwhile, the settle charges this entry in full, however + // full the budget is: usage can then exceed the maximum, by up to one + // entry for each insert that raced the removal of its key, until a + // later insert that grows the budget evicts. + // + // A disk record holds no memory, so it installs however full the + // memory budget reads. Its write cannot then be dropped, taking its + // charge, while the caller retries the record. + let assumed_old_size = self + .index + .get(&entry_id) + .map_or(0, |slot| slot.entry.memory_usage_bytes()); + let reserved = new_memory_size.saturating_sub(assumed_old_size); + if reserved > 0 && self.budget.try_reserve_memory(reserved).is_err() { return Err(to_insert); } let cached_batch_type = CachedBatchType::from(&to_insert); - let displaced = self - .index - .insert(&entry_id, identity, to_insert) - .and_then(|displaced| self.charge_of(&displaced.entry)); + let displaced = self.index.insert(&entry_id, identity, to_insert); + let actual_old_size = displaced + .as_ref() + .map_or(0, |slot| slot.entry.memory_usage_bytes()); + // The insert's net change is `new - actual_old`, of which `reserved` + // is charged already. + let remaining = new_memory_size as i128 - actual_old_size as i128 - reserved as i128; + if remaining > 0 { + self.budget.charge_memory(remaining as usize); + } else if remaining < 0 { + self.budget + .try_update_memory_usage((-remaining) as usize, 0) + .expect("memory release cannot fail"); + } + let displaced = displaced.and_then(|displaced| self.charge_of(&displaced.entry)); let orphan = match write { Some(DiskWrite { key_lock, @@ -743,6 +784,33 @@ impl LiquidCache { } } + /// Remove `entry_id` if the index still holds `observed`, the memory entry + /// an eviction picked. + /// + /// Another write can have replaced the key since, under the same identity + /// or another; that entry is newer than the victim and stays. The policy + /// hears of the removal only when this call made it. + fn drop_memory_victim(&self, entry_id: EntryID, observed: &CacheEntry) { + let Some(removed) = self + .index + .remove_if(&entry_id, |slot| same_memory_entry(&slot.entry, observed)) + else { + return; + }; + self.budget + .try_update_memory_usage(removed.entry.memory_usage_bytes(), 0) + .expect("memory release cannot fail"); + self.cache_policy.notify_remove(&entry_id); + // A write can have installed the key again, and told the policy, + // between the removal and the notice above, which then took the new + // entry out of the policy too. An entry the policy does not know is + // never evicted, so it is told again; a policy files a key once. + if let Some(slot) = self.index.get(&entry_id) { + self.cache_policy + .notify_insert(&entry_id, CachedBatchType::from(&slot.entry)); + } + } + /// Remove `entry_id` if the index still holds it in memory. /// /// The flush that calls this read the entry earlier. Another write can @@ -849,9 +917,24 @@ impl LiquidCache { // another identity takes the key over meanwhile, the rewrite still // carries the old identity, so the new owner reads it as a miss. let identity = slot.identity; + let observed = slot.entry.clone(); let mut victim_entry = slot.entry.clone(); self.trace(InternalEvent::EvictionVictim { entry: victim }); loop { + // A memory victim that the disk cannot take is dropped: dropping + // it is the only way to free its memory, and without it a full + // memory tier refuses every insert and keeps what it cached first. + // A later read misses and goes to the source, as the flush does. + // + // With no disk tier, a Liquid victim's next step is always the + // disk, so it is dropped before the policy serialises it for a + // write that cannot happen. + if self.config.max_disk_bytes() == 0 + && matches!(victim_entry, CacheEntry::MemoryLiquid(_)) + { + self.drop_memory_victim(victim, &observed); + break; + } let outcome = self .eviction_policy .evict(&victim_entry, self.metadata.lineage(&victim).as_deref()); @@ -862,10 +945,16 @@ impl LiquidCache { bytes_to_write, } => { let write = match bytes_to_write { - Some(bytes_to_write) => Some( - self.write_batch_to_disk(victim, identity, &new_batch, bytes_to_write) - .await?, - ), + Some(bytes_to_write) => match self + .write_batch_to_disk(victim, identity, &new_batch, bytes_to_write) + .await + { + Ok(write) => Some(write), + Err(CacheFull) => { + self.drop_memory_victim(victim, &observed); + break; + } + }, // The policy left the record on disk as it was. The // key may hold something newer by now, and the object // that record named may be gone, so nothing is @@ -1289,7 +1378,7 @@ mod tests { utils::{arrow_to_bytes, create_cache_store, create_test_array, create_test_arrow_array}, }; use crate::sync::thread; - use arrow::array::{Array, ArrayRef, Int32Array}; + use arrow::array::{Array, ArrayRef, Int32Array, Int64Array}; use std::future::Future; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1618,6 +1707,164 @@ mod tests { assert!(!cache.contains(&EntryID::from(900usize), 0)); } + fn int64_array(len: usize, seed: i64) -> ArrayRef { + Arc::new(Int64Array::from_iter_values( + (0..len as i64).map(|i| (i ^ (seed << 40)).wrapping_mul(0x1E37_79B9_7F4A_7C15)), + )) + } + + #[tokio::test] + async fn a_full_memory_only_cache_drops_old_entries_to_admit_new_ones() { + let budget = 1 << 20; + let cache = LiquidCacheBuilder::new() + .with_max_memory_bytes(budget) + .with_max_disk_bytes(0) + .with_eviction_policy(Box::new(TranscodeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build() + .await; + + for k in 0..200usize { + cache + .insert(EntryID::from(k), 0, int64_array(4096, k as i64)) + .await + .unwrap_or_else(|_| panic!("insert {k} was refused")); + assert!(cache.budget.memory_usage_bytes() <= budget); + } + assert!(!cache.contains(&EntryID::from(0usize), 0)); + assert!(cache.contains(&EntryID::from(199usize), 0)); + assert_eq!(cache.budget.disk_usage_bytes(), 0); + } + + #[tokio::test] + async fn an_entry_larger_than_the_memory_tier_leaves_the_cache_as_it_was() { + let cache = LiquidCacheBuilder::new() + .with_max_memory_bytes(1 << 20) + .with_max_disk_bytes(0) + .with_eviction_policy(Box::new(TranscodeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build() + .await; + for k in 0..40usize { + let _ = cache + .insert(EntryID::from(k), 0, int64_array(4096, k as i64)) + .await; + } + let entries_before = cache.index().entry_count(); + let usage_before = cache.budget.memory_usage_bytes(); + + let result = cache + .insert(EntryID::from(999usize), 0, int64_array(200_000, 999)) + .await; + + assert_eq!(result, Err(CacheFull)); + assert_eq!(cache.index().entry_count(), entries_before); + assert_eq!(cache.budget.memory_usage_bytes(), usage_before); + } + + #[tokio::test] + async fn an_eviction_does_not_drop_an_entry_written_after_it_picked_its_victim() { + let cache = LiquidCacheBuilder::new() + .with_max_memory_bytes(1 << 20) + .with_max_disk_bytes(0) + .build() + .await; + let key = EntryID::from(7usize); + cache.insert(key, 0, int64_array(64, 1)).await.unwrap(); + let observed = cache.index().get(&key).unwrap().entry.clone(); + + // Another write replaces the key, under this identity and then another. + for identity in [0u64, 1] { + let newer = int64_array(64, 2 + identity as i64); + cache + .insert_inner(key, identity, CacheEntry::memory_arrow(newer.clone())) + .await + .unwrap(); + cache.drop_memory_victim(key, &observed); + let slot = cache + .index() + .get(&key) + .expect("the newer entry was dropped"); + assert!(matches!(&slot.entry, CacheEntry::MemoryArrow(a) if Arc::ptr_eq(a, &newer))); + } + + // The entry the eviction picked is dropped, and its memory released. + let current = cache.index().get(&key).unwrap().entry.clone(); + cache.drop_memory_victim(key, ¤t); + assert!(cache.index().get(&key).is_none()); + assert_eq!(cache.budget.memory_usage_bytes(), 0); + } + + /// Many writers on a few keys of a full memory-only cache, so inserts, + /// replacements and evictions of one key overlap. Afterwards the budget + /// charges exactly what the index holds, and the cache still admits. + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn overlapping_writes_to_a_full_memory_only_cache_keep_its_accounting() { + let budget = 256 << 10; + let cache = Arc::new( + LiquidCacheBuilder::new() + .with_max_memory_bytes(budget) + .with_max_disk_bytes(0) + .with_eviction_policy(Box::new(TranscodeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build() + .await, + ); + let mut tasks = Vec::new(); + for writer in 0..16i64 { + let cache = Arc::clone(&cache); + tasks.push(tokio::spawn(async move { + for round in 0..400i64 { + let key = EntryID::from(((writer * 3 + round) % 8) as usize); + let len = 512 + ((writer + round) % 4) as usize * 512; + let _ = cache + .insert( + key, + (round % 2) as u64, + int64_array(len, writer * 1000 + round), + ) + .await; + tokio::task::yield_now().await; + } + })); + } + for task in tasks { + task.await.unwrap(); + } + + let (mut held, mut largest) = (0, 0); + cache.for_each_entry(|_, entry| { + held += entry.memory_usage_bytes(); + largest = largest.max(entry.memory_usage_bytes()); + }); + assert_eq!(cache.budget.memory_usage_bytes(), held); + // A settle can charge an entry over a full budget: at most one entry + // for each writer whose insert raced the removal of its key. + assert!(held <= budget + 16 * largest); + + let fresh = EntryID::from(10_000usize); + cache + .insert(fresh, 0, int64_array(1024, 77)) + .await + .expect("a fresh entry was refused"); + assert!(cache.contains(&fresh, 0)); + + // Every entry the index holds is one the policy can still pick: an + // entry it lost is never evicted. + let known: std::collections::HashSet = cache + .cache_policy + .find_memory_victim(100_000) + .into_iter() + .map(usize::from) + .collect(); + cache.for_each_entry(|id, _| { + assert!( + known.contains(&usize::from(*id)), + "entry {id:?} is cached but unknown to the policy" + ); + }); + } + #[tokio::test] async fn insert_until_disk_full_then_evicts_oldest_disk_entry() { let first_array: ArrayRef = Arc::new(Int32Array::from_iter_values(0..16)); @@ -1753,6 +2000,81 @@ mod tests { } } + type BeforeRemoveHook = Box; + + /// A [`LiquidPolicy`] that runs a hook once, when told of a removal and + /// before it acts on the notice. + struct HookBeforeRemove { + inner: LiquidPolicy, + hook: Arc>>, + } + + impl std::fmt::Debug for HookBeforeRemove { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HookBeforeRemove").finish() + } + } + + impl CachePolicy for HookBeforeRemove { + fn find_memory_victim(&self, cnt: usize) -> Vec { + self.inner.find_memory_victim(cnt) + } + + fn notify_insert(&self, entry_id: &EntryID, batch_type: CachedBatchType) { + self.inner.notify_insert(entry_id, batch_type); + } + + fn notify_remove(&self, entry_id: &EntryID) { + let hook = self.hook.lock().unwrap().take(); + if let Some(hook) = hook { + hook(); + } + self.inner.notify_remove(entry_id); + } + } + + /// A write installs the key again, and tells the policy, between a + /// victim's removal and the removal's notice to the policy. The notice + /// then takes the new entry out of the policy too; the drop files it + /// again, so it stays evictable. + #[tokio::test] + async fn a_key_written_again_while_its_victim_is_dropped_stays_evictable() { + let hook = Arc::new(std::sync::Mutex::new(None::)); + let cache = Arc::new( + LiquidCacheBuilder::new() + .with_max_memory_bytes(1 << 20) + .with_max_disk_bytes(0) + .with_cache_policy(Box::new(HookBeforeRemove { + inner: LiquidPolicy::new(), + hook: hook.clone(), + })) + .build() + .await, + ); + let key = EntryID::from(5usize); + cache.insert(key, 0, int64_array(64, 1)).await.unwrap(); + let observed = cache.index().get(&key).unwrap().entry.clone(); + + let writer = Arc::clone(&cache); + let newer = int64_array(64, 2); + let installed = newer.clone(); + *hook.lock().unwrap() = Some(Box::new(move || { + let Ok(Orphan(None)) = + writer.try_insert(key, 0, CacheEntry::memory_arrow(installed), None) + else { + panic!("the write was refused or displaced a disk record"); + }; + })); + + cache.drop_memory_victim(key, &observed); + + assert!(hook.lock().unwrap().is_none(), "the hook never ran"); + let slot = cache.index().get(&key).expect("the newer entry is cached"); + assert!(matches!(&slot.entry, CacheEntry::MemoryArrow(a) if Arc::ptr_eq(a, &newer))); + assert_eq!(cache.cache_policy.find_memory_victim(8), vec![key]); + assert_memory_budget_matches_index(&cache); + } + /// The memory budget equals what the index's memory entries hold. fn assert_memory_budget_matches_index(cache: &LiquidCache) { let mut held = 0; diff --git a/src/core/src/cache/policies/cache/mod.rs b/src/core/src/cache/policies/cache/mod.rs index f2f2ef6e..3464a313 100644 --- a/src/core/src/cache/policies/cache/mod.rs +++ b/src/core/src/cache/policies/cache/mod.rs @@ -19,6 +19,9 @@ pub trait CachePolicy: std::fmt::Debug + Send + Sync { } /// Notify the cache policy that an entry was inserted. + /// + /// The cache can tell the policy of the same key again, in any order + /// with other notices of it, so this must file the key once (an upsert). fn notify_insert(&self, _entry_id: &EntryID, _batch_type: CachedBatchType) {} /// Notify the cache policy that an entry was accessed. diff --git a/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap b/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap index 14eedaf8..f494f2e9 100644 --- a/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap +++ b/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap @@ -5,15 +5,6 @@ expression: trace EventTrace: [ event=insert_success entry=0 kind=MemoryArrow event=insert_failed entry=1 kind=MemoryArrow -event=eviction_begin victims=[0] -event=eviction_victim entry=0 -event=insert_success entry=0 kind=MemoryLiquid -event=insert_failed entry=1 kind=MemoryArrow -event=eviction_begin victims=[0] -event=eviction_victim entry=0 -event=io_write entry=0 kind=DiskLiquid bytes=[bytes] -event=insert_success entry=0 kind=DiskLiquid -event=insert_failed entry=1 kind=MemoryArrow event=insert_failed entry=1 kind=MemoryLiquid event=io_write entry=1 kind=MemoryLiquid bytes=[bytes] event=insert_success entry=1 kind=DiskLiquid diff --git a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap index 1113f7c9..c0b3c558 100644 --- a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap +++ b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap @@ -21,9 +21,9 @@ plan: stats: entries.total: 8 entries.after_first_run: 8 -entries.memory.arrow: 5 -entries.memory.liquid: 1 -entries.disk.liquid: 2 +entries.memory.arrow: 2 +entries.memory.liquid: 6 +entries.disk.liquid: 0 entries.disk.arrow: 0 usage.memory_bytes: [bytes] usage.disk_bytes: [bytes] @@ -32,7 +32,7 @@ RuntimeStatsSnapshot: get_with_selection: 4 eval_predicate: 2 try_read_liquid_calls: 0 - read_io_count: 0 + read_io_count: 1 write_io_count: 0 disk_evictions: 0 disk_reservation_failures: 0 @@ -170,15 +170,15 @@ plan: stats: entries.total: 8 entries.after_first_run: 8 -entries.memory.arrow: 5 -entries.memory.liquid: 1 -entries.disk.liquid: 2 +entries.memory.arrow: 2 +entries.memory.liquid: 6 +entries.disk.liquid: 0 entries.disk.arrow: 0 usage.memory_bytes: [bytes] usage.disk_bytes: [bytes] RuntimeStatsSnapshot: - get: 5 - get_with_selection: 5 + get: 3 + get_with_selection: 3 eval_predicate: 0 try_read_liquid_calls: 3 read_io_count: 0 @@ -209,8 +209,8 @@ plan: stats: entries.total: 8 entries.after_first_run: 8 -entries.memory.arrow: 5 -entries.memory.liquid: 3 +entries.memory.arrow: 2 +entries.memory.liquid: 6 entries.disk.liquid: 0 entries.disk.arrow: 0 usage.memory_bytes: [bytes] diff --git a/src/datafusion-local/tests/memory_only_freeze.rs b/src/datafusion-local/tests/memory_only_freeze.rs new file mode 100644 index 00000000..88a3a4c2 --- /dev/null +++ b/src/datafusion-local/tests/memory_only_freeze.rs @@ -0,0 +1,249 @@ +//! Does a cache with no disk tier keep admitting new data once it is full? +//! +//! `LiquidCacheLocalBuilder` with the builder's default policies, the disk +//! tier set to zero, and an admission gate of expansion 4, safety 1, +//! tolerance 1, strict. +//! +//! Eviction moves an Arrow entry to Liquid in memory, and a Liquid entry to +//! disk. With no disk, a Liquid victim is dropped instead, so a full memory +//! tier still makes room. The scenario: queries over a few tables (A) fill the +//! cache, then queries over another table (B) run repeatedly. Every scan +//! passes the admission gate on its own; together they exceed the budget. +//! +//! This is an integration test so the crates build as they do in production +//! (concurrent eviction is on outside `cfg(test)`). + +use std::collections::HashSet; +use std::path::Path; +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int64Array}; +use arrow::record_batch::RecordBatch; +use arrow_schema::{DataType, Field, Schema}; +use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext}; +use liquid_cache::cache::EntryID; +use liquid_cache_datafusion::LiquidCacheParquetRef; +use liquid_cache_datafusion_local::LiquidCacheLocalBuilder; +use parquet::arrow::ArrowWriter; + +/// The cache's memory budget. +const MEMORY_BYTES: usize = 8 << 20; +/// Rows per table. One Int64 column of values that do not compress, so a +/// table is about 2 MiB of parquet, of Arrow, and of Liquid alike: a scan's +/// gate estimate (parquet bytes × 4) stays just under the budget. +const ROWS: u64 = 200_000; +/// Tables queried first, to fill the cache. Together they hold more than the +/// budget. +const FILL_TABLES: usize = 6; +/// Times each phase queries each of its tables. +const PASSES: usize = 3; +/// Cache entries one table makes: one per batch of the builder's default +/// batch size (8192 rows). +const TABLE_BATCHES: usize = ROWS.div_ceil(8192) as usize; + +/// The admission gate. +const EXPANSION: f64 = 4.0; +const SAFETY: f64 = 1.0; +const TOLERANCE: f64 = 1.0; +const STRICT: bool = true; + +fn write_table(path: &Path, seed: u64) { + let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)])); + let values: ArrayRef = Arc::new(Int64Array::from_iter_values((0..ROWS).map(|i| { + (i.wrapping_add(seed << 40)) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .rotate_left(17) as i64 + }))); + let batch = RecordBatch::try_new(schema.clone(), vec![values]).unwrap(); + let mut writer = + ArrowWriter::try_new(std::fs::File::create(path).unwrap(), schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +/// What the cache held after a phase, and what the phase asked of it. +#[derive(Debug, Default)] +struct Phase { + memory_usage_bytes: usize, + memory_arrow_entries: usize, + memory_liquid_entries: usize, + disk_entries: usize, + /// Entries cached for the fill tables still held. + fill_entries_held: usize, + /// Entries cached for table B held. + b_entries_held: usize, + /// Inserts the cache refused: an entry whose insert failed and never + /// succeeded within the same query, summed over the phase's queries. + refused_inserts: usize, +} + +fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { + let mut ids = HashSet::new(); + cache.storage().for_each_entry(|id: &EntryID, _| { + ids.insert(usize::from(*id)); + }); + ids +} + +/// Inserts refused during the queries since the trace was last drained. +fn refused_inserts(cache: &LiquidCacheParquetRef) -> usize { + let trace = format!("{}", cache.consume_event_trace()); + let mut failed = HashSet::new(); + let mut succeeded = HashSet::new(); + for line in trace.lines() { + let entry = || { + line.split_whitespace() + .find_map(|field| field.strip_prefix("entry=")) + .unwrap() + .to_string() + }; + if line.starts_with("event=insert_failed") { + failed.insert(entry()); + } else if line.starts_with("event=insert_success") { + succeeded.insert(entry()); + } + } + failed.difference(&succeeded).count() +} + +async fn query(ctx: &SessionContext, table: &str) { + let batches = ctx + .sql(&format!("SELECT SUM(v) AS s, COUNT(*) AS n FROM {table}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 1); +} + +fn snapshot( + cache: &LiquidCacheParquetRef, + fill_ids: &HashSet, + refused_inserts: usize, +) -> Phase { + let stats = cache.storage().stats(); + let ids = entry_ids(cache); + Phase { + memory_usage_bytes: stats.memory_usage_bytes, + memory_arrow_entries: stats.memory_arrow_entries, + memory_liquid_entries: stats.memory_liquid_entries, + disk_entries: stats.disk_arrow_entries + stats.disk_liquid_entries, + fill_entries_held: ids.intersection(fill_ids).count(), + b_entries_held: ids.difference(fill_ids).count(), + refused_inserts, + } +} + +/// Fill the cache from the fill tables, then query table B. Returns the cache +/// after the fill phase and after the B phase. +async fn fill_then_query_another_table(max_disk_bytes: usize) -> (Phase, Phase) { + let data = tempfile::tempdir().unwrap(); + let cache_dir = tempfile::tempdir().unwrap(); + let (ctx, cache) = LiquidCacheLocalBuilder::new() + .with_max_memory_bytes(MEMORY_BYTES) + .with_cache_dir(cache_dir.path().to_path_buf()) + .with_max_disk_bytes(max_disk_bytes) + .with_admission_gate(EXPANSION, SAFETY, TOLERANCE, STRICT) + .build(SessionConfig::new()) + .await + .unwrap(); + cache.enable_trace(); + + let fill: Vec = (0..FILL_TABLES).map(|t| format!("a{t}")).collect(); + for (seed, table) in fill + .iter() + .chain(std::iter::once(&"b".to_string())) + .enumerate() + { + let path = data.path().join(format!("{table}.parquet")); + write_table(&path, seed as u64); + let size = std::fs::metadata(&path).unwrap().len() as f64; + assert!( + size * EXPANSION * SAFETY <= MEMORY_BYTES as f64 * TOLERANCE, + "{table} ({size} bytes) would not pass the admission gate" + ); + ctx.register_parquet(table, path.to_str().unwrap(), ParquetReadOptions::default()) + .await + .unwrap(); + } + + let _ = cache.consume_event_trace(); + let mut refused = 0; + for _ in 0..PASSES { + for table in &fill { + query(&ctx, table).await; + refused += refused_inserts(&cache); + } + } + let fill_ids = entry_ids(&cache); + let after_fill = snapshot(&cache, &fill_ids, refused); + assert!( + after_fill.fill_entries_held > 0, + "the fill tables were not cached at all" + ); + + let mut refused = 0; + for _ in 0..PASSES { + query(&ctx, "b").await; + refused += refused_inserts(&cache); + } + let after_b = snapshot(&cache, &fill_ids, refused); + + println!( + "max_disk_bytes={max_disk_bytes} max_memory_bytes={MEMORY_BYTES}\n \ + after fill: {after_fill:?}\n after B: {after_b:?}" + ); + (after_fill, after_b) +} + +/// With no disk tier, a table queried after the cache filled is cached, as it +/// is with a disk tier: the entries cached first are dropped to make room. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_memory_only_cache_admits_a_new_table_once_full() { + let (after_fill, after_b) = fill_then_query_another_table(0).await; + + assert!( + after_fill.fill_entries_held < FILL_TABLES * TABLE_BATCHES, + "the cache holds every fill entry; the test does not fill the cache" + ); + assert!( + after_b.fill_entries_held < after_fill.fill_entries_held, + "no fill entry was dropped to make room for table B" + ); + assert_eq!( + after_b.refused_inserts, 0, + "inserts for table B were refused" + ); + assert_eq!( + after_b.b_entries_held, + TABLE_BATCHES, + "a memory-only cache full of Liquid entries cached {} of table B's \ + {TABLE_BATCHES} batches: {} inserts refused over {PASSES} queries, \ + while the fill tables keep {} entries ({} Liquid, {} Arrow, {} of \ + {MEMORY_BYTES} bytes)", + after_b.b_entries_held, + after_b.refused_inserts, + after_b.fill_entries_held, + after_b.memory_liquid_entries, + after_b.memory_arrow_entries, + after_b.memory_usage_bytes, + ); +} + +/// The control: with a disk tier (the builder's default of ten times +/// memory), Liquid entries spill to disk and table B is cached. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_cache_with_a_disk_tier_admits_a_new_table_once_full() { + let (after_fill, after_b) = fill_then_query_another_table(10 * MEMORY_BYTES).await; + + assert!( + after_fill.disk_entries > 0, + "the fill tables never reached disk" + ); + assert_eq!( + after_b.b_entries_held, TABLE_BATCHES, + "table B was not cached" + ); + assert_eq!(after_b.refused_inserts, 0); +} diff --git a/src/datafusion-local/tests/memory_only_workloads.rs b/src/datafusion-local/tests/memory_only_workloads.rs new file mode 100644 index 00000000..026b2d82 --- /dev/null +++ b/src/datafusion-local/tests/memory_only_workloads.rs @@ -0,0 +1,340 @@ +//! Workloads against a cache with no disk tier, once its memory is full. +//! +//! Each test builds the cache as a memory-only deployment would: +//! `LiquidCacheLocalBuilder` with the builder's default policies, the disk +//! tier set to zero, and an admission gate of expansion 4, safety 1, +//! tolerance 1, strict. Every scan passes the gate on its own; together the +//! tables hold several times the budget, so the cache turns over. +//! +//! Every query's result is checked against DataFusion reading the same files +//! with no cache, so a query that silently returns wrong rows fails as surely +//! as one that errors. +//! +//! These are integration tests so the crates build as they do in production +//! (concurrent eviction is on outside `cfg(test)`). + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ArrayRef, BooleanArray, Int64Array, StringArray, TimestampMicrosecondArray}; +use arrow::record_batch::RecordBatch; +use arrow::util::pretty::pretty_format_batches; +use arrow_schema::{DataType, Field, Schema, TimeUnit}; +use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext}; +use liquid_cache_datafusion::LiquidCacheParquetRef; +use liquid_cache_datafusion_local::LiquidCacheLocalBuilder; +use parquet::arrow::ArrowWriter; + +/// The cache's memory budget. +const MEMORY_BYTES: usize = 8 << 20; +/// Rows per table: three columns, about 1.4 MiB of parquet, so a scan's gate +/// estimate (parquet bytes × 4) stays under the budget. +const ROWS: u64 = 60_000; + +/// The type of a table's first column, `p`, which the filters read. +#[derive(Clone, Copy, Debug)] +enum Kind { + /// Timestamp with a time zone, which Liquid does not encode. + TimestampTz, + /// Boolean, which Liquid does not encode. + Boolean, + /// Int64, which Liquid encodes. + Int64, + /// Utf8, which Liquid encodes. + Utf8, +} + +impl Kind { + /// A filter on `p` that keeps about half the rows. + fn predicate(self) -> &'static str { + match self { + Kind::TimestampTz => "p > TIMESTAMP '2033-01-01T00:00:00Z'", + Kind::Boolean => "p", + Kind::Int64 => "p > 0", + Kind::Utf8 => "p > 'm'", + } + } +} + +fn mix(i: u64, seed: u64) -> i64 { + (i.wrapping_add(seed << 40)) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .rotate_left(17) as i64 +} + +fn write_table(path: &Path, seed: u64, kind: Kind) { + let (p_field, p): (Field, ArrayRef) = match kind { + Kind::TimestampTz => ( + Field::new( + "p", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Arc::new( + TimestampMicrosecondArray::from_iter_values( + (0..ROWS).map(|i| mix(i, seed + 99).rem_euclid(4_000_000_000_000_000)), + ) + .with_timezone("UTC"), + ), + ), + Kind::Boolean => ( + Field::new("p", DataType::Boolean, false), + Arc::new(BooleanArray::from_iter( + (0..ROWS).map(|i| Some(mix(i, seed + 99) & 1 == 0)), + )), + ), + Kind::Int64 => ( + Field::new("p", DataType::Int64, false), + Arc::new(Int64Array::from_iter_values( + (0..ROWS).map(|i| mix(i, seed + 99)), + )), + ), + Kind::Utf8 => ( + Field::new("p", DataType::Utf8, false), + Arc::new(StringArray::from_iter_values((0..ROWS).map(|i| { + let letter = (b'a' + mix(i, seed + 99).rem_euclid(26) as u8) as char; + format!("{letter}{}", mix(i, seed + 5).rem_euclid(1000)) + }))), + ), + }; + let schema = Arc::new(Schema::new(vec![ + p_field, + Field::new("v", DataType::Int64, false), + Field::new("w", DataType::Int64, false), + ])); + let v: ArrayRef = Arc::new(Int64Array::from_iter_values( + (0..ROWS).map(|i| mix(i, seed)), + )); + let w: ArrayRef = Arc::new(Int64Array::from_iter_values( + (0..ROWS).map(|i| mix(i, seed + 7)), + )); + let batch = RecordBatch::try_new(schema.clone(), vec![p, v, w]).unwrap(); + let mut writer = + ArrowWriter::try_new(std::fs::File::create(path).unwrap(), schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +struct Fixture { + cached: SessionContext, + plain: SessionContext, + cache: LiquidCacheParquetRef, + tables: Vec<(String, Kind)>, + _data: tempfile::TempDir, + _cache_dir: tempfile::TempDir, +} + +/// `tables` tables, their kinds cycling through `kinds`, registered with a +/// memory-only cache and with a plain context to check results against. +async fn fixture(tables: usize, kinds: &[Kind]) -> Fixture { + let data = tempfile::tempdir().unwrap(); + let cache_dir = tempfile::tempdir().unwrap(); + let (cached, cache) = LiquidCacheLocalBuilder::new() + .with_max_memory_bytes(MEMORY_BYTES) + .with_cache_dir(cache_dir.path().to_path_buf()) + .with_max_disk_bytes(0) + .with_admission_gate(4.0, 1.0, 1.0, true) + .build(SessionConfig::new()) + .await + .unwrap(); + let plain = SessionContext::new(); + let mut names = Vec::new(); + for t in 0..tables { + let kind = kinds[t % kinds.len()]; + let name = format!("t{t}"); + let path: PathBuf = data.path().join(format!("{name}.parquet")); + write_table(&path, t as u64, kind); + let size = std::fs::metadata(&path).unwrap().len() as usize; + assert!( + size * 4 <= MEMORY_BYTES, + "{name} ({size} bytes) would not pass the admission gate" + ); + for ctx in [&cached, &plain] { + ctx.register_parquet(&name, path.to_str().unwrap(), ParquetReadOptions::default()) + .await + .unwrap(); + } + names.push((name, kind)); + } + Fixture { + cached, + plain, + cache, + tables: names, + _data: data, + _cache_dir: cache_dir, + } +} + +/// The SQL each test runs against a table: a filter on `p` and on the two +/// Int64 columns, so a batch's predicate columns and its projected columns +/// are cached one after another. +fn sql(table: &str, kind: Kind) -> String { + format!( + "SELECT COUNT(*) AS n, SUM(v) AS sv, MIN(w) AS mw FROM {table} \ + WHERE {} AND v > -4000000000000000000 AND w < 4000000000000000000", + kind.predicate() + ) +} + +async fn run(ctx: &SessionContext, sql: &str) -> datafusion::error::Result { + let batches = ctx.sql(sql).await?.collect().await?; + Ok(pretty_format_batches(&batches)?.to_string()) +} + +/// Run `table`'s query on the cache and check it against the plain context. +async fn query_and_check(f: &Fixture, table: &str, kind: Kind) { + let sql = sql(table, kind); + let got = run(&f.cached, &sql) + .await + .unwrap_or_else(|e| panic!("{table} ({kind:?}) failed on the cache: {e}")); + let want = run(&f.plain, &sql).await.unwrap(); + assert_eq!(got, want, "{table} ({kind:?}) returned a different result"); +} + +fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { + let mut ids = HashSet::new(); + cache.storage().for_each_entry(|id, _| { + ids.insert(usize::from(*id)); + }); + ids +} + +/// The cache charges exactly the memory its entries hold. Called once no query +/// is running. +fn assert_accounting_exact(cache: &LiquidCacheParquetRef, after: &str) -> usize { + let used = cache.storage().stats().memory_usage_bytes; + let mut held = 0; + cache + .storage() + .for_each_entry(|_, entry| held += entry.memory_usage_bytes()); + assert_eq!( + used, held, + "after {after} the cache charges {used} bytes for entries holding {held}" + ); + used +} + +/// The accounting is exact and within the budget. An insert that raced the +/// removal of its key can charge one entry over a full budget, until a later +/// insert that grows evicts; so after concurrent queries, call this only +/// after a sequential query has run. +fn assert_within_budget(cache: &LiquidCacheParquetRef, after: &str) { + let used = assert_accounting_exact(cache, after); + assert!( + used <= MEMORY_BYTES, + "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES}" + ); +} + +/// A filter that reads a column Liquid cannot encode used to fail once the +/// cache was full: filling the cache with a batch's later columns dropped its +/// earlier ones, and the predicate found nothing to read. Every column type +/// must keep its queries correct. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn filtered_queries_on_every_column_type_stay_correct_once_full() { + for kind in [Kind::TimestampTz, Kind::Boolean, Kind::Int64, Kind::Utf8] { + let f = fixture(16, &[kind]).await; + for _pass in 0..3 { + for (table, kind) in f.tables.clone() { + query_and_check(&f, &table, kind).await; + assert_within_budget(&f.cache, &table); + } + } + } +} + +/// Many more tables than the cache holds, queried in turn for several passes: +/// the steady state of a long-running deployment. The table queried last must +/// always be cached, however long the cache has been full. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_long_rotation_keeps_caching_the_table_queried_last() { + let kinds = [Kind::Int64, Kind::Utf8, Kind::TimestampTz, Kind::Boolean]; + let f = fixture(24, &kinds).await; + let mut cached_after_query = Vec::new(); + for pass in 0..4 { + for (table, kind) in f.tables.clone() { + let before = entry_ids(&f.cache); + query_and_check(&f, &table, kind).await; + assert_within_budget(&f.cache, &table); + let after = entry_ids(&f.cache); + if pass > 0 { + cached_after_query.push((table.clone(), after.difference(&before).count())); + } + } + } + let stalled: Vec<_> = cached_after_query + .iter() + .filter(|(_, new_entries)| *new_entries == 0) + .collect(); + assert!( + stalled.is_empty(), + "once full, these queries cached nothing of their table: {stalled:?}" + ); +} + +/// The same rotation from eight queries at once. Results stay correct, the +/// budget holds, and afterwards a table not queried yet is cached. +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn concurrent_queries_keep_the_cache_admitting() { + let kinds = [Kind::Int64, Kind::Utf8, Kind::TimestampTz, Kind::Boolean]; + let f = Arc::new(fixture(25, &kinds).await); + let (last, last_kind) = f.tables.last().cloned().unwrap(); + let mut tasks = Vec::new(); + for worker in 0..8 { + let f = Arc::clone(&f); + tasks.push(tokio::spawn(async move { + for round in 0..24 { + let (table, kind) = f.tables[(worker * 5 + round) % 24].clone(); + query_and_check(&f, &table, kind).await; + } + })); + } + for task in tasks { + task.await.unwrap(); + } + assert_accounting_exact(&f.cache, "the concurrent queries"); + + let before = entry_ids(&f.cache); + query_and_check(&f, &last, last_kind).await; + let new_entries = entry_ids(&f.cache).difference(&before).count(); + assert!( + new_entries > 0, + "after the concurrent queries, {last} cached nothing" + ); + assert_within_budget(&f.cache, &last); +} + +/// Queries cancelled part way, many times over, must not leave the cache +/// unable to make room: afterwards a table not queried yet is cached. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cancelled_queries_do_not_freeze_the_cache() { + let kinds = [Kind::Int64, Kind::Utf8, Kind::TimestampTz, Kind::Boolean]; + let f = Arc::new(fixture(13, &kinds).await); + let (last, last_kind) = f.tables.last().cloned().unwrap(); + for round in 0..120u64 { + let (table, kind) = f.tables[(round % 12) as usize].clone(); + let sql = sql(&table, kind); + let cached = f.cached.clone(); + let task = tokio::spawn(async move { run(&cached, &sql).await }); + // Cut each query off at a different point. + tokio::time::sleep(Duration::from_micros(200 + (round * 397) % 4000)).await; // sleep-ok: the cancellation point is the fixture + task.abort(); + let _ = task.await; + } + assert_accounting_exact(&f.cache, "the cancelled queries"); + + for (table, kind) in f.tables[..12].iter().cloned() { + query_and_check(&f, &table, kind).await; + } + let before = entry_ids(&f.cache); + query_and_check(&f, &last, last_kind).await; + let new_entries = entry_ids(&f.cache).difference(&before).count(); + assert!( + new_entries > 0, + "after the cancelled queries, {last} cached nothing" + ); + assert_within_budget(&f.cache, &last); +} diff --git a/src/datafusion/src/reader/plantime/morselizer.rs b/src/datafusion/src/reader/plantime/morselizer.rs index 6e9ee546..6e0d6f89 100644 --- a/src/datafusion/src/reader/plantime/morselizer.rs +++ b/src/datafusion/src/reader/plantime/morselizer.rs @@ -1375,6 +1375,16 @@ mod tests { } } + /// The last batch the scan read found the cache full, and was cached: + /// with no disk tier, older batches were dropped to make room for it. + async fn assert_full_cache_made_room( + row_group0: &CachedRowGroupRef, + row_group1: &CachedRowGroupRef, + ) { + assert!(contains(row_group1, 1, 0).await); + assert!(!contains(row_group0, 0, 0).await); + } + async fn contains(row_group: &CachedRowGroupRef, column_id: usize, batch_idx: u16) -> bool { row_group .get_column(column_id as u64) @@ -1595,7 +1605,7 @@ mod tests { } #[tokio::test] - async fn cache_full_keeps_inserted_batches_and_skips_failed_inserts() { + async fn cache_full_drops_the_oldest_batch_to_admit_the_newest() { let one_array_memory = Arc::new(Int32Array::from(vec![0, 1, 2, 3])).get_array_memory_size(); let planned = plan_test_file(PlanOptions { max_memory_bytes: one_array_memory * 3, @@ -1609,10 +1619,7 @@ mod tests { let (a, b) = collect_columns(planned.morsels).await; assert_eq!(a, vec![0, 1, 2, 3, 4, 5, 6, 7]); assert_eq!(b, vec![10, 11, 12, 13, 14, 15, 16, 17]); - assert!(contains(&row_group0, 0, 0).await); - assert!(contains(&row_group0, 1, 0).await); - assert!(contains(&row_group1, 0, 0).await); - assert!(!contains(&row_group1, 1, 0).await); + assert_full_cache_made_room(&row_group0, &row_group1).await; } #[tokio::test] @@ -1630,10 +1637,7 @@ mod tests { let (a, b) = collect_columns(planned.morsels).await; assert_eq!(a, vec![3, 4, 5, 6, 7]); assert_eq!(b, vec![13, 14, 15, 16, 17]); - assert!(contains(&row_group0, 0, 0).await); - assert!(contains(&row_group0, 1, 0).await); - assert!(contains(&row_group1, 0, 0).await); - assert!(!contains(&row_group1, 1, 0).await); + assert_full_cache_made_room(&row_group0, &row_group1).await; } #[tokio::test] @@ -1672,10 +1676,7 @@ mod tests { collect_columns(planned.morsels).await.0, vec![3, 4, 5, 6, 7] ); - assert!(contains(&row_group0, 0, 0).await); - assert!(contains(&row_group0, 1, 0).await); - assert!(contains(&row_group1, 0, 0).await); - assert!(!contains(&row_group1, 1, 0).await); + assert_full_cache_made_room(&row_group0, &row_group1).await; } #[tokio::test] diff --git a/src/datafusion/src/reader/runtime/liquid_cache_reader.rs b/src/datafusion/src/reader/runtime/liquid_cache_reader.rs index 2b6f55f1..db60a86e 100644 --- a/src/datafusion/src/reader/runtime/liquid_cache_reader.rs +++ b/src/datafusion/src/reader/runtime/liquid_cache_reader.rs @@ -7,10 +7,10 @@ use arrow::array::{Array, ArrayRef, BooleanArray, RecordBatch}; use arrow::buffer::BooleanBuffer; use arrow::compute::prep_null_mask_filter; use arrow::record_batch::RecordBatchOptions; -use arrow_schema::{ArrowError, SchemaRef}; +use arrow_schema::{ArrowError, Schema, SchemaRef}; use futures::{Stream, StreamExt, future::BoxFuture, stream::BoxStream}; use parquet::arrow::arrow_reader::{ - ArrowReaderMetadata, ArrowReaderOptions, RowSelection, RowSelector, + ArrowPredicate, ArrowReaderMetadata, ArrowReaderOptions, RowSelection, RowSelector, }; use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask}; use parquet::errors::ParquetError; @@ -331,18 +331,66 @@ impl LiquidCacheReaderInner { return Ok(selection); } - self.read_parquet_batch_and_fill_cache(self.current_batch_id) + let record_batch = self + .read_parquet_batch_and_fill_cache(self.current_batch_id) .await?; - apply_predicates( + if let Some(selection) = apply_predicates( &self.cached_row_group, self.current_batch_id, - input_selection, + input_selection.clone(), filter, ) .await? - .ok_or_else(|| { - ArrowError::ComputeError("predicate unavailable after materialization".to_string()) - }) + { + return Ok(selection); + } + // A full cache can evict this batch's own columns while it caches the + // rest of them, so the predicates fall back to the batch just read. + self.apply_predicates_to_batch(&record_batch, input_selection, filter) + } + + /// Evaluate `filter` over the rows of `record_batch` that `selection` + /// keeps, as [`apply_predicates`] does over the cache. + fn apply_predicates_to_batch( + &self, + record_batch: &RecordBatch, + mut selection: BooleanBuffer, + filter: &mut LiquidRowFilter, + ) -> Result { + for predicate in filter.predicates_mut() { + let selected_rows = selection.count_set_bits(); + if selected_rows == 0 { + break; + } + let mut fields = Vec::new(); + let mut arrays = Vec::new(); + for column_id in predicate.predicate_column_ids() { + let column = self + .cached_row_group + .get_column(column_id as u64) + .ok_or_else(|| { + ArrowError::ComputeError(format!( + "column {column_id} not present in liquid cache" + )) + })?; + fields.push(column.field()); + arrays.push(filter_array( + self.parquet_array(record_batch, column_id)?, + &selection, + )?); + } + let options = RecordBatchOptions::new().with_row_count(Some(selected_rows)); + let batch = + RecordBatch::try_new_with_options(Arc::new(Schema::new(fields)), arrays, &options)?; + let boolean_array = predicate.evaluate(batch)?; + let boolean_mask = if boolean_array.null_count() == 0 { + boolean_array.into_parts().0 + } else { + prep_null_mask_filter(&boolean_array).into_parts().0 + }; + selection = boolean_buffer_and_then(&selection, &boolean_mask); + } + Ok(selection) } #[fastrace::trace]