From f2baa45bed5a6741f63b63c2c7a9f8cb48c24c4d Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Wed, 18 Nov 2026 01:41:31 +0530 Subject: [PATCH 1/6] fix(cache): keep admitting once a memory-only cache is full With no disk tier, eviction spilled a Liquid victim to disk, the write failed with CacheFull, and the insert was refused without freeing anything. A full cache then kept what it cached first and refused every new entry. Eviction now drops a memory victim the disk cannot take, and drops a Liquid victim before serialising it when the disk tier is zero. The drop removes the slot only if it still holds the entry the eviction picked, so a newer write of the key survives. An entry larger than the whole memory tier no longer evicts anything before it is refused. Dropping victims means a batch's earlier columns can leave the cache while its later ones are inserted. The reader evaluated predicates from the cache after filling it, and failed with "predicate unavailable after materialization". It now falls back to the batch it just read. --- src/core/src/cache/core.rs | 157 ++++++++- ...ests__policies__insert_wont_fit_cache.snap | 9 - ...afusion_local__tests__provide_schema2.snap | 22 +- .../tests/memory_only_freeze.rs | 249 ++++++++++++++ .../tests/memory_only_workloads.rs | 320 ++++++++++++++++++ .../src/reader/plantime/morselizer.rs | 27 +- .../src/reader/runtime/liquid_cache_reader.rs | 64 +++- 7 files changed, 801 insertions(+), 47 deletions(-) create mode 100644 src/datafusion-local/tests/memory_only_freeze.rs create mode 100644 src/datafusion-local/tests/memory_only_workloads.rs diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 03350b31..41318719 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -220,6 +220,16 @@ pub enum PrefetchResult { /// Length of the identity trailer appended to every object written to disk. const IDENTITY_TRAILER_LEN: usize = std::mem::size_of::(); +/// Whether `slot` is the memory entry `observed`: the same array, not only an +/// equal one, so an entry written since is not mistaken for it. +fn same_memory_entry(slot: &CacheEntry, observed: &CacheEntry) -> bool { + match (slot, observed) { + (CacheEntry::MemoryArrow(a), CacheEntry::MemoryArrow(b)) => std::sync::Arc::ptr_eq(a, b), + (CacheEntry::MemoryLiquid(a), CacheEntry::MemoryLiquid(b)) => std::sync::Arc::ptr_eq(a, b), + _ => false, + } +} + /// Builder returned by [`LiquidCache::insert`] for configuring cache writes. impl LiquidCache { /// Return current cache statistics: counts and resource usage. @@ -554,7 +564,14 @@ impl LiquidCache { kind: CachedBatchType::from(¬_inserted), }); - let victims = self.cache_policy.find_memory_victim(8); + // No eviction makes room for an entry larger than the whole memory + // tier; evicting for it would only empty the cache before refusing + // it. It goes the way of an insert into an empty cache instead. + let victims = if not_inserted.memory_usage_bytes() > self.config.max_memory_bytes() { + Vec::new() + } else { + self.cache_policy.find_memory_victim(8) + }; if victims.is_empty() { // no advice, because the cache is already empty // this can happen if the entry to be inserted is too large, in that case, @@ -743,6 +760,25 @@ impl LiquidCache { } } + /// Remove `entry_id` if the index still holds `observed`, the memory entry + /// an eviction picked. + /// + /// Another write can have replaced the key since, under the same identity + /// or another; that entry is newer than the victim and stays. The policy + /// hears of the removal only when this call made it. + fn drop_memory_victim(&self, entry_id: EntryID, observed: &CacheEntry) { + let Some(removed) = self + .index + .remove_if(&entry_id, |slot| same_memory_entry(&slot.entry, observed)) + else { + return; + }; + self.budget + .try_update_memory_usage(removed.entry.memory_usage_bytes(), 0) + .expect("memory release cannot fail"); + self.cache_policy.notify_remove(&entry_id); + } + /// Remove `entry_id` if the index still holds it in memory. /// /// The flush that calls this read the entry earlier. Another write can @@ -849,9 +885,24 @@ impl LiquidCache { // another identity takes the key over meanwhile, the rewrite still // carries the old identity, so the new owner reads it as a miss. let identity = slot.identity; + let observed = slot.entry.clone(); let mut victim_entry = slot.entry.clone(); self.trace(InternalEvent::EvictionVictim { entry: victim }); loop { + // A memory victim that the disk cannot take is dropped: dropping + // it is the only way to free its memory, and without it a full + // memory tier refuses every insert and keeps what it cached first. + // A later read misses and goes to the source, as the flush does. + // + // With no disk tier, a Liquid victim's next step is always the + // disk, so it is dropped before the policy serialises it for a + // write that cannot happen. + if self.config.max_disk_bytes() == 0 + && matches!(victim_entry, CacheEntry::MemoryLiquid(_)) + { + self.drop_memory_victim(victim, &observed); + break; + } let outcome = self .eviction_policy .evict(&victim_entry, self.metadata.lineage(&victim).as_deref()); @@ -862,10 +913,16 @@ impl LiquidCache { bytes_to_write, } => { let write = match bytes_to_write { - Some(bytes_to_write) => Some( - self.write_batch_to_disk(victim, identity, &new_batch, bytes_to_write) - .await?, - ), + Some(bytes_to_write) => match self + .write_batch_to_disk(victim, identity, &new_batch, bytes_to_write) + .await + { + Ok(write) => Some(write), + Err(CacheFull) => { + self.drop_memory_victim(victim, &observed); + break; + } + }, // The policy left the record on disk as it was. The // key may hold something newer by now, and the object // that record named may be gone, so nothing is @@ -1289,7 +1346,7 @@ mod tests { utils::{arrow_to_bytes, create_cache_store, create_test_array, create_test_arrow_array}, }; use crate::sync::thread; - use arrow::array::{Array, ArrayRef, Int32Array}; + use arrow::array::{Array, ArrayRef, Int32Array, Int64Array}; use std::future::Future; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1618,6 +1675,94 @@ mod tests { assert!(!cache.contains(&EntryID::from(900usize), 0)); } + fn int64_array(len: usize, seed: i64) -> ArrayRef { + Arc::new(Int64Array::from_iter_values( + (0..len as i64).map(|i| (i ^ (seed << 40)).wrapping_mul(0x1E37_79B9_7F4A_7C15)), + )) + } + + #[tokio::test] + async fn a_full_memory_only_cache_drops_old_entries_to_admit_new_ones() { + let budget = 1 << 20; + let cache = LiquidCacheBuilder::new() + .with_max_memory_bytes(budget) + .with_max_disk_bytes(0) + .with_eviction_policy(Box::new(TranscodeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build() + .await; + + for k in 0..200usize { + cache + .insert(EntryID::from(k), 0, int64_array(4096, k as i64)) + .await + .unwrap_or_else(|_| panic!("insert {k} was refused")); + assert!(cache.budget.memory_usage_bytes() <= budget); + } + assert!(!cache.contains(&EntryID::from(0usize), 0)); + assert!(cache.contains(&EntryID::from(199usize), 0)); + assert_eq!(cache.budget.disk_usage_bytes(), 0); + } + + #[tokio::test] + async fn an_entry_larger_than_the_memory_tier_leaves_the_cache_as_it_was() { + let cache = LiquidCacheBuilder::new() + .with_max_memory_bytes(1 << 20) + .with_max_disk_bytes(0) + .with_eviction_policy(Box::new(TranscodeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build() + .await; + for k in 0..40usize { + let _ = cache + .insert(EntryID::from(k), 0, int64_array(4096, k as i64)) + .await; + } + let entries_before = cache.index().entry_count(); + let usage_before = cache.budget.memory_usage_bytes(); + + let result = cache + .insert(EntryID::from(999usize), 0, int64_array(200_000, 999)) + .await; + + assert_eq!(result, Err(CacheFull)); + assert_eq!(cache.index().entry_count(), entries_before); + assert_eq!(cache.budget.memory_usage_bytes(), usage_before); + } + + #[tokio::test] + async fn an_eviction_does_not_drop_an_entry_written_after_it_picked_its_victim() { + let cache = LiquidCacheBuilder::new() + .with_max_memory_bytes(1 << 20) + .with_max_disk_bytes(0) + .build() + .await; + let key = EntryID::from(7usize); + cache.insert(key, 0, int64_array(64, 1)).await.unwrap(); + let observed = cache.index().get(&key).unwrap().entry.clone(); + + // Another write replaces the key, under this identity and then another. + for identity in [0u64, 1] { + let newer = int64_array(64, 2 + identity as i64); + cache + .insert_inner(key, identity, CacheEntry::memory_arrow(newer.clone())) + .await + .unwrap(); + cache.drop_memory_victim(key, &observed); + let slot = cache + .index() + .get(&key) + .expect("the newer entry was dropped"); + assert!(matches!(&slot.entry, CacheEntry::MemoryArrow(a) if Arc::ptr_eq(a, &newer))); + } + + // The entry the eviction picked is dropped, and its memory released. + let current = cache.index().get(&key).unwrap().entry.clone(); + cache.drop_memory_victim(key, ¤t); + assert!(cache.index().get(&key).is_none()); + assert_eq!(cache.budget.memory_usage_bytes(), 0); + } + #[tokio::test] async fn insert_until_disk_full_then_evicts_oldest_disk_entry() { let first_array: ArrayRef = Arc::new(Int32Array::from_iter_values(0..16)); diff --git a/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap b/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap index 14eedaf8..f494f2e9 100644 --- a/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap +++ b/src/core/src/cache/tests/snapshots/liquid_cache__cache__tests__policies__insert_wont_fit_cache.snap @@ -5,15 +5,6 @@ expression: trace EventTrace: [ event=insert_success entry=0 kind=MemoryArrow event=insert_failed entry=1 kind=MemoryArrow -event=eviction_begin victims=[0] -event=eviction_victim entry=0 -event=insert_success entry=0 kind=MemoryLiquid -event=insert_failed entry=1 kind=MemoryArrow -event=eviction_begin victims=[0] -event=eviction_victim entry=0 -event=io_write entry=0 kind=DiskLiquid bytes=[bytes] -event=insert_success entry=0 kind=DiskLiquid -event=insert_failed entry=1 kind=MemoryArrow event=insert_failed entry=1 kind=MemoryLiquid event=io_write entry=1 kind=MemoryLiquid bytes=[bytes] event=insert_success entry=1 kind=DiskLiquid diff --git a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap index 1113f7c9..c0b3c558 100644 --- a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap +++ b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap @@ -21,9 +21,9 @@ plan: stats: entries.total: 8 entries.after_first_run: 8 -entries.memory.arrow: 5 -entries.memory.liquid: 1 -entries.disk.liquid: 2 +entries.memory.arrow: 2 +entries.memory.liquid: 6 +entries.disk.liquid: 0 entries.disk.arrow: 0 usage.memory_bytes: [bytes] usage.disk_bytes: [bytes] @@ -32,7 +32,7 @@ RuntimeStatsSnapshot: get_with_selection: 4 eval_predicate: 2 try_read_liquid_calls: 0 - read_io_count: 0 + read_io_count: 1 write_io_count: 0 disk_evictions: 0 disk_reservation_failures: 0 @@ -170,15 +170,15 @@ plan: stats: entries.total: 8 entries.after_first_run: 8 -entries.memory.arrow: 5 -entries.memory.liquid: 1 -entries.disk.liquid: 2 +entries.memory.arrow: 2 +entries.memory.liquid: 6 +entries.disk.liquid: 0 entries.disk.arrow: 0 usage.memory_bytes: [bytes] usage.disk_bytes: [bytes] RuntimeStatsSnapshot: - get: 5 - get_with_selection: 5 + get: 3 + get_with_selection: 3 eval_predicate: 0 try_read_liquid_calls: 3 read_io_count: 0 @@ -209,8 +209,8 @@ plan: stats: entries.total: 8 entries.after_first_run: 8 -entries.memory.arrow: 5 -entries.memory.liquid: 3 +entries.memory.arrow: 2 +entries.memory.liquid: 6 entries.disk.liquid: 0 entries.disk.arrow: 0 usage.memory_bytes: [bytes] diff --git a/src/datafusion-local/tests/memory_only_freeze.rs b/src/datafusion-local/tests/memory_only_freeze.rs new file mode 100644 index 00000000..88a3a4c2 --- /dev/null +++ b/src/datafusion-local/tests/memory_only_freeze.rs @@ -0,0 +1,249 @@ +//! Does a cache with no disk tier keep admitting new data once it is full? +//! +//! `LiquidCacheLocalBuilder` with the builder's default policies, the disk +//! tier set to zero, and an admission gate of expansion 4, safety 1, +//! tolerance 1, strict. +//! +//! Eviction moves an Arrow entry to Liquid in memory, and a Liquid entry to +//! disk. With no disk, a Liquid victim is dropped instead, so a full memory +//! tier still makes room. The scenario: queries over a few tables (A) fill the +//! cache, then queries over another table (B) run repeatedly. Every scan +//! passes the admission gate on its own; together they exceed the budget. +//! +//! This is an integration test so the crates build as they do in production +//! (concurrent eviction is on outside `cfg(test)`). + +use std::collections::HashSet; +use std::path::Path; +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int64Array}; +use arrow::record_batch::RecordBatch; +use arrow_schema::{DataType, Field, Schema}; +use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext}; +use liquid_cache::cache::EntryID; +use liquid_cache_datafusion::LiquidCacheParquetRef; +use liquid_cache_datafusion_local::LiquidCacheLocalBuilder; +use parquet::arrow::ArrowWriter; + +/// The cache's memory budget. +const MEMORY_BYTES: usize = 8 << 20; +/// Rows per table. One Int64 column of values that do not compress, so a +/// table is about 2 MiB of parquet, of Arrow, and of Liquid alike: a scan's +/// gate estimate (parquet bytes × 4) stays just under the budget. +const ROWS: u64 = 200_000; +/// Tables queried first, to fill the cache. Together they hold more than the +/// budget. +const FILL_TABLES: usize = 6; +/// Times each phase queries each of its tables. +const PASSES: usize = 3; +/// Cache entries one table makes: one per batch of the builder's default +/// batch size (8192 rows). +const TABLE_BATCHES: usize = ROWS.div_ceil(8192) as usize; + +/// The admission gate. +const EXPANSION: f64 = 4.0; +const SAFETY: f64 = 1.0; +const TOLERANCE: f64 = 1.0; +const STRICT: bool = true; + +fn write_table(path: &Path, seed: u64) { + let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)])); + let values: ArrayRef = Arc::new(Int64Array::from_iter_values((0..ROWS).map(|i| { + (i.wrapping_add(seed << 40)) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .rotate_left(17) as i64 + }))); + let batch = RecordBatch::try_new(schema.clone(), vec![values]).unwrap(); + let mut writer = + ArrowWriter::try_new(std::fs::File::create(path).unwrap(), schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +/// What the cache held after a phase, and what the phase asked of it. +#[derive(Debug, Default)] +struct Phase { + memory_usage_bytes: usize, + memory_arrow_entries: usize, + memory_liquid_entries: usize, + disk_entries: usize, + /// Entries cached for the fill tables still held. + fill_entries_held: usize, + /// Entries cached for table B held. + b_entries_held: usize, + /// Inserts the cache refused: an entry whose insert failed and never + /// succeeded within the same query, summed over the phase's queries. + refused_inserts: usize, +} + +fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { + let mut ids = HashSet::new(); + cache.storage().for_each_entry(|id: &EntryID, _| { + ids.insert(usize::from(*id)); + }); + ids +} + +/// Inserts refused during the queries since the trace was last drained. +fn refused_inserts(cache: &LiquidCacheParquetRef) -> usize { + let trace = format!("{}", cache.consume_event_trace()); + let mut failed = HashSet::new(); + let mut succeeded = HashSet::new(); + for line in trace.lines() { + let entry = || { + line.split_whitespace() + .find_map(|field| field.strip_prefix("entry=")) + .unwrap() + .to_string() + }; + if line.starts_with("event=insert_failed") { + failed.insert(entry()); + } else if line.starts_with("event=insert_success") { + succeeded.insert(entry()); + } + } + failed.difference(&succeeded).count() +} + +async fn query(ctx: &SessionContext, table: &str) { + let batches = ctx + .sql(&format!("SELECT SUM(v) AS s, COUNT(*) AS n FROM {table}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 1); +} + +fn snapshot( + cache: &LiquidCacheParquetRef, + fill_ids: &HashSet, + refused_inserts: usize, +) -> Phase { + let stats = cache.storage().stats(); + let ids = entry_ids(cache); + Phase { + memory_usage_bytes: stats.memory_usage_bytes, + memory_arrow_entries: stats.memory_arrow_entries, + memory_liquid_entries: stats.memory_liquid_entries, + disk_entries: stats.disk_arrow_entries + stats.disk_liquid_entries, + fill_entries_held: ids.intersection(fill_ids).count(), + b_entries_held: ids.difference(fill_ids).count(), + refused_inserts, + } +} + +/// Fill the cache from the fill tables, then query table B. Returns the cache +/// after the fill phase and after the B phase. +async fn fill_then_query_another_table(max_disk_bytes: usize) -> (Phase, Phase) { + let data = tempfile::tempdir().unwrap(); + let cache_dir = tempfile::tempdir().unwrap(); + let (ctx, cache) = LiquidCacheLocalBuilder::new() + .with_max_memory_bytes(MEMORY_BYTES) + .with_cache_dir(cache_dir.path().to_path_buf()) + .with_max_disk_bytes(max_disk_bytes) + .with_admission_gate(EXPANSION, SAFETY, TOLERANCE, STRICT) + .build(SessionConfig::new()) + .await + .unwrap(); + cache.enable_trace(); + + let fill: Vec = (0..FILL_TABLES).map(|t| format!("a{t}")).collect(); + for (seed, table) in fill + .iter() + .chain(std::iter::once(&"b".to_string())) + .enumerate() + { + let path = data.path().join(format!("{table}.parquet")); + write_table(&path, seed as u64); + let size = std::fs::metadata(&path).unwrap().len() as f64; + assert!( + size * EXPANSION * SAFETY <= MEMORY_BYTES as f64 * TOLERANCE, + "{table} ({size} bytes) would not pass the admission gate" + ); + ctx.register_parquet(table, path.to_str().unwrap(), ParquetReadOptions::default()) + .await + .unwrap(); + } + + let _ = cache.consume_event_trace(); + let mut refused = 0; + for _ in 0..PASSES { + for table in &fill { + query(&ctx, table).await; + refused += refused_inserts(&cache); + } + } + let fill_ids = entry_ids(&cache); + let after_fill = snapshot(&cache, &fill_ids, refused); + assert!( + after_fill.fill_entries_held > 0, + "the fill tables were not cached at all" + ); + + let mut refused = 0; + for _ in 0..PASSES { + query(&ctx, "b").await; + refused += refused_inserts(&cache); + } + let after_b = snapshot(&cache, &fill_ids, refused); + + println!( + "max_disk_bytes={max_disk_bytes} max_memory_bytes={MEMORY_BYTES}\n \ + after fill: {after_fill:?}\n after B: {after_b:?}" + ); + (after_fill, after_b) +} + +/// With no disk tier, a table queried after the cache filled is cached, as it +/// is with a disk tier: the entries cached first are dropped to make room. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_memory_only_cache_admits_a_new_table_once_full() { + let (after_fill, after_b) = fill_then_query_another_table(0).await; + + assert!( + after_fill.fill_entries_held < FILL_TABLES * TABLE_BATCHES, + "the cache holds every fill entry; the test does not fill the cache" + ); + assert!( + after_b.fill_entries_held < after_fill.fill_entries_held, + "no fill entry was dropped to make room for table B" + ); + assert_eq!( + after_b.refused_inserts, 0, + "inserts for table B were refused" + ); + assert_eq!( + after_b.b_entries_held, + TABLE_BATCHES, + "a memory-only cache full of Liquid entries cached {} of table B's \ + {TABLE_BATCHES} batches: {} inserts refused over {PASSES} queries, \ + while the fill tables keep {} entries ({} Liquid, {} Arrow, {} of \ + {MEMORY_BYTES} bytes)", + after_b.b_entries_held, + after_b.refused_inserts, + after_b.fill_entries_held, + after_b.memory_liquid_entries, + after_b.memory_arrow_entries, + after_b.memory_usage_bytes, + ); +} + +/// The control: with a disk tier (the builder's default of ten times +/// memory), Liquid entries spill to disk and table B is cached. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_cache_with_a_disk_tier_admits_a_new_table_once_full() { + let (after_fill, after_b) = fill_then_query_another_table(10 * MEMORY_BYTES).await; + + assert!( + after_fill.disk_entries > 0, + "the fill tables never reached disk" + ); + assert_eq!( + after_b.b_entries_held, TABLE_BATCHES, + "table B was not cached" + ); + assert_eq!(after_b.refused_inserts, 0); +} diff --git a/src/datafusion-local/tests/memory_only_workloads.rs b/src/datafusion-local/tests/memory_only_workloads.rs new file mode 100644 index 00000000..df84df35 --- /dev/null +++ b/src/datafusion-local/tests/memory_only_workloads.rs @@ -0,0 +1,320 @@ +//! Workloads against a cache with no disk tier, once its memory is full. +//! +//! Each test builds the cache as a memory-only deployment would: +//! `LiquidCacheLocalBuilder` with the builder's default policies, the disk +//! tier set to zero, and an admission gate of expansion 4, safety 1, +//! tolerance 1, strict. Every scan passes the gate on its own; together the +//! tables hold several times the budget, so the cache turns over. +//! +//! Every query's result is checked against DataFusion reading the same files +//! with no cache, so a query that silently returns wrong rows fails as surely +//! as one that errors. +//! +//! These are integration tests so the crates build as they do in production +//! (concurrent eviction is on outside `cfg(test)`). + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ArrayRef, BooleanArray, Int64Array, StringArray, TimestampMicrosecondArray}; +use arrow::record_batch::RecordBatch; +use arrow::util::pretty::pretty_format_batches; +use arrow_schema::{DataType, Field, Schema, TimeUnit}; +use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext}; +use liquid_cache_datafusion::LiquidCacheParquetRef; +use liquid_cache_datafusion_local::LiquidCacheLocalBuilder; +use parquet::arrow::ArrowWriter; + +/// The cache's memory budget. +const MEMORY_BYTES: usize = 8 << 20; +/// Rows per table: three columns, about 1.4 MiB of parquet, so a scan's gate +/// estimate (parquet bytes × 4) stays under the budget. +const ROWS: u64 = 60_000; + +/// The type of a table's first column, `p`, which the filters read. +#[derive(Clone, Copy, Debug)] +enum Kind { + /// Timestamp with a time zone, which Liquid does not encode. + TimestampTz, + /// Boolean, which Liquid does not encode. + Boolean, + /// Int64, which Liquid encodes. + Int64, + /// Utf8, which Liquid encodes. + Utf8, +} + +impl Kind { + /// A filter on `p` that keeps about half the rows. + fn predicate(self) -> &'static str { + match self { + Kind::TimestampTz => "p > TIMESTAMP '2033-01-01T00:00:00Z'", + Kind::Boolean => "p", + Kind::Int64 => "p > 0", + Kind::Utf8 => "p > 'm'", + } + } +} + +fn mix(i: u64, seed: u64) -> i64 { + (i.wrapping_add(seed << 40)) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .rotate_left(17) as i64 +} + +fn write_table(path: &Path, seed: u64, kind: Kind) { + let (p_field, p): (Field, ArrayRef) = match kind { + Kind::TimestampTz => ( + Field::new( + "p", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Arc::new( + TimestampMicrosecondArray::from_iter_values( + (0..ROWS).map(|i| mix(i, seed + 99).rem_euclid(4_000_000_000_000_000)), + ) + .with_timezone("UTC"), + ), + ), + Kind::Boolean => ( + Field::new("p", DataType::Boolean, false), + Arc::new(BooleanArray::from_iter( + (0..ROWS).map(|i| Some(mix(i, seed + 99) & 1 == 0)), + )), + ), + Kind::Int64 => ( + Field::new("p", DataType::Int64, false), + Arc::new(Int64Array::from_iter_values( + (0..ROWS).map(|i| mix(i, seed + 99)), + )), + ), + Kind::Utf8 => ( + Field::new("p", DataType::Utf8, false), + Arc::new(StringArray::from_iter_values((0..ROWS).map(|i| { + let letter = (b'a' + mix(i, seed + 99).rem_euclid(26) as u8) as char; + format!("{letter}{}", mix(i, seed + 5).rem_euclid(1000)) + }))), + ), + }; + let schema = Arc::new(Schema::new(vec![ + p_field, + Field::new("v", DataType::Int64, false), + Field::new("w", DataType::Int64, false), + ])); + let v: ArrayRef = Arc::new(Int64Array::from_iter_values( + (0..ROWS).map(|i| mix(i, seed)), + )); + let w: ArrayRef = Arc::new(Int64Array::from_iter_values( + (0..ROWS).map(|i| mix(i, seed + 7)), + )); + let batch = RecordBatch::try_new(schema.clone(), vec![p, v, w]).unwrap(); + let mut writer = + ArrowWriter::try_new(std::fs::File::create(path).unwrap(), schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +struct Fixture { + cached: SessionContext, + plain: SessionContext, + cache: LiquidCacheParquetRef, + tables: Vec<(String, Kind)>, + _data: tempfile::TempDir, + _cache_dir: tempfile::TempDir, +} + +/// `tables` tables, their kinds cycling through `kinds`, registered with a +/// memory-only cache and with a plain context to check results against. +async fn fixture(tables: usize, kinds: &[Kind]) -> Fixture { + let data = tempfile::tempdir().unwrap(); + let cache_dir = tempfile::tempdir().unwrap(); + let (cached, cache) = LiquidCacheLocalBuilder::new() + .with_max_memory_bytes(MEMORY_BYTES) + .with_cache_dir(cache_dir.path().to_path_buf()) + .with_max_disk_bytes(0) + .with_admission_gate(4.0, 1.0, 1.0, true) + .build(SessionConfig::new()) + .await + .unwrap(); + let plain = SessionContext::new(); + let mut names = Vec::new(); + for t in 0..tables { + let kind = kinds[t % kinds.len()]; + let name = format!("t{t}"); + let path: PathBuf = data.path().join(format!("{name}.parquet")); + write_table(&path, t as u64, kind); + let size = std::fs::metadata(&path).unwrap().len() as usize; + assert!( + size * 4 <= MEMORY_BYTES, + "{name} ({size} bytes) would not pass the admission gate" + ); + for ctx in [&cached, &plain] { + ctx.register_parquet(&name, path.to_str().unwrap(), ParquetReadOptions::default()) + .await + .unwrap(); + } + names.push((name, kind)); + } + Fixture { + cached, + plain, + cache, + tables: names, + _data: data, + _cache_dir: cache_dir, + } +} + +/// The SQL each test runs against a table: a filter on `p` and on the two +/// Int64 columns, so a batch's predicate columns and its projected columns +/// are cached one after another. +fn sql(table: &str, kind: Kind) -> String { + format!( + "SELECT COUNT(*) AS n, SUM(v) AS sv, MIN(w) AS mw FROM {table} \ + WHERE {} AND v > -4000000000000000000 AND w < 4000000000000000000", + kind.predicate() + ) +} + +async fn run(ctx: &SessionContext, sql: &str) -> datafusion::error::Result { + let batches = ctx.sql(sql).await?.collect().await?; + Ok(pretty_format_batches(&batches)?.to_string()) +} + +/// Run `table`'s query on the cache and check it against the plain context. +async fn query_and_check(f: &Fixture, table: &str, kind: Kind) { + let sql = sql(table, kind); + let got = run(&f.cached, &sql) + .await + .unwrap_or_else(|e| panic!("{table} ({kind:?}) failed on the cache: {e}")); + let want = run(&f.plain, &sql).await.unwrap(); + assert_eq!(got, want, "{table} ({kind:?}) returned a different result"); +} + +fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { + let mut ids = HashSet::new(); + cache.storage().for_each_entry(|id, _| { + ids.insert(usize::from(*id)); + }); + ids +} + +fn assert_within_budget(cache: &LiquidCacheParquetRef, after: &str) { + let used = cache.storage().stats().memory_usage_bytes; + assert!( + used <= MEMORY_BYTES, + "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES}" + ); +} + +/// A filter that reads a column Liquid cannot encode used to fail once the +/// cache was full: filling the cache with a batch's later columns dropped its +/// earlier ones, and the predicate found nothing to read. Every column type +/// must keep its queries correct. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn filtered_queries_on_every_column_type_stay_correct_once_full() { + for kind in [Kind::TimestampTz, Kind::Boolean, Kind::Int64, Kind::Utf8] { + let f = fixture(16, &[kind]).await; + for _pass in 0..3 { + for (table, kind) in f.tables.clone() { + query_and_check(&f, &table, kind).await; + assert_within_budget(&f.cache, &table); + } + } + } +} + +/// Many more tables than the cache holds, queried in turn for several passes: +/// the steady state of a long-running deployment. The table queried last must +/// always be cached, however long the cache has been full. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_long_rotation_keeps_caching_the_table_queried_last() { + let kinds = [Kind::Int64, Kind::Utf8, Kind::TimestampTz, Kind::Boolean]; + let f = fixture(24, &kinds).await; + let mut cached_after_query = Vec::new(); + for pass in 0..4 { + for (table, kind) in f.tables.clone() { + let before = entry_ids(&f.cache); + query_and_check(&f, &table, kind).await; + assert_within_budget(&f.cache, &table); + let after = entry_ids(&f.cache); + if pass > 0 { + cached_after_query.push((table.clone(), after.difference(&before).count())); + } + } + } + let stalled: Vec<_> = cached_after_query + .iter() + .filter(|(_, new_entries)| *new_entries == 0) + .collect(); + assert!( + stalled.is_empty(), + "once full, these queries cached nothing of their table: {stalled:?}" + ); +} + +/// The same rotation from eight queries at once. Results stay correct, the +/// budget holds, and afterwards a table not queried yet is cached. +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn concurrent_queries_keep_the_cache_admitting() { + let kinds = [Kind::Int64, Kind::Utf8, Kind::TimestampTz, Kind::Boolean]; + let f = Arc::new(fixture(25, &kinds).await); + let (last, last_kind) = f.tables.last().cloned().unwrap(); + let mut tasks = Vec::new(); + for worker in 0..8 { + let f = Arc::clone(&f); + tasks.push(tokio::spawn(async move { + for round in 0..24 { + let (table, kind) = f.tables[(worker * 5 + round) % 24].clone(); + query_and_check(&f, &table, kind).await; + } + })); + } + for task in tasks { + task.await.unwrap(); + } + assert_within_budget(&f.cache, "the concurrent queries"); + + let before = entry_ids(&f.cache); + query_and_check(&f, &last, last_kind).await; + let new_entries = entry_ids(&f.cache).difference(&before).count(); + assert!( + new_entries > 0, + "after the concurrent queries, {last} cached nothing" + ); +} + +/// Queries cancelled part way, many times over, must not leave the cache +/// unable to make room: afterwards a table not queried yet is cached. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cancelled_queries_do_not_freeze_the_cache() { + let kinds = [Kind::Int64, Kind::Utf8, Kind::TimestampTz, Kind::Boolean]; + let f = Arc::new(fixture(13, &kinds).await); + let (last, last_kind) = f.tables.last().cloned().unwrap(); + for round in 0..120u64 { + let (table, kind) = f.tables[(round % 12) as usize].clone(); + let sql = sql(&table, kind); + let cached = f.cached.clone(); + let task = tokio::spawn(async move { run(&cached, &sql).await }); + // Cut each query off at a different point. + tokio::time::sleep(Duration::from_micros(200 + (round * 397) % 4000)).await; // sleep-ok: the cancellation point is the fixture + task.abort(); + let _ = task.await; + } + assert_within_budget(&f.cache, "the cancelled queries"); + + for (table, kind) in f.tables[..12].iter().cloned() { + query_and_check(&f, &table, kind).await; + } + let before = entry_ids(&f.cache); + query_and_check(&f, &last, last_kind).await; + let new_entries = entry_ids(&f.cache).difference(&before).count(); + assert!( + new_entries > 0, + "after the cancelled queries, {last} cached nothing" + ); + assert_within_budget(&f.cache, &last); +} diff --git a/src/datafusion/src/reader/plantime/morselizer.rs b/src/datafusion/src/reader/plantime/morselizer.rs index 6e9ee546..6e0d6f89 100644 --- a/src/datafusion/src/reader/plantime/morselizer.rs +++ b/src/datafusion/src/reader/plantime/morselizer.rs @@ -1375,6 +1375,16 @@ mod tests { } } + /// The last batch the scan read found the cache full, and was cached: + /// with no disk tier, older batches were dropped to make room for it. + async fn assert_full_cache_made_room( + row_group0: &CachedRowGroupRef, + row_group1: &CachedRowGroupRef, + ) { + assert!(contains(row_group1, 1, 0).await); + assert!(!contains(row_group0, 0, 0).await); + } + async fn contains(row_group: &CachedRowGroupRef, column_id: usize, batch_idx: u16) -> bool { row_group .get_column(column_id as u64) @@ -1595,7 +1605,7 @@ mod tests { } #[tokio::test] - async fn cache_full_keeps_inserted_batches_and_skips_failed_inserts() { + async fn cache_full_drops_the_oldest_batch_to_admit_the_newest() { let one_array_memory = Arc::new(Int32Array::from(vec![0, 1, 2, 3])).get_array_memory_size(); let planned = plan_test_file(PlanOptions { max_memory_bytes: one_array_memory * 3, @@ -1609,10 +1619,7 @@ mod tests { let (a, b) = collect_columns(planned.morsels).await; assert_eq!(a, vec![0, 1, 2, 3, 4, 5, 6, 7]); assert_eq!(b, vec![10, 11, 12, 13, 14, 15, 16, 17]); - assert!(contains(&row_group0, 0, 0).await); - assert!(contains(&row_group0, 1, 0).await); - assert!(contains(&row_group1, 0, 0).await); - assert!(!contains(&row_group1, 1, 0).await); + assert_full_cache_made_room(&row_group0, &row_group1).await; } #[tokio::test] @@ -1630,10 +1637,7 @@ mod tests { let (a, b) = collect_columns(planned.morsels).await; assert_eq!(a, vec![3, 4, 5, 6, 7]); assert_eq!(b, vec![13, 14, 15, 16, 17]); - assert!(contains(&row_group0, 0, 0).await); - assert!(contains(&row_group0, 1, 0).await); - assert!(contains(&row_group1, 0, 0).await); - assert!(!contains(&row_group1, 1, 0).await); + assert_full_cache_made_room(&row_group0, &row_group1).await; } #[tokio::test] @@ -1672,10 +1676,7 @@ mod tests { collect_columns(planned.morsels).await.0, vec![3, 4, 5, 6, 7] ); - assert!(contains(&row_group0, 0, 0).await); - assert!(contains(&row_group0, 1, 0).await); - assert!(contains(&row_group1, 0, 0).await); - assert!(!contains(&row_group1, 1, 0).await); + assert_full_cache_made_room(&row_group0, &row_group1).await; } #[tokio::test] diff --git a/src/datafusion/src/reader/runtime/liquid_cache_reader.rs b/src/datafusion/src/reader/runtime/liquid_cache_reader.rs index 2b6f55f1..db60a86e 100644 --- a/src/datafusion/src/reader/runtime/liquid_cache_reader.rs +++ b/src/datafusion/src/reader/runtime/liquid_cache_reader.rs @@ -7,10 +7,10 @@ use arrow::array::{Array, ArrayRef, BooleanArray, RecordBatch}; use arrow::buffer::BooleanBuffer; use arrow::compute::prep_null_mask_filter; use arrow::record_batch::RecordBatchOptions; -use arrow_schema::{ArrowError, SchemaRef}; +use arrow_schema::{ArrowError, Schema, SchemaRef}; use futures::{Stream, StreamExt, future::BoxFuture, stream::BoxStream}; use parquet::arrow::arrow_reader::{ - ArrowReaderMetadata, ArrowReaderOptions, RowSelection, RowSelector, + ArrowPredicate, ArrowReaderMetadata, ArrowReaderOptions, RowSelection, RowSelector, }; use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask}; use parquet::errors::ParquetError; @@ -331,18 +331,66 @@ impl LiquidCacheReaderInner { return Ok(selection); } - self.read_parquet_batch_and_fill_cache(self.current_batch_id) + let record_batch = self + .read_parquet_batch_and_fill_cache(self.current_batch_id) .await?; - apply_predicates( + if let Some(selection) = apply_predicates( &self.cached_row_group, self.current_batch_id, - input_selection, + input_selection.clone(), filter, ) .await? - .ok_or_else(|| { - ArrowError::ComputeError("predicate unavailable after materialization".to_string()) - }) + { + return Ok(selection); + } + // A full cache can evict this batch's own columns while it caches the + // rest of them, so the predicates fall back to the batch just read. + self.apply_predicates_to_batch(&record_batch, input_selection, filter) + } + + /// Evaluate `filter` over the rows of `record_batch` that `selection` + /// keeps, as [`apply_predicates`] does over the cache. + fn apply_predicates_to_batch( + &self, + record_batch: &RecordBatch, + mut selection: BooleanBuffer, + filter: &mut LiquidRowFilter, + ) -> Result { + for predicate in filter.predicates_mut() { + let selected_rows = selection.count_set_bits(); + if selected_rows == 0 { + break; + } + let mut fields = Vec::new(); + let mut arrays = Vec::new(); + for column_id in predicate.predicate_column_ids() { + let column = self + .cached_row_group + .get_column(column_id as u64) + .ok_or_else(|| { + ArrowError::ComputeError(format!( + "column {column_id} not present in liquid cache" + )) + })?; + fields.push(column.field()); + arrays.push(filter_array( + self.parquet_array(record_batch, column_id)?, + &selection, + )?); + } + let options = RecordBatchOptions::new().with_row_count(Some(selected_rows)); + let batch = + RecordBatch::try_new_with_options(Arc::new(Schema::new(fields)), arrays, &options)?; + let boolean_array = predicate.evaluate(batch)?; + let boolean_mask = if boolean_array.null_count() == 0 { + boolean_array.into_parts().0 + } else { + prep_null_mask_filter(&boolean_array).into_parts().0 + }; + selection = boolean_buffer_and_then(&selection, &boolean_mask); + } + Ok(selection) } #[fastrace::trace] From 856b2068b71626c97d73008ac34a48b0193fdcd5 Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Wed, 18 Nov 2026 01:47:57 +0530 Subject: [PATCH 2/6] fix(cache): settle a replacement's charge against what it displaced try_insert charged the memory budget against the slot it read, then installed over whatever slot was there by then. A concurrent drop or replacement of the key in between left the budget off by that slot's size, and enough of that can wrap the counter and refuse every insert. The charge is now settled against the slot actually displaced. A drop that removed a key and then told the policy could also remove a newer entry of the key from the policy, leaving it unevictable. The drop now files the key again if the index holds it. --- src/core/src/cache/budget.rs | 6 ++ src/core/src/cache/core.rs | 90 +++++++++++++++++-- .../tests/memory_only_workloads.rs | 10 +++ 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/src/core/src/cache/budget.rs b/src/core/src/cache/budget.rs index 2b7bc5d0..95d7a6a4 100644 --- a/src/core/src/cache/budget.rs +++ b/src/core/src/cache/budget.rs @@ -52,6 +52,12 @@ impl BudgetAccounting { } } + /// Charge `bytes` of memory whatever the budget reads: for memory an + /// entry already holds, which the budget missed. + pub(super) fn charge_memory(&self, bytes: usize) { + self.used_memory_bytes.fetch_add(bytes, Ordering::Relaxed); + } + /// Adjust memory usage after transcoding. /// Returns ok if the usage was adjusted, err if the memory budget is full when new_size is larger than old_size. pub(super) fn try_update_memory_usage( diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 41318719..90a59305 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -648,10 +648,14 @@ impl LiquidCache { "a disk record is installed only by the write that charged it" ); let new_memory_size = to_insert.memory_usage_bytes(); - let reserved = match self.index.get(&entry_id) { - Some(slot) => self + let assumed_old_size = self + .index + .get(&entry_id) + .map(|slot| slot.entry.memory_usage_bytes()); + let reserved = match assumed_old_size { + Some(old_size) => self .budget - .try_update_memory_usage(slot.entry.memory_usage_bytes(), new_memory_size), + .try_update_memory_usage(old_size, new_memory_size), // A disk record holds no memory, so it installs however full the // memory budget reads. Its write cannot then be dropped, taking // its charge, while the caller retries the record. @@ -662,10 +666,24 @@ impl LiquidCache { return Err(to_insert); } let cached_batch_type = CachedBatchType::from(&to_insert); - let displaced = self - .index - .insert(&entry_id, identity, to_insert) - .and_then(|displaced| self.charge_of(&displaced.entry)); + let displaced = self.index.insert(&entry_id, identity, to_insert); + // The charge above assumed the slot read before it. Another write can + // have replaced or removed that slot since, so the charge is settled + // against the slot this insert actually displaced: the budget then + // counts exactly the entries the index holds. + let actual_old_size = displaced + .as_ref() + .map_or(0, |slot| slot.entry.memory_usage_bytes()); + let assumed_old_size = assumed_old_size.unwrap_or(0); + if assumed_old_size > actual_old_size { + self.budget + .charge_memory(assumed_old_size - actual_old_size); + } else if actual_old_size > assumed_old_size { + self.budget + .try_update_memory_usage(actual_old_size - assumed_old_size, 0) + .expect("memory release cannot fail"); + } + let displaced = displaced.and_then(|displaced| self.charge_of(&displaced.entry)); let orphan = match write { Some(DiskWrite { key_lock, @@ -777,6 +795,14 @@ impl LiquidCache { .try_update_memory_usage(removed.entry.memory_usage_bytes(), 0) .expect("memory release cannot fail"); self.cache_policy.notify_remove(&entry_id); + // A write can have installed the key again, and told the policy, + // between the removal and the notice above, which then took the new + // entry out of the policy too. An entry the policy does not know is + // never evicted, so it is told again; a policy files a key once. + if let Some(slot) = self.index.get(&entry_id) { + self.cache_policy + .notify_insert(&entry_id, CachedBatchType::from(&slot.entry)); + } } /// Remove `entry_id` if the index still holds it in memory. @@ -1763,6 +1789,56 @@ mod tests { assert_eq!(cache.budget.memory_usage_bytes(), 0); } + /// Many writers on a few keys of a full memory-only cache, so inserts, + /// replacements and evictions of one key overlap. Afterwards the budget + /// charges exactly what the index holds, and the cache still admits. + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn overlapping_writes_to_a_full_memory_only_cache_keep_its_accounting() { + let budget = 256 << 10; + let cache = Arc::new( + LiquidCacheBuilder::new() + .with_max_memory_bytes(budget) + .with_max_disk_bytes(0) + .with_eviction_policy(Box::new(TranscodeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build() + .await, + ); + let mut tasks = Vec::new(); + for writer in 0..16i64 { + let cache = Arc::clone(&cache); + tasks.push(tokio::spawn(async move { + for round in 0..400i64 { + let key = EntryID::from(((writer * 7 + round) % 24) as usize); + let len = 512 + ((writer + round) % 4) as usize * 512; + let _ = cache + .insert( + key, + (round % 2) as u64, + int64_array(len, writer * 1000 + round), + ) + .await; + tokio::task::yield_now().await; + } + })); + } + for task in tasks { + task.await.unwrap(); + } + + let mut held = 0; + cache.for_each_entry(|_, entry| held += entry.memory_usage_bytes()); + assert_eq!(cache.budget.memory_usage_bytes(), held); + assert!(held <= budget); + + let fresh = EntryID::from(10_000usize); + cache + .insert(fresh, 0, int64_array(1024, 77)) + .await + .expect("a fresh entry was refused"); + assert!(cache.contains(&fresh, 0)); + } + #[tokio::test] async fn insert_until_disk_full_then_evicts_oldest_disk_entry() { let first_array: ArrayRef = Arc::new(Int32Array::from_iter_values(0..16)); diff --git a/src/datafusion-local/tests/memory_only_workloads.rs b/src/datafusion-local/tests/memory_only_workloads.rs index df84df35..b06e2632 100644 --- a/src/datafusion-local/tests/memory_only_workloads.rs +++ b/src/datafusion-local/tests/memory_only_workloads.rs @@ -202,8 +202,18 @@ fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { ids } +/// The cache charges exactly the memory its entries hold, and no more than +/// its budget. Called once no query is running. fn assert_within_budget(cache: &LiquidCacheParquetRef, after: &str) { let used = cache.storage().stats().memory_usage_bytes; + let mut held = 0; + cache + .storage() + .for_each_entry(|_, entry| held += entry.memory_usage_bytes()); + assert_eq!( + used, held, + "after {after} the cache charges {used} bytes for entries holding {held}" + ); assert!( used <= MEMORY_BYTES, "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES}" From 3b61ae5f2a3ec147df58345268b20f4ec2f85256 Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Wed, 18 Nov 2026 01:50:59 +0530 Subject: [PATCH 3/6] perf(cache): skip serialising a refused entry with no disk tier --- src/core/src/cache/core.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 90a59305..52d1343b 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -525,6 +525,11 @@ impl LiquidCache { Ok((new_batch, write)) } CacheEntry::MemoryLiquid(liquid_array) => { + // With no disk tier the write below can only fail; it is + // refused before the array is serialised for it. + if self.config.max_disk_bytes() == 0 { + return Err(CacheFull); + } let liquid_bytes = Bytes::from(liquid_array.to_bytes()); let disk_bytes = liquid_bytes.len(); let write = self From 01f1812c088a0c37843285b3a1eba3adc5cd408c Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Wed, 18 Nov 2026 02:01:25 +0530 Subject: [PATCH 4/6] fix(cache): reserve only growth before an insert settles its charge Releasing a shrink before the insert could under-read the budget while another write of the key raced it. Only growth is now reserved up front, and the rest is settled once the insert knows what it displaced, so the budget can over-read for a moment but never under-read. Also test the re-file of a key written again while its victim drops, and note that a policy's notify_insert must be an upsert. --- src/core/src/cache/budget.rs | 2 +- src/core/src/cache/core.rs | 134 +++++++++++++++++++---- src/core/src/cache/policies/cache/mod.rs | 3 + 3 files changed, 115 insertions(+), 24 deletions(-) diff --git a/src/core/src/cache/budget.rs b/src/core/src/cache/budget.rs index 95d7a6a4..8e15dfdd 100644 --- a/src/core/src/cache/budget.rs +++ b/src/core/src/cache/budget.rs @@ -37,7 +37,7 @@ impl BudgetAccounting { /// Returns ok if the memory was reserved, err if the memory budget is full. pub(super) fn try_reserve_memory(&self, request_bytes: usize) -> Result<(), ()> { let used = self.used_memory_bytes.load(Ordering::Relaxed); - if used + request_bytes > self.max_memory_bytes { + if used.saturating_add(request_bytes) > self.max_memory_bytes { return Err(()); } diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 52d1343b..165c3e36 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -653,39 +653,37 @@ impl LiquidCache { "a disk record is installed only by the write that charged it" ); let new_memory_size = to_insert.memory_usage_bytes(); + // Only growth over the slot read here is reserved up front, against + // the budget. The rest is settled once the insert has displaced + // whatever slot is there by then, which another write can have + // replaced or removed since: the budget then counts exactly the + // entries the index holds. Settling after the insert can only + // over-read the budget for a moment, never under-read it. + // + // A disk record holds no memory, so it installs however full the + // memory budget reads. Its write cannot then be dropped, taking its + // charge, while the caller retries the record. let assumed_old_size = self .index .get(&entry_id) - .map(|slot| slot.entry.memory_usage_bytes()); - let reserved = match assumed_old_size { - Some(old_size) => self - .budget - .try_update_memory_usage(old_size, new_memory_size), - // A disk record holds no memory, so it installs however full the - // memory budget reads. Its write cannot then be dropped, taking - // its charge, while the caller retries the record. - None if new_memory_size == 0 => Ok(()), - None => self.budget.try_reserve_memory(new_memory_size), - }; - if reserved.is_err() { + .map_or(0, |slot| slot.entry.memory_usage_bytes()); + let reserved = new_memory_size.saturating_sub(assumed_old_size); + if reserved > 0 && self.budget.try_reserve_memory(reserved).is_err() { return Err(to_insert); } let cached_batch_type = CachedBatchType::from(&to_insert); let displaced = self.index.insert(&entry_id, identity, to_insert); - // The charge above assumed the slot read before it. Another write can - // have replaced or removed that slot since, so the charge is settled - // against the slot this insert actually displaced: the budget then - // counts exactly the entries the index holds. let actual_old_size = displaced .as_ref() .map_or(0, |slot| slot.entry.memory_usage_bytes()); - let assumed_old_size = assumed_old_size.unwrap_or(0); - if assumed_old_size > actual_old_size { - self.budget - .charge_memory(assumed_old_size - actual_old_size); - } else if actual_old_size > assumed_old_size { + // The insert's net change is `new - actual_old`, of which `reserved` + // is charged already. + let remaining = new_memory_size as i128 - actual_old_size as i128 - reserved as i128; + if remaining > 0 { + self.budget.charge_memory(remaining as usize); + } else if remaining < 0 { self.budget - .try_update_memory_usage(actual_old_size - assumed_old_size, 0) + .try_update_memory_usage((-remaining) as usize, 0) .expect("memory release cannot fail"); } let displaced = displaced.and_then(|displaced| self.charge_of(&displaced.entry)); @@ -1814,7 +1812,7 @@ mod tests { let cache = Arc::clone(&cache); tasks.push(tokio::spawn(async move { for round in 0..400i64 { - let key = EntryID::from(((writer * 7 + round) % 24) as usize); + let key = EntryID::from(((writer * 3 + round) % 8) as usize); let len = 512 + ((writer + round) % 4) as usize * 512; let _ = cache .insert( @@ -1842,6 +1840,21 @@ mod tests { .await .expect("a fresh entry was refused"); assert!(cache.contains(&fresh, 0)); + + // Every entry the index holds is one the policy can still pick: an + // entry it lost is never evicted. + let known: std::collections::HashSet = cache + .cache_policy + .find_memory_victim(100_000) + .into_iter() + .map(usize::from) + .collect(); + cache.for_each_entry(|id, _| { + assert!( + known.contains(&usize::from(*id)), + "entry {id:?} is cached but unknown to the policy" + ); + }); } #[tokio::test] @@ -1979,6 +1992,81 @@ mod tests { } } + type BeforeRemoveHook = Box; + + /// A [`LiquidPolicy`] that runs a hook once, when told of a removal and + /// before it acts on the notice. + struct HookBeforeRemove { + inner: LiquidPolicy, + hook: Arc>>, + } + + impl std::fmt::Debug for HookBeforeRemove { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HookBeforeRemove").finish() + } + } + + impl CachePolicy for HookBeforeRemove { + fn find_memory_victim(&self, cnt: usize) -> Vec { + self.inner.find_memory_victim(cnt) + } + + fn notify_insert(&self, entry_id: &EntryID, batch_type: CachedBatchType) { + self.inner.notify_insert(entry_id, batch_type); + } + + fn notify_remove(&self, entry_id: &EntryID) { + let hook = self.hook.lock().unwrap().take(); + if let Some(hook) = hook { + hook(); + } + self.inner.notify_remove(entry_id); + } + } + + /// A write installs the key again, and tells the policy, between a + /// victim's removal and the removal's notice to the policy. The notice + /// then takes the new entry out of the policy too; the drop files it + /// again, so it stays evictable. + #[tokio::test] + async fn a_key_written_again_while_its_victim_is_dropped_stays_evictable() { + let hook = Arc::new(std::sync::Mutex::new(None::)); + let cache = Arc::new( + LiquidCacheBuilder::new() + .with_max_memory_bytes(1 << 20) + .with_max_disk_bytes(0) + .with_cache_policy(Box::new(HookBeforeRemove { + inner: LiquidPolicy::new(), + hook: hook.clone(), + })) + .build() + .await, + ); + let key = EntryID::from(5usize); + cache.insert(key, 0, int64_array(64, 1)).await.unwrap(); + let observed = cache.index().get(&key).unwrap().entry.clone(); + + let writer = Arc::clone(&cache); + let newer = int64_array(64, 2); + let installed = newer.clone(); + *hook.lock().unwrap() = Some(Box::new(move || { + let Ok(Orphan(None)) = + writer.try_insert(key, 0, CacheEntry::memory_arrow(installed), None) + else { + panic!("the write was refused or displaced a disk record"); + }; + })); + + cache.drop_memory_victim(key, &observed); + + assert!(hook.lock().unwrap().is_none(), "the hook never ran"); + let slot = cache.index().get(&key).expect("the newer entry is cached"); + assert!(matches!(&slot.entry, CacheEntry::MemoryArrow(a) if Arc::ptr_eq(a, &newer))); + assert_eq!(cache.cache_policy.find_memory_victim(8), vec![key]); + assert_memory_budget_matches_index(&cache); + } + /// The memory budget equals what the index's memory entries hold. fn assert_memory_budget_matches_index(cache: &LiquidCache) { let mut held = 0; diff --git a/src/core/src/cache/policies/cache/mod.rs b/src/core/src/cache/policies/cache/mod.rs index f2f2ef6e..3464a313 100644 --- a/src/core/src/cache/policies/cache/mod.rs +++ b/src/core/src/cache/policies/cache/mod.rs @@ -19,6 +19,9 @@ pub trait CachePolicy: std::fmt::Debug + Send + Sync { } /// Notify the cache policy that an entry was inserted. + /// + /// The cache can tell the policy of the same key again, in any order + /// with other notices of it, so this must file the key once (an upsert). fn notify_insert(&self, _entry_id: &EntryID, _batch_type: CachedBatchType) {} /// Notify the cache policy that an entry was accessed. From 54f4d22b96a6d77bcee3fe6023bb399a0bf7bffb Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Wed, 18 Nov 2026 02:43:13 +0530 Subject: [PATCH 5/6] test(cache): allow one entry over budget after a settle race --- src/core/src/cache/core.rs | 16 +++++++++++----- .../tests/memory_only_workloads.rs | 17 ++++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 165c3e36..8d8c1e2d 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -657,8 +657,10 @@ impl LiquidCache { // the budget. The rest is settled once the insert has displaced // whatever slot is there by then, which another write can have // replaced or removed since: the budget then counts exactly the - // entries the index holds. Settling after the insert can only - // over-read the budget for a moment, never under-read it. + // entries the index holds, never fewer. When the slot read here was + // removed meanwhile, the settle charges this entry in full, however + // full the budget is: usage can then exceed the maximum by up to one + // entry until a later insert evicts. // // A disk record holds no memory, so it installs however full the // memory budget reads. Its write cannot then be dropped, taking its @@ -1829,10 +1831,14 @@ mod tests { task.await.unwrap(); } - let mut held = 0; - cache.for_each_entry(|_, entry| held += entry.memory_usage_bytes()); + let (mut held, mut largest) = (0, 0); + cache.for_each_entry(|_, entry| { + held += entry.memory_usage_bytes(); + largest = largest.max(entry.memory_usage_bytes()); + }); assert_eq!(cache.budget.memory_usage_bytes(), held); - assert!(held <= budget); + // A settle can charge an entry over a full budget; one entry at most. + assert!(held <= budget + largest); let fresh = EntryID::from(10_000usize); cache diff --git a/src/datafusion-local/tests/memory_only_workloads.rs b/src/datafusion-local/tests/memory_only_workloads.rs index b06e2632..1fc6818e 100644 --- a/src/datafusion-local/tests/memory_only_workloads.rs +++ b/src/datafusion-local/tests/memory_only_workloads.rs @@ -203,20 +203,23 @@ fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { } /// The cache charges exactly the memory its entries hold, and no more than -/// its budget. Called once no query is running. +/// its budget plus one entry: an insert racing the drop of its key can charge +/// one entry over a full budget, until a later insert evicts. Called once no +/// query is running. fn assert_within_budget(cache: &LiquidCacheParquetRef, after: &str) { let used = cache.storage().stats().memory_usage_bytes; - let mut held = 0; - cache - .storage() - .for_each_entry(|_, entry| held += entry.memory_usage_bytes()); + let (mut held, mut largest) = (0, 0); + cache.storage().for_each_entry(|_, entry| { + held += entry.memory_usage_bytes(); + largest = largest.max(entry.memory_usage_bytes()); + }); assert_eq!( used, held, "after {after} the cache charges {used} bytes for entries holding {held}" ); assert!( - used <= MEMORY_BYTES, - "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES}" + used <= MEMORY_BYTES + largest, + "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES} by more than one entry" ); } From f9060667eab5f35466bca599b66ec29ec8b8481f Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Wed, 18 Nov 2026 03:06:55 +0530 Subject: [PATCH 6/6] test(cache): bound the overshoot per racing insert --- src/core/src/cache/core.rs | 10 +++--- .../tests/memory_only_workloads.rs | 35 +++++++++++-------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/core/src/cache/core.rs b/src/core/src/cache/core.rs index 8d8c1e2d..16739ae8 100644 --- a/src/core/src/cache/core.rs +++ b/src/core/src/cache/core.rs @@ -659,8 +659,9 @@ impl LiquidCache { // replaced or removed since: the budget then counts exactly the // entries the index holds, never fewer. When the slot read here was // removed meanwhile, the settle charges this entry in full, however - // full the budget is: usage can then exceed the maximum by up to one - // entry until a later insert evicts. + // full the budget is: usage can then exceed the maximum, by up to one + // entry for each insert that raced the removal of its key, until a + // later insert that grows the budget evicts. // // A disk record holds no memory, so it installs however full the // memory budget reads. Its write cannot then be dropped, taking its @@ -1837,8 +1838,9 @@ mod tests { largest = largest.max(entry.memory_usage_bytes()); }); assert_eq!(cache.budget.memory_usage_bytes(), held); - // A settle can charge an entry over a full budget; one entry at most. - assert!(held <= budget + largest); + // A settle can charge an entry over a full budget: at most one entry + // for each writer whose insert raced the removal of its key. + assert!(held <= budget + 16 * largest); let fresh = EntryID::from(10_000usize); cache diff --git a/src/datafusion-local/tests/memory_only_workloads.rs b/src/datafusion-local/tests/memory_only_workloads.rs index 1fc6818e..026b2d82 100644 --- a/src/datafusion-local/tests/memory_only_workloads.rs +++ b/src/datafusion-local/tests/memory_only_workloads.rs @@ -202,24 +202,30 @@ fn entry_ids(cache: &LiquidCacheParquetRef) -> HashSet { ids } -/// The cache charges exactly the memory its entries hold, and no more than -/// its budget plus one entry: an insert racing the drop of its key can charge -/// one entry over a full budget, until a later insert evicts. Called once no -/// query is running. -fn assert_within_budget(cache: &LiquidCacheParquetRef, after: &str) { +/// The cache charges exactly the memory its entries hold. Called once no query +/// is running. +fn assert_accounting_exact(cache: &LiquidCacheParquetRef, after: &str) -> usize { let used = cache.storage().stats().memory_usage_bytes; - let (mut held, mut largest) = (0, 0); - cache.storage().for_each_entry(|_, entry| { - held += entry.memory_usage_bytes(); - largest = largest.max(entry.memory_usage_bytes()); - }); + let mut held = 0; + cache + .storage() + .for_each_entry(|_, entry| held += entry.memory_usage_bytes()); assert_eq!( used, held, "after {after} the cache charges {used} bytes for entries holding {held}" ); + used +} + +/// The accounting is exact and within the budget. An insert that raced the +/// removal of its key can charge one entry over a full budget, until a later +/// insert that grows evicts; so after concurrent queries, call this only +/// after a sequential query has run. +fn assert_within_budget(cache: &LiquidCacheParquetRef, after: &str) { + let used = assert_accounting_exact(cache, after); assert!( - used <= MEMORY_BYTES + largest, - "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES} by more than one entry" + used <= MEMORY_BYTES, + "after {after} the cache charges {used} bytes, over its {MEMORY_BYTES}" ); } @@ -289,7 +295,7 @@ async fn concurrent_queries_keep_the_cache_admitting() { for task in tasks { task.await.unwrap(); } - assert_within_budget(&f.cache, "the concurrent queries"); + assert_accounting_exact(&f.cache, "the concurrent queries"); let before = entry_ids(&f.cache); query_and_check(&f, &last, last_kind).await; @@ -298,6 +304,7 @@ async fn concurrent_queries_keep_the_cache_admitting() { new_entries > 0, "after the concurrent queries, {last} cached nothing" ); + assert_within_budget(&f.cache, &last); } /// Queries cancelled part way, many times over, must not leave the cache @@ -317,7 +324,7 @@ async fn cancelled_queries_do_not_freeze_the_cache() { task.abort(); let _ = task.await; } - assert_within_budget(&f.cache, "the cancelled queries"); + assert_accounting_exact(&f.cache, "the cancelled queries"); for (table, kind) in f.tables[..12].iter().cloned() { query_and_check(&f, &table, kind).await;