From 7c26a2d13a532e4f5955cb2aa676b3e415838a0a Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Mon, 2 Nov 2026 15:28:58 +0000 Subject: [PATCH 1/2] perf(scan): merge narrow splits at boundaries only some fields of a struct have A scan splits wherever any column's chunk ends, and a split decodes the whole chunk of every column it overlaps. Where a struct's fields chunk at unrelated rows, as list elements and dictionary-encoded columns of a wide nested file do, the union holds spans of a few dozen rows, and successive splits each decode the same chunks of the other fields. The struct reader now marks the boundaries only some of its fields have as soft. SplitBy::LayoutSubSplitting merges spans ending at soft boundaries up to DEFAULT_MIN_SPLIT_ROWS (8,192, or max_rows if smaller). Boundaries every field has, and those layouts above the struct register, such as top-level chunk ends, are never merged, so files whose columns chunk alike split as before. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christopher Wirt --- vortex-layout/src/layouts/struct_/reader.rs | 147 +++++++++++++++++++- vortex-layout/src/reader.rs | 28 ++++ vortex-layout/src/scan/split_by.rs | 79 ++++++++++- 3 files changed, 247 insertions(+), 7 deletions(-) diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index c29314ed969..f46418d70b6 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -528,10 +528,33 @@ impl LayoutReader for StructReader { validity_ref.register_splits(field_mask, split_range, splits)?; } + // Each field's boundaries, kept apart so the ones only some fields have can be marked soft. + let mut fields: Vec = Vec::new(); self.layout.matching_fields(field_mask, |mask, idx| { + let mut field = RowSplits::new_capacity(16); self.field_reader_by_index(idx)? - .register_splits(&[mask], split_range, splits) + .register_splits(&[mask], split_range, &mut field)?; + fields.push(field); + Ok(()) })?; + if let [field] = &mut fields[..] { + splits.append(std::mem::replace(field, RowSplits::new_capacity(0))); + } else if !fields.is_empty() { + let fields: Vec<(Vec, Vec)> = fields + .into_iter() + .map(RowSplits::into_sorted_deduped_with_soft) + .collect(); + let shared = |row: &u64| { + fields + .iter() + .all(|(boundaries, _)| boundaries.binary_search(row).is_ok()) + }; + for (boundaries, soft) in fields.iter() { + splits.extend_ascending(boundaries.iter().copied()); + splits.mark_soft(soft.iter().copied()); + splits.mark_soft(boundaries.iter().copied().filter(|row| !shared(row))); + } + } // In the case of an empty struct, we need to register the end split. Pushed last so it // extends the final field's ascending run rather than starting a run of its own. @@ -679,7 +702,9 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; + use vortex_array::dtype::FieldMask; use vortex_array::dtype::FieldName; + use vortex_array::dtype::FieldPath; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::RESERVED_VALIDITY_FIELD_NAME; @@ -712,8 +737,12 @@ mod tests { use crate::LayoutReaderRef; use crate::LayoutRef; use crate::LayoutStrategy; + use crate::RowSplits; + use crate::SplitRange; use crate::layouts::chunked::writer::ChunkedLayoutStrategy; use crate::layouts::flat::writer::FlatLayoutStrategy; + use crate::layouts::repartition::RepartitionStrategy; + use crate::layouts::repartition::RepartitionWriterOptions; use crate::layouts::table::TableStrategy; use crate::layouts::zoned::writer::ZonedLayoutOptions; use crate::layouts::zoned::writer::ZonedStrategy; @@ -803,6 +832,122 @@ mod tests { (segments, layout) } + /// A leaf writer that chunks every `rows` rows. + fn chunked_every(rows: usize) -> Arc { + Arc::new(RepartitionStrategy::new( + ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), + RepartitionWriterOptions { + block_size_minimum: 0, + block_len_multiple: rows, + block_size_target: None, + canonicalize: false, + }, + )) + } + + /// A struct of 12,000 rows whose field `a` chunks every 1,000 rows and `b` every 3,000, + /// written in `pieces` pushes. With `chunked`, the struct is itself a chunk of a chunked layout + /// per push. + fn misaligned_struct_layout( + pieces: usize, + chunked: bool, + ) -> (Arc, LayoutRef) { + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + let table = + TableStrategy::new(Arc::new(FlatLayoutStrategy::default()), chunked_every(1000)) + .with_field_writer(FieldPath::from_name("b"), chunked_every(3000)); + let strategy: Arc = if chunked { + Arc::new(ChunkedLayoutStrategy::new(table)) + } else { + Arc::new(table) + }; + let rows = 12_000 / pieces; + let chunks: Vec = (0..pieces) + .map(|piece| { + let start = i32::try_from(piece * rows).unwrap(); + let end = start + i32::try_from(rows).unwrap(); + StructArray::from_fields( + [ + ("a", PrimitiveArray::from_iter(start..end).into_array()), + ("b", PrimitiveArray::from_iter(start..end).into_array()), + ] + .as_slice(), + ) + .unwrap() + .into_array() + }) + .collect(); + let dtype = chunks[0].dtype().clone(); + let segments2 = Arc::::clone(&segments); + let layout = block_on(|handle| async move { + let session = new_session().with_handle(handle); + strategy + .write_stream( + ctx.into(), + segments2, + ChunkedArray::try_new(chunks, dtype) + .unwrap() + .into_array() + .to_array_stream() + .sequenced(ptr), + eof, + &session, + ) + .await + }) + .unwrap(); + (segments, layout) + } + + /// Registers `layout`'s boundaries over its rows, as a scan does. + fn boundaries_and_soft( + segments: Arc, + layout: &LayoutRef, + ) -> (Vec, Vec) { + let reader = layout + .new_reader("".into(), segments, &SESSION, &Default::default()) + .unwrap(); + let mut splits = RowSplits::new_capacity(16); + splits.push(0); + reader + .register_splits( + &[FieldMask::All], + &SplitRange::root(0..reader.row_count()).unwrap(), + &mut splits, + ) + .unwrap(); + splits.into_sorted_deduped_with_soft() + } + + #[test] + fn boundaries_only_some_fields_have_are_soft() { + let (segments, layout) = misaligned_struct_layout(1, false); + let (boundaries, soft) = boundaries_and_soft(segments, &layout); + // `a` chunks every 1,000 rows and `b` every 3,000, so only `b`'s are shared. + assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::>()); + assert_eq!( + soft, + (1..12) + .filter(|k| k % 3 != 0) + .map(|k| k * 1000) + .collect::>() + ); + } + + #[test] + fn chunk_ends_above_the_struct_stay_hard() { + // Two top-level chunks of 6,000 rows, each a struct whose `a` chunks every 1,000 rows and + // `b` every 3,000: the top-level chunk end at 6,000 is every column's, so it stays hard. + let (segments, layout) = misaligned_struct_layout(2, true); + let (boundaries, soft) = boundaries_and_soft(segments, &layout); + assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::>()); + assert!(!soft.contains(&6000), "the chunk end is soft: {soft:?}"); + assert!(!soft.contains(&3000) && !soft.contains(&9000), "{soft:?}"); + assert!(soft.contains(&1000) && soft.contains(&7000), "{soft:?}"); + } + #[fixture] /// Create a chunked layout with three chunks of primitive arrays. fn null_struct_layout() -> (Arc, LayoutRef) { diff --git a/vortex-layout/src/reader.rs b/vortex-layout/src/reader.rs index 91dda0e23c5..80b35609f42 100644 --- a/vortex-layout/src/reader.rs +++ b/vortex-layout/src/reader.rs @@ -100,12 +100,18 @@ impl SplitRange { /// starts a new run. A run that exactly repeats the previous surviving run — common when sibling /// columns share chunk boundaries — is dropped as it completes, and the final sort is skipped /// when only a single run survives (the boundaries are then already ascending). +/// +/// A boundary can also be marked soft: one that only some of a struct's fields have. A split there +/// lets the fields that chunk across it decode the same chunk in two splits, so the scan may merge +/// it away (see [`crate::scan::split_by::SplitBy::LayoutSubSplitting`]). pub struct RowSplits { splits: Vec, /// Start index of the run currently being appended. run_start: usize, /// Start index of the surviving run immediately before `run_start`. prev_run_start: usize, + /// Boundaries marked soft, unsorted and possibly repeated. + soft: Vec, } impl RowSplits { @@ -162,15 +168,37 @@ impl RowSplits { self.splits.reserve(additional); } + /// Mark boundaries soft: ones that only some fields of a struct have. They must also be + /// pushed. + pub fn mark_soft(&mut self, rows: impl IntoIterator) { + self.soft.extend(rows); + } + + /// Push `other`'s boundaries into this set as a batch, keeping its soft marks. + pub(crate) fn append(&mut self, other: RowSplits) { + let soft = other.soft.clone(); + self.extend_ascending(other.into_sorted_deduped()); + self.soft.extend(soft); + } + /// Create a new RowSplits with preallocated "capacity" pub(crate) fn new_capacity(capacity: usize) -> Self { Self { splits: Vec::with_capacity(capacity), run_start: 0, prev_run_start: 0, + soft: Vec::new(), } } + /// The sorted, deduplicated boundaries, and the sorted soft ones among them. + pub(crate) fn into_sorted_deduped_with_soft(mut self) -> (Vec, Vec) { + let mut soft = std::mem::take(&mut self.soft); + soft.sort_unstable(); + soft.dedup(); + (self.into_sorted_deduped(), soft) + } + pub(crate) fn into_sorted_deduped(mut self) -> Vec { let final_run_dropped = self.drop_repeated_run(); // Surviving runs always have a descent between them, so the boundaries are ascending diff --git a/vortex-layout/src/scan/split_by.rs b/vortex-layout/src/scan/split_by.rs index b995e46222f..e69d81904d9 100644 --- a/vortex-layout/src/scan/split_by.rs +++ b/vortex-layout/src/scan/split_by.rs @@ -19,6 +19,14 @@ use crate::scan::IDEAL_SPLIT_SIZE; /// be decoded across multiple cores rather than one. pub const DEFAULT_MAX_SPLIT_ROWS: u64 = IDEAL_SPLIT_SIZE; +/// The narrowest span [`SplitBy::LayoutSubSplitting`] leaves ending at a soft boundary, one that +/// only some fields of a struct have; narrower spans merge into a neighbour. It is the writer's +/// default row block, and is lowered to `max_rows` when that is smaller. +/// +/// A split decodes the whole chunk of every column it overlaps, so splits much narrower than a +/// column's chunks decode those chunks once each. +pub const DEFAULT_MIN_SPLIT_ROWS: u64 = 8192; + /// Defines how the Vortex file is split into batches for reading. /// /// Note that each split must fit into the platform's maximum usize. @@ -28,10 +36,11 @@ pub enum SplitBy { /// splits that follow the file's chunking exactly, trading intra-file decode parallelism /// for fewer, larger batches. Layout, - /// Splits like [`SplitBy::Layout`], except that spans between adjacent chunk boundaries - /// wider than `max_rows` are further sub-divided into evenly sized splits of at most - /// `max_rows` rows, so that a file with few, large chunks can still be decoded across - /// multiple cores. `max_rows` must be non-zero. + /// Splits like [`SplitBy::Layout`], except that spans narrower than + /// [`DEFAULT_MIN_SPLIT_ROWS`] (or `max_rows`, if smaller) ending at a soft boundary are merged + /// into a neighbour, and spans wider than `max_rows` are further sub-divided into evenly sized + /// splits of at most `max_rows` rows, so that a file with few, large chunks can still be + /// decoded across multiple cores. `max_rows` must be non-zero. /// /// This is the default, with [`DEFAULT_MAX_SPLIT_ROWS`]. LayoutSubSplitting { max_rows: u64 }, @@ -64,8 +73,10 @@ impl SplitBy { max_rows > 0, "SplitBy::LayoutSubSplitting requires a non-zero max_rows" ); + let (boundaries, soft) = register_boundaries(layout_reader, row_range, field_mask)? + .into_sorted_deduped_with_soft(); subdivide_large_spans( - layout_boundaries(layout_reader, row_range, field_mask)?, + merge_small_spans(boundaries, &soft, DEFAULT_MIN_SPLIT_ROWS.min(max_rows)), max_rows, ) } @@ -85,6 +96,16 @@ fn layout_boundaries( row_range: &Range, field_mask: &[FieldMask], ) -> VortexResult> { + Ok(register_boundaries(layout_reader, row_range, field_mask)?.into_sorted_deduped()) +} + +/// The boundaries the layout registers within `row_range`, after the range's start (the layout +/// registers its end). +fn register_boundaries( + layout_reader: &dyn LayoutReader, + row_range: &Range, + field_mask: &[FieldMask], +) -> VortexResult { // We usually have under 100 splits so reserving upfront saves us some allocations let mut row_splits = RowSplits::new_capacity(128); row_splits.push(row_range.start); @@ -93,7 +114,53 @@ fn layout_boundaries( &SplitRange::root(row_range.clone())?, &mut row_splits, )?; - Ok(row_splits.into_sorted_deduped()) + Ok(row_splits) +} + +/// Remove interior split boundaries in the sorted `soft` until no span ending at one is narrower +/// than `min_span`. +/// +/// Like [`subdivide_large_spans`], this works on the sorted, deduplicated boundary list. It never +/// adds or moves a boundary, and always keeps the first, the last and every boundary not in `soft`, +/// so the ranges still cover the same rows contiguously, each exactly once. Boundaries are dropped +/// from the front: a soft boundary is kept once the span since the last kept one reaches +/// `min_span`. A short span left before the end, or before a hard boundary, keeps its end and drops +/// its start where that is soft. +fn merge_small_spans(boundaries: Vec, soft: &[u64], min_span: u64) -> Vec { + debug_assert!(boundaries.is_sorted(), "boundaries must be sorted"); + debug_assert!(soft.is_sorted(), "soft boundaries must be sorted"); + let is_hard = |b: u64| soft.binary_search(&b).is_err(); + + // Fast path: nothing to merge (also covers the empty / single-boundary cases). + if soft.is_empty() + || boundaries.len() < 3 + || boundaries.windows(2).all(|w| w[1] - w[0] >= min_span) + { + return boundaries; + } + + let last = boundaries[boundaries.len() - 1]; + let mut out = Vec::with_capacity(boundaries.len()); + out.push(boundaries[0]); + for &boundary in &boundaries[1..] { + let prev = out[out.len() - 1]; + if boundary == last || is_hard(boundary) { + // A short span before a kept boundary merges into the one before it, unless its + // start must be kept too. + if boundary - prev < min_span && out.len() > 1 && !is_hard(prev) { + out.pop(); + } + out.push(boundary); + } else if boundary - prev >= min_span { + out.push(boundary); + } + } + + debug_assert!( + out.windows(2).all(|w| w[0] < w[1]), + "merged boundaries must stay strictly increasing" + ); + out } /// Sub-divide any gap between adjacent split boundaries that is wider than `max_span` into evenly From aab5de1a01341357be4219630a4de050e19f0d67 Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Mon, 2 Nov 2026 21:08:33 +0000 Subject: [PATCH 2/2] fix(scan): address soft split boundary review Compute the shared boundary intersection once, preserve validity boundaries, and move inherited soft marks without cloning their buffer. Add merge invariants, real-layout regression coverage, and a wider split-collection benchmark case. Signed-off-by: Christopher Wirt --- vortex-file/benches/split_collection.rs | 2 +- vortex-layout/src/layouts/struct_/reader.rs | 85 ++++++++++++++++----- vortex-layout/src/reader.rs | 12 ++- vortex-layout/src/scan/split_by.rs | 65 ++++++++++++++++ 4 files changed, 140 insertions(+), 24 deletions(-) diff --git a/vortex-file/benches/split_collection.rs b/vortex-file/benches/split_collection.rs index 8985f26e833..6ddd0abe3d0 100644 --- a/vortex-file/benches/split_collection.rs +++ b/vortex-file/benches/split_collection.rs @@ -55,7 +55,7 @@ fn main() { const ROWS_PER_CHUNK: usize = 1024; /// (columns, chunks) configurations. -const CONFIGS: &[(usize, usize)] = &[(16, 64)]; +const CONFIGS: &[(usize, usize)] = &[(16, 64), (256, 64)]; static RUNTIME: LazyLock = LazyLock::new(|| { tokio::runtime::Builder::new_current_thread() diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index f46418d70b6..248a7b249df 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -523,10 +523,15 @@ impl LayoutReader for StructReader { split_range: &SplitRange, splits: &mut RowSplits, ) -> VortexResult<()> { - // Register splits for the validity child, if there is one - if let Some(validity_ref) = self.validity()? { - validity_ref.register_splits(field_mask, split_range, splits)?; - } + // Validity boundaries must stay hard even when they overlap a soft data-field boundary. + let validity_boundaries = if let Some(validity_ref) = self.validity()? { + let mut validity_splits = RowSplits::new_capacity(16); + validity_ref.register_splits(field_mask, split_range, &mut validity_splits)?; + validity_splits.into_sorted_deduped() + } else { + Vec::new() + }; + splits.extend_ascending(validity_boundaries.iter().copied()); // Each field's boundaries, kept apart so the ones only some fields have can be marked soft. let mut fields: Vec = Vec::new(); @@ -537,24 +542,35 @@ impl LayoutReader for StructReader { fields.push(field); Ok(()) })?; + let mut field_splits = RowSplits::new_capacity(0); if let [field] = &mut fields[..] { - splits.append(std::mem::replace(field, RowSplits::new_capacity(0))); + field_splits = std::mem::replace(field, RowSplits::new_capacity(0)); } else if !fields.is_empty() { let fields: Vec<(Vec, Vec)> = fields .into_iter() .map(RowSplits::into_sorted_deduped_with_soft) .collect(); - let shared = |row: &u64| { - fields - .iter() - .all(|(boundaries, _)| boundaries.binary_search(row).is_ok()) - }; + // Intersect once; checking every field for every boundary is quadratic in field count. + let mut shared = fields[0].0.clone(); + for (boundaries, _) in &fields[1..] { + shared.retain(|row| boundaries.binary_search(row).is_ok()); + if shared.is_empty() { + break; + } + } for (boundaries, soft) in fields.iter() { - splits.extend_ascending(boundaries.iter().copied()); - splits.mark_soft(soft.iter().copied()); - splits.mark_soft(boundaries.iter().copied().filter(|row| !shared(row))); + field_splits.extend_ascending(boundaries.iter().copied()); + field_splits.mark_soft(soft.iter().copied()); + field_splits.mark_soft( + boundaries + .iter() + .copied() + .filter(|row| shared.binary_search(row).is_err()), + ); } } + field_splits.remove_soft(&validity_boundaries); + splits.append(field_splits); // In the case of an empty struct, we need to register the end split. Pushed last so it // extends the final field's ascending run rather than starting a run of its own. @@ -746,6 +762,7 @@ mod tests { use crate::layouts::table::TableStrategy; use crate::layouts::zoned::writer::ZonedLayoutOptions; use crate::layouts::zoned::writer::ZonedStrategy; + use crate::scan::split_by::SplitBy; use crate::segments::CountingSegmentSource; use crate::segments::SegmentFuture; use crate::segments::SegmentId; @@ -851,13 +868,13 @@ mod tests { fn misaligned_struct_layout( pieces: usize, chunked: bool, + nullable: bool, ) -> (Arc, LayoutRef) { let ctx = ArrayContext::empty(); let segments = Arc::new(TestSegments::default()); let (ptr, eof) = SequenceId::root().split(); - let table = - TableStrategy::new(Arc::new(FlatLayoutStrategy::default()), chunked_every(1000)) - .with_field_writer(FieldPath::from_name("b"), chunked_every(3000)); + let table = TableStrategy::new(chunked_every(2000), chunked_every(1000)) + .with_field_writer(FieldPath::from_name("b"), chunked_every(3000)); let strategy: Arc = if chunked { Arc::new(ChunkedLayoutStrategy::new(table)) } else { @@ -868,12 +885,18 @@ mod tests { .map(|piece| { let start = i32::try_from(piece * rows).unwrap(); let end = start + i32::try_from(rows).unwrap(); - StructArray::from_fields( + StructArray::try_from_iter_with_validity( [ ("a", PrimitiveArray::from_iter(start..end).into_array()), ("b", PrimitiveArray::from_iter(start..end).into_array()), - ] - .as_slice(), + ], + if nullable { + Validity::Array( + BoolArray::from_iter((start..end).map(|row| row % 2 == 0)).into_array(), + ) + } else { + Validity::NonNullable + }, ) .unwrap() .into_array() @@ -923,7 +946,7 @@ mod tests { #[test] fn boundaries_only_some_fields_have_are_soft() { - let (segments, layout) = misaligned_struct_layout(1, false); + let (segments, layout) = misaligned_struct_layout(1, false, false); let (boundaries, soft) = boundaries_and_soft(segments, &layout); // `a` chunks every 1,000 rows and `b` every 3,000, so only `b`'s are shared. assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::>()); @@ -940,7 +963,7 @@ mod tests { fn chunk_ends_above_the_struct_stay_hard() { // Two top-level chunks of 6,000 rows, each a struct whose `a` chunks every 1,000 rows and // `b` every 3,000: the top-level chunk end at 6,000 is every column's, so it stays hard. - let (segments, layout) = misaligned_struct_layout(2, true); + let (segments, layout) = misaligned_struct_layout(2, true, false); let (boundaries, soft) = boundaries_and_soft(segments, &layout); assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::>()); assert!(!soft.contains(&6000), "the chunk end is soft: {soft:?}"); @@ -948,6 +971,26 @@ mod tests { assert!(soft.contains(&1000) && soft.contains(&7000), "{soft:?}"); } + #[rstest] + #[case::non_nullable(false, vec![0, 3000, 6000, 9000, 12000])] + #[case::nullable(true, vec![0, 2000, 3000, 4000, 6000, 8000, 9000, 10000, 12000])] + fn scan_merges_soft_boundaries_and_keeps_validity_boundaries( + #[case] nullable: bool, + #[case] expected: Vec, + ) -> VortexResult<()> { + let (segments, layout) = misaligned_struct_layout(1, false, nullable); + let reader = layout.new_reader("".into(), segments, &SESSION, &Default::default())?; + assert_eq!( + SplitBy::default().splits( + reader.as_ref(), + &(0..reader.row_count()), + &[FieldMask::All], + )?, + expected, + ); + Ok(()) + } + #[fixture] /// Create a chunked layout with three chunks of primitive arrays. fn null_struct_layout() -> (Arc, LayoutRef) { diff --git a/vortex-layout/src/reader.rs b/vortex-layout/src/reader.rs index 80b35609f42..850fda4ba04 100644 --- a/vortex-layout/src/reader.rs +++ b/vortex-layout/src/reader.rs @@ -175,12 +175,20 @@ impl RowSplits { } /// Push `other`'s boundaries into this set as a batch, keeping its soft marks. - pub(crate) fn append(&mut self, other: RowSplits) { - let soft = other.soft.clone(); + pub(crate) fn append(&mut self, mut other: RowSplits) { + let soft = std::mem::take(&mut other.soft); self.extend_ascending(other.into_sorted_deduped()); self.soft.extend(soft); } + /// Remove soft marks at the supplied sorted boundaries. + pub(crate) fn remove_soft(&mut self, rows: &[u64]) { + debug_assert!(rows.is_sorted()); + if !rows.is_empty() { + self.soft.retain(|row| rows.binary_search(row).is_err()); + } + } + /// Create a new RowSplits with preallocated "capacity" pub(crate) fn new_capacity(capacity: usize) -> Self { Self { diff --git a/vortex-layout/src/scan/split_by.rs b/vortex-layout/src/scan/split_by.rs index e69d81904d9..d7d96c1781a 100644 --- a/vortex-layout/src/scan/split_by.rs +++ b/vortex-layout/src/scan/split_by.rs @@ -223,6 +223,10 @@ mod test { use std::sync::Arc; use futures::future::BoxFuture; + use rand::RngExt; + use rand::SeedableRng; + use rand::prelude::StdRng; + use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -430,6 +434,67 @@ mod test { assert_eq!(subdivide_large_spans(vec![7], 100), vec![7]); } + #[rstest] + #[case::empty(vec![], vec![], 5, vec![])] + #[case::single(vec![7], vec![7], 5, vec![7])] + #[case::hard(vec![0, 1, 2, 10], vec![], 5, vec![0, 1, 2, 10])] + #[case::soft(vec![0, 1, 2, 10], vec![1, 2], 5, vec![0, 10])] + #[case::short_tail(vec![0, 5, 6], vec![5], 5, vec![0, 6])] + #[case::hard_collision(vec![0, 3, 5, 6], vec![3, 5], 5, vec![0, 6])] + #[case::hard_start(vec![0, 3, 5, 6], vec![5], 5, vec![0, 3, 6])] + #[case::threshold(vec![0, 5, 10], vec![5], 5, vec![0, 5, 10])] + #[case::nonzero_range(vec![100, 101, 102, 110], vec![101, 102], 5, vec![100, 110])] + fn merge_small_spans_cases( + #[case] boundaries: Vec, + #[case] soft: Vec, + #[case] minimum: u64, + #[case] expected: Vec, + ) { + assert_eq!(merge_small_spans(boundaries, &soft, minimum), expected); + } + + #[test] + fn merging_preserves_hard_boundaries_and_exact_coverage() { + let mut rng = StdRng::seed_from_u64(67); + for _ in 0..1000 { + let start = rng.random_range(0..1000u64); + let mut boundaries = vec![start]; + for _ in 0..rng.random_range(1..50) { + boundaries.push(boundaries.last().unwrap() + rng.random_range(1..1000)); + } + let soft = boundaries + .iter() + .copied() + .filter(|_| rng.random_bool(0.5)) + .collect::>(); + let max_rows = rng.random_range(1..3000); + let minimum = DEFAULT_MIN_SPLIT_ROWS.min(max_rows); + let merged = merge_small_spans(boundaries.clone(), &soft, minimum); + assert_eq!(merged.first(), boundaries.first()); + assert_eq!(merged.last(), boundaries.last()); + assert!(merged.windows(2).all(|w| w[0] < w[1])); + assert!( + merged + .iter() + .all(|row| boundaries.binary_search(row).is_ok()) + ); + assert!(boundaries.iter().all(|row| { + soft.binary_search(row).is_ok() || merged.binary_search(row).is_ok() + })); + assert!(merged.windows(2).all(|w| { + Some(&w[1]) == merged.last() + || soft.binary_search(&w[1]).is_err() + || w[1] - w[0] >= minimum + })); + let subdivided = subdivide_large_spans(merged, max_rows); + assert!(subdivided.windows(2).all(|w| w[1] - w[0] <= max_rows)); + assert_eq!( + subdivided.windows(2).map(|w| w[1] - w[0]).sum::(), + boundaries.last().unwrap() - start, + ); + } + } + #[test] fn subdivide_near_u64_max_does_not_overflow() { // The increment past the last interior point would overflow without saturating math.