diff --git a/vortex-file/benches/split_collection.rs b/vortex-file/benches/split_collection.rs index 8985f26e833..6ddd0abe3d0 100644 --- a/vortex-file/benches/split_collection.rs +++ b/vortex-file/benches/split_collection.rs @@ -55,7 +55,7 @@ fn main() { const ROWS_PER_CHUNK: usize = 1024; /// (columns, chunks) configurations. -const CONFIGS: &[(usize, usize)] = &[(16, 64)]; +const CONFIGS: &[(usize, usize)] = &[(16, 64), (256, 64)]; static RUNTIME: LazyLock = LazyLock::new(|| { tokio::runtime::Builder::new_current_thread() diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index c29314ed969..248a7b249df 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -523,15 +523,54 @@ impl LayoutReader for StructReader { split_range: &SplitRange, splits: &mut RowSplits, ) -> VortexResult<()> { - // Register splits for the validity child, if there is one - if let Some(validity_ref) = self.validity()? { - validity_ref.register_splits(field_mask, split_range, splits)?; - } + // Validity boundaries must stay hard even when they overlap a soft data-field boundary. + let validity_boundaries = if let Some(validity_ref) = self.validity()? { + let mut validity_splits = RowSplits::new_capacity(16); + validity_ref.register_splits(field_mask, split_range, &mut validity_splits)?; + validity_splits.into_sorted_deduped() + } else { + Vec::new() + }; + splits.extend_ascending(validity_boundaries.iter().copied()); + // Each field's boundaries, kept apart so the ones only some fields have can be marked soft. + let mut fields: Vec = Vec::new(); self.layout.matching_fields(field_mask, |mask, idx| { + let mut field = RowSplits::new_capacity(16); self.field_reader_by_index(idx)? - .register_splits(&[mask], split_range, splits) + .register_splits(&[mask], split_range, &mut field)?; + fields.push(field); + Ok(()) })?; + let mut field_splits = RowSplits::new_capacity(0); + if let [field] = &mut fields[..] { + field_splits = std::mem::replace(field, RowSplits::new_capacity(0)); + } else if !fields.is_empty() { + let fields: Vec<(Vec, Vec)> = fields + .into_iter() + .map(RowSplits::into_sorted_deduped_with_soft) + .collect(); + // Intersect once; checking every field for every boundary is quadratic in field count. + let mut shared = fields[0].0.clone(); + for (boundaries, _) in &fields[1..] { + shared.retain(|row| boundaries.binary_search(row).is_ok()); + if shared.is_empty() { + break; + } + } + for (boundaries, soft) in fields.iter() { + field_splits.extend_ascending(boundaries.iter().copied()); + field_splits.mark_soft(soft.iter().copied()); + field_splits.mark_soft( + boundaries + .iter() + .copied() + .filter(|row| shared.binary_search(row).is_err()), + ); + } + } + field_splits.remove_soft(&validity_boundaries); + splits.append(field_splits); // In the case of an empty struct, we need to register the end split. Pushed last so it // extends the final field's ascending run rather than starting a run of its own. @@ -679,7 +718,9 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; + use vortex_array::dtype::FieldMask; use vortex_array::dtype::FieldName; + use vortex_array::dtype::FieldPath; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::RESERVED_VALIDITY_FIELD_NAME; @@ -712,11 +753,16 @@ mod tests { use crate::LayoutReaderRef; use crate::LayoutRef; use crate::LayoutStrategy; + use crate::RowSplits; + use crate::SplitRange; use crate::layouts::chunked::writer::ChunkedLayoutStrategy; use crate::layouts::flat::writer::FlatLayoutStrategy; + use crate::layouts::repartition::RepartitionStrategy; + use crate::layouts::repartition::RepartitionWriterOptions; use crate::layouts::table::TableStrategy; use crate::layouts::zoned::writer::ZonedLayoutOptions; use crate::layouts::zoned::writer::ZonedStrategy; + use crate::scan::split_by::SplitBy; use crate::segments::CountingSegmentSource; use crate::segments::SegmentFuture; use crate::segments::SegmentId; @@ -803,6 +849,148 @@ mod tests { (segments, layout) } + /// A leaf writer that chunks every `rows` rows. + fn chunked_every(rows: usize) -> Arc { + Arc::new(RepartitionStrategy::new( + ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), + RepartitionWriterOptions { + block_size_minimum: 0, + block_len_multiple: rows, + block_size_target: None, + canonicalize: false, + }, + )) + } + + /// A struct of 12,000 rows whose field `a` chunks every 1,000 rows and `b` every 3,000, + /// written in `pieces` pushes. With `chunked`, the struct is itself a chunk of a chunked layout + /// per push. + fn misaligned_struct_layout( + pieces: usize, + chunked: bool, + nullable: bool, + ) -> (Arc, LayoutRef) { + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + let table = TableStrategy::new(chunked_every(2000), chunked_every(1000)) + .with_field_writer(FieldPath::from_name("b"), chunked_every(3000)); + let strategy: Arc = if chunked { + Arc::new(ChunkedLayoutStrategy::new(table)) + } else { + Arc::new(table) + }; + let rows = 12_000 / pieces; + let chunks: Vec = (0..pieces) + .map(|piece| { + let start = i32::try_from(piece * rows).unwrap(); + let end = start + i32::try_from(rows).unwrap(); + StructArray::try_from_iter_with_validity( + [ + ("a", PrimitiveArray::from_iter(start..end).into_array()), + ("b", PrimitiveArray::from_iter(start..end).into_array()), + ], + if nullable { + Validity::Array( + BoolArray::from_iter((start..end).map(|row| row % 2 == 0)).into_array(), + ) + } else { + Validity::NonNullable + }, + ) + .unwrap() + .into_array() + }) + .collect(); + let dtype = chunks[0].dtype().clone(); + let segments2 = Arc::::clone(&segments); + let layout = block_on(|handle| async move { + let session = new_session().with_handle(handle); + strategy + .write_stream( + ctx.into(), + segments2, + ChunkedArray::try_new(chunks, dtype) + .unwrap() + .into_array() + .to_array_stream() + .sequenced(ptr), + eof, + &session, + ) + .await + }) + .unwrap(); + (segments, layout) + } + + /// Registers `layout`'s boundaries over its rows, as a scan does. + fn boundaries_and_soft( + segments: Arc, + layout: &LayoutRef, + ) -> (Vec, Vec) { + let reader = layout + .new_reader("".into(), segments, &SESSION, &Default::default()) + .unwrap(); + let mut splits = RowSplits::new_capacity(16); + splits.push(0); + reader + .register_splits( + &[FieldMask::All], + &SplitRange::root(0..reader.row_count()).unwrap(), + &mut splits, + ) + .unwrap(); + splits.into_sorted_deduped_with_soft() + } + + #[test] + fn boundaries_only_some_fields_have_are_soft() { + let (segments, layout) = misaligned_struct_layout(1, false, false); + let (boundaries, soft) = boundaries_and_soft(segments, &layout); + // `a` chunks every 1,000 rows and `b` every 3,000, so only `b`'s are shared. + assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::>()); + assert_eq!( + soft, + (1..12) + .filter(|k| k % 3 != 0) + .map(|k| k * 1000) + .collect::>() + ); + } + + #[test] + fn chunk_ends_above_the_struct_stay_hard() { + // Two top-level chunks of 6,000 rows, each a struct whose `a` chunks every 1,000 rows and + // `b` every 3,000: the top-level chunk end at 6,000 is every column's, so it stays hard. + let (segments, layout) = misaligned_struct_layout(2, true, false); + let (boundaries, soft) = boundaries_and_soft(segments, &layout); + assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::>()); + assert!(!soft.contains(&6000), "the chunk end is soft: {soft:?}"); + assert!(!soft.contains(&3000) && !soft.contains(&9000), "{soft:?}"); + assert!(soft.contains(&1000) && soft.contains(&7000), "{soft:?}"); + } + + #[rstest] + #[case::non_nullable(false, vec![0, 3000, 6000, 9000, 12000])] + #[case::nullable(true, vec![0, 2000, 3000, 4000, 6000, 8000, 9000, 10000, 12000])] + fn scan_merges_soft_boundaries_and_keeps_validity_boundaries( + #[case] nullable: bool, + #[case] expected: Vec, + ) -> VortexResult<()> { + let (segments, layout) = misaligned_struct_layout(1, false, nullable); + let reader = layout.new_reader("".into(), segments, &SESSION, &Default::default())?; + assert_eq!( + SplitBy::default().splits( + reader.as_ref(), + &(0..reader.row_count()), + &[FieldMask::All], + )?, + expected, + ); + Ok(()) + } + #[fixture] /// Create a chunked layout with three chunks of primitive arrays. fn null_struct_layout() -> (Arc, LayoutRef) { diff --git a/vortex-layout/src/reader.rs b/vortex-layout/src/reader.rs index 91dda0e23c5..850fda4ba04 100644 --- a/vortex-layout/src/reader.rs +++ b/vortex-layout/src/reader.rs @@ -100,12 +100,18 @@ impl SplitRange { /// starts a new run. A run that exactly repeats the previous surviving run — common when sibling /// columns share chunk boundaries — is dropped as it completes, and the final sort is skipped /// when only a single run survives (the boundaries are then already ascending). +/// +/// A boundary can also be marked soft: one that only some of a struct's fields have. A split there +/// lets the fields that chunk across it decode the same chunk in two splits, so the scan may merge +/// it away (see [`crate::scan::split_by::SplitBy::LayoutSubSplitting`]). pub struct RowSplits { splits: Vec, /// Start index of the run currently being appended. run_start: usize, /// Start index of the surviving run immediately before `run_start`. prev_run_start: usize, + /// Boundaries marked soft, unsorted and possibly repeated. + soft: Vec, } impl RowSplits { @@ -162,15 +168,45 @@ impl RowSplits { self.splits.reserve(additional); } + /// Mark boundaries soft: ones that only some fields of a struct have. They must also be + /// pushed. + pub fn mark_soft(&mut self, rows: impl IntoIterator) { + self.soft.extend(rows); + } + + /// Push `other`'s boundaries into this set as a batch, keeping its soft marks. + pub(crate) fn append(&mut self, mut other: RowSplits) { + let soft = std::mem::take(&mut other.soft); + self.extend_ascending(other.into_sorted_deduped()); + self.soft.extend(soft); + } + + /// Remove soft marks at the supplied sorted boundaries. + pub(crate) fn remove_soft(&mut self, rows: &[u64]) { + debug_assert!(rows.is_sorted()); + if !rows.is_empty() { + self.soft.retain(|row| rows.binary_search(row).is_err()); + } + } + /// Create a new RowSplits with preallocated "capacity" pub(crate) fn new_capacity(capacity: usize) -> Self { Self { splits: Vec::with_capacity(capacity), run_start: 0, prev_run_start: 0, + soft: Vec::new(), } } + /// The sorted, deduplicated boundaries, and the sorted soft ones among them. + pub(crate) fn into_sorted_deduped_with_soft(mut self) -> (Vec, Vec) { + let mut soft = std::mem::take(&mut self.soft); + soft.sort_unstable(); + soft.dedup(); + (self.into_sorted_deduped(), soft) + } + pub(crate) fn into_sorted_deduped(mut self) -> Vec { let final_run_dropped = self.drop_repeated_run(); // Surviving runs always have a descent between them, so the boundaries are ascending diff --git a/vortex-layout/src/scan/split_by.rs b/vortex-layout/src/scan/split_by.rs index b995e46222f..d7d96c1781a 100644 --- a/vortex-layout/src/scan/split_by.rs +++ b/vortex-layout/src/scan/split_by.rs @@ -19,6 +19,14 @@ use crate::scan::IDEAL_SPLIT_SIZE; /// be decoded across multiple cores rather than one. pub const DEFAULT_MAX_SPLIT_ROWS: u64 = IDEAL_SPLIT_SIZE; +/// The narrowest span [`SplitBy::LayoutSubSplitting`] leaves ending at a soft boundary, one that +/// only some fields of a struct have; narrower spans merge into a neighbour. It is the writer's +/// default row block, and is lowered to `max_rows` when that is smaller. +/// +/// A split decodes the whole chunk of every column it overlaps, so splits much narrower than a +/// column's chunks decode those chunks once each. +pub const DEFAULT_MIN_SPLIT_ROWS: u64 = 8192; + /// Defines how the Vortex file is split into batches for reading. /// /// Note that each split must fit into the platform's maximum usize. @@ -28,10 +36,11 @@ pub enum SplitBy { /// splits that follow the file's chunking exactly, trading intra-file decode parallelism /// for fewer, larger batches. Layout, - /// Splits like [`SplitBy::Layout`], except that spans between adjacent chunk boundaries - /// wider than `max_rows` are further sub-divided into evenly sized splits of at most - /// `max_rows` rows, so that a file with few, large chunks can still be decoded across - /// multiple cores. `max_rows` must be non-zero. + /// Splits like [`SplitBy::Layout`], except that spans narrower than + /// [`DEFAULT_MIN_SPLIT_ROWS`] (or `max_rows`, if smaller) ending at a soft boundary are merged + /// into a neighbour, and spans wider than `max_rows` are further sub-divided into evenly sized + /// splits of at most `max_rows` rows, so that a file with few, large chunks can still be + /// decoded across multiple cores. `max_rows` must be non-zero. /// /// This is the default, with [`DEFAULT_MAX_SPLIT_ROWS`]. LayoutSubSplitting { max_rows: u64 }, @@ -64,8 +73,10 @@ impl SplitBy { max_rows > 0, "SplitBy::LayoutSubSplitting requires a non-zero max_rows" ); + let (boundaries, soft) = register_boundaries(layout_reader, row_range, field_mask)? + .into_sorted_deduped_with_soft(); subdivide_large_spans( - layout_boundaries(layout_reader, row_range, field_mask)?, + merge_small_spans(boundaries, &soft, DEFAULT_MIN_SPLIT_ROWS.min(max_rows)), max_rows, ) } @@ -85,6 +96,16 @@ fn layout_boundaries( row_range: &Range, field_mask: &[FieldMask], ) -> VortexResult> { + Ok(register_boundaries(layout_reader, row_range, field_mask)?.into_sorted_deduped()) +} + +/// The boundaries the layout registers within `row_range`, after the range's start (the layout +/// registers its end). +fn register_boundaries( + layout_reader: &dyn LayoutReader, + row_range: &Range, + field_mask: &[FieldMask], +) -> VortexResult { // We usually have under 100 splits so reserving upfront saves us some allocations let mut row_splits = RowSplits::new_capacity(128); row_splits.push(row_range.start); @@ -93,7 +114,53 @@ fn layout_boundaries( &SplitRange::root(row_range.clone())?, &mut row_splits, )?; - Ok(row_splits.into_sorted_deduped()) + Ok(row_splits) +} + +/// Remove interior split boundaries in the sorted `soft` until no span ending at one is narrower +/// than `min_span`. +/// +/// Like [`subdivide_large_spans`], this works on the sorted, deduplicated boundary list. It never +/// adds or moves a boundary, and always keeps the first, the last and every boundary not in `soft`, +/// so the ranges still cover the same rows contiguously, each exactly once. Boundaries are dropped +/// from the front: a soft boundary is kept once the span since the last kept one reaches +/// `min_span`. A short span left before the end, or before a hard boundary, keeps its end and drops +/// its start where that is soft. +fn merge_small_spans(boundaries: Vec, soft: &[u64], min_span: u64) -> Vec { + debug_assert!(boundaries.is_sorted(), "boundaries must be sorted"); + debug_assert!(soft.is_sorted(), "soft boundaries must be sorted"); + let is_hard = |b: u64| soft.binary_search(&b).is_err(); + + // Fast path: nothing to merge (also covers the empty / single-boundary cases). + if soft.is_empty() + || boundaries.len() < 3 + || boundaries.windows(2).all(|w| w[1] - w[0] >= min_span) + { + return boundaries; + } + + let last = boundaries[boundaries.len() - 1]; + let mut out = Vec::with_capacity(boundaries.len()); + out.push(boundaries[0]); + for &boundary in &boundaries[1..] { + let prev = out[out.len() - 1]; + if boundary == last || is_hard(boundary) { + // A short span before a kept boundary merges into the one before it, unless its + // start must be kept too. + if boundary - prev < min_span && out.len() > 1 && !is_hard(prev) { + out.pop(); + } + out.push(boundary); + } else if boundary - prev >= min_span { + out.push(boundary); + } + } + + debug_assert!( + out.windows(2).all(|w| w[0] < w[1]), + "merged boundaries must stay strictly increasing" + ); + out } /// Sub-divide any gap between adjacent split boundaries that is wider than `max_span` into evenly @@ -156,6 +223,10 @@ mod test { use std::sync::Arc; use futures::future::BoxFuture; + use rand::RngExt; + use rand::SeedableRng; + use rand::prelude::StdRng; + use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -363,6 +434,67 @@ mod test { assert_eq!(subdivide_large_spans(vec![7], 100), vec![7]); } + #[rstest] + #[case::empty(vec![], vec![], 5, vec![])] + #[case::single(vec![7], vec![7], 5, vec![7])] + #[case::hard(vec![0, 1, 2, 10], vec![], 5, vec![0, 1, 2, 10])] + #[case::soft(vec![0, 1, 2, 10], vec![1, 2], 5, vec![0, 10])] + #[case::short_tail(vec![0, 5, 6], vec![5], 5, vec![0, 6])] + #[case::hard_collision(vec![0, 3, 5, 6], vec![3, 5], 5, vec![0, 6])] + #[case::hard_start(vec![0, 3, 5, 6], vec![5], 5, vec![0, 3, 6])] + #[case::threshold(vec![0, 5, 10], vec![5], 5, vec![0, 5, 10])] + #[case::nonzero_range(vec![100, 101, 102, 110], vec![101, 102], 5, vec![100, 110])] + fn merge_small_spans_cases( + #[case] boundaries: Vec, + #[case] soft: Vec, + #[case] minimum: u64, + #[case] expected: Vec, + ) { + assert_eq!(merge_small_spans(boundaries, &soft, minimum), expected); + } + + #[test] + fn merging_preserves_hard_boundaries_and_exact_coverage() { + let mut rng = StdRng::seed_from_u64(67); + for _ in 0..1000 { + let start = rng.random_range(0..1000u64); + let mut boundaries = vec![start]; + for _ in 0..rng.random_range(1..50) { + boundaries.push(boundaries.last().unwrap() + rng.random_range(1..1000)); + } + let soft = boundaries + .iter() + .copied() + .filter(|_| rng.random_bool(0.5)) + .collect::>(); + let max_rows = rng.random_range(1..3000); + let minimum = DEFAULT_MIN_SPLIT_ROWS.min(max_rows); + let merged = merge_small_spans(boundaries.clone(), &soft, minimum); + assert_eq!(merged.first(), boundaries.first()); + assert_eq!(merged.last(), boundaries.last()); + assert!(merged.windows(2).all(|w| w[0] < w[1])); + assert!( + merged + .iter() + .all(|row| boundaries.binary_search(row).is_ok()) + ); + assert!(boundaries.iter().all(|row| { + soft.binary_search(row).is_ok() || merged.binary_search(row).is_ok() + })); + assert!(merged.windows(2).all(|w| { + Some(&w[1]) == merged.last() + || soft.binary_search(&w[1]).is_err() + || w[1] - w[0] >= minimum + })); + let subdivided = subdivide_large_spans(merged, max_rows); + assert!(subdivided.windows(2).all(|w| w[1] - w[0] <= max_rows)); + assert_eq!( + subdivided.windows(2).map(|w| w[1] - w[0]).sum::(), + boundaries.last().unwrap() - start, + ); + } + } + #[test] fn subdivide_near_u64_max_does_not_overflow() { // The increment past the last interior point would overflow without saturating math.