Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion vortex-file/benches/split_collection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ fn main() {
const ROWS_PER_CHUNK: usize = 1024;

/// (columns, chunks) configurations.
const CONFIGS: &[(usize, usize)] = &[(16, 64)];
const CONFIGS: &[(usize, usize)] = &[(16, 64), (256, 64)];

static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
tokio::runtime::Builder::new_current_thread()
Expand Down
198 changes: 193 additions & 5 deletions vortex-layout/src/layouts/struct_/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -523,15 +523,54 @@ impl LayoutReader for StructReader {
split_range: &SplitRange,
splits: &mut RowSplits,
) -> VortexResult<()> {
// Register splits for the validity child, if there is one
if let Some(validity_ref) = self.validity()? {
validity_ref.register_splits(field_mask, split_range, splits)?;
}
// Validity boundaries must stay hard even when they overlap a soft data-field boundary.
let validity_boundaries = if let Some(validity_ref) = self.validity()? {
let mut validity_splits = RowSplits::new_capacity(16);
validity_ref.register_splits(field_mask, split_range, &mut validity_splits)?;
validity_splits.into_sorted_deduped()
} else {
Vec::new()
};
splits.extend_ascending(validity_boundaries.iter().copied());

// Each field's boundaries, kept apart so the ones only some fields have can be marked soft.
let mut fields: Vec<RowSplits> = Vec::new();
self.layout.matching_fields(field_mask, |mask, idx| {
let mut field = RowSplits::new_capacity(16);
self.field_reader_by_index(idx)?
.register_splits(&[mask], split_range, splits)
.register_splits(&[mask], split_range, &mut field)?;
fields.push(field);
Ok(())
})?;
let mut field_splits = RowSplits::new_capacity(0);
if let [field] = &mut fields[..] {
field_splits = std::mem::replace(field, RowSplits::new_capacity(0));
} else if !fields.is_empty() {
let fields: Vec<(Vec<u64>, Vec<u64>)> = fields
.into_iter()
.map(RowSplits::into_sorted_deduped_with_soft)
.collect();
// Intersect once; checking every field for every boundary is quadratic in field count.
let mut shared = fields[0].0.clone();
for (boundaries, _) in &fields[1..] {
shared.retain(|row| boundaries.binary_search(row).is_ok());
if shared.is_empty() {
break;
}
}
for (boundaries, soft) in fields.iter() {
field_splits.extend_ascending(boundaries.iter().copied());
field_splits.mark_soft(soft.iter().copied());
field_splits.mark_soft(
boundaries
.iter()
.copied()
.filter(|row| shared.binary_search(row).is_err()),
);
}
}
field_splits.remove_soft(&validity_boundaries);
splits.append(field_splits);

// In the case of an empty struct, we need to register the end split. Pushed last so it
// extends the final field's ascending run rather than starting a run of its own.
Expand Down Expand Up @@ -679,7 +718,9 @@ mod tests {
use vortex_array::assert_arrays_eq;
use vortex_array::assert_nth_scalar;
use vortex_array::dtype::DType;
use vortex_array::dtype::FieldMask;
use vortex_array::dtype::FieldName;
use vortex_array::dtype::FieldPath;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
use vortex_array::dtype::RESERVED_VALIDITY_FIELD_NAME;
Expand Down Expand Up @@ -712,11 +753,16 @@ mod tests {
use crate::LayoutReaderRef;
use crate::LayoutRef;
use crate::LayoutStrategy;
use crate::RowSplits;
use crate::SplitRange;
use crate::layouts::chunked::writer::ChunkedLayoutStrategy;
use crate::layouts::flat::writer::FlatLayoutStrategy;
use crate::layouts::repartition::RepartitionStrategy;
use crate::layouts::repartition::RepartitionWriterOptions;
use crate::layouts::table::TableStrategy;
use crate::layouts::zoned::writer::ZonedLayoutOptions;
use crate::layouts::zoned::writer::ZonedStrategy;
use crate::scan::split_by::SplitBy;
use crate::segments::CountingSegmentSource;
use crate::segments::SegmentFuture;
use crate::segments::SegmentId;
Expand Down Expand Up @@ -803,6 +849,148 @@ mod tests {
(segments, layout)
}

/// A leaf writer that chunks every `rows` rows.
fn chunked_every(rows: usize) -> Arc<dyn LayoutStrategy> {
Arc::new(RepartitionStrategy::new(
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
RepartitionWriterOptions {
block_size_minimum: 0,
block_len_multiple: rows,
block_size_target: None,
canonicalize: false,
},
))
}

/// A struct of 12,000 rows whose field `a` chunks every 1,000 rows and `b` every 3,000,
/// written in `pieces` pushes. With `chunked`, the struct is itself a chunk of a chunked layout
/// per push.
fn misaligned_struct_layout(
pieces: usize,
chunked: bool,
nullable: bool,
) -> (Arc<dyn SegmentSource>, LayoutRef) {
let ctx = ArrayContext::empty();
let segments = Arc::new(TestSegments::default());
let (ptr, eof) = SequenceId::root().split();
let table = TableStrategy::new(chunked_every(2000), chunked_every(1000))
.with_field_writer(FieldPath::from_name("b"), chunked_every(3000));
let strategy: Arc<dyn LayoutStrategy> = if chunked {
Arc::new(ChunkedLayoutStrategy::new(table))
} else {
Arc::new(table)
};
let rows = 12_000 / pieces;
let chunks: Vec<ArrayRef> = (0..pieces)
.map(|piece| {
let start = i32::try_from(piece * rows).unwrap();
let end = start + i32::try_from(rows).unwrap();
StructArray::try_from_iter_with_validity(
[
("a", PrimitiveArray::from_iter(start..end).into_array()),
("b", PrimitiveArray::from_iter(start..end).into_array()),
],
if nullable {
Validity::Array(
BoolArray::from_iter((start..end).map(|row| row % 2 == 0)).into_array(),
)
} else {
Validity::NonNullable
},
)
.unwrap()
.into_array()
})
.collect();
let dtype = chunks[0].dtype().clone();
let segments2 = Arc::<TestSegments>::clone(&segments);
let layout = block_on(|handle| async move {
let session = new_session().with_handle(handle);
strategy
.write_stream(
ctx.into(),
segments2,
ChunkedArray::try_new(chunks, dtype)
.unwrap()
.into_array()
.to_array_stream()
.sequenced(ptr),
eof,
&session,
)
.await
})
.unwrap();
(segments, layout)
}

/// Registers `layout`'s boundaries over its rows, as a scan does.
fn boundaries_and_soft(
segments: Arc<dyn SegmentSource>,
layout: &LayoutRef,
) -> (Vec<u64>, Vec<u64>) {
let reader = layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap();
let mut splits = RowSplits::new_capacity(16);
splits.push(0);
reader
.register_splits(
&[FieldMask::All],
&SplitRange::root(0..reader.row_count()).unwrap(),
&mut splits,
)
.unwrap();
splits.into_sorted_deduped_with_soft()
}

#[test]
fn boundaries_only_some_fields_have_are_soft() {
let (segments, layout) = misaligned_struct_layout(1, false, false);
let (boundaries, soft) = boundaries_and_soft(segments, &layout);
// `a` chunks every 1,000 rows and `b` every 3,000, so only `b`'s are shared.
assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::<Vec<u64>>());
assert_eq!(
soft,
(1..12)
.filter(|k| k % 3 != 0)
.map(|k| k * 1000)
.collect::<Vec<u64>>()
);
}

#[test]
fn chunk_ends_above_the_struct_stay_hard() {
// Two top-level chunks of 6,000 rows, each a struct whose `a` chunks every 1,000 rows and
// `b` every 3,000: the top-level chunk end at 6,000 is every column's, so it stays hard.
let (segments, layout) = misaligned_struct_layout(2, true, false);
let (boundaries, soft) = boundaries_and_soft(segments, &layout);
assert_eq!(boundaries, (0..=12).map(|k| k * 1000).collect::<Vec<u64>>());
assert!(!soft.contains(&6000), "the chunk end is soft: {soft:?}");
assert!(!soft.contains(&3000) && !soft.contains(&9000), "{soft:?}");
assert!(soft.contains(&1000) && soft.contains(&7000), "{soft:?}");
}

#[rstest]
#[case::non_nullable(false, vec![0, 3000, 6000, 9000, 12000])]
#[case::nullable(true, vec![0, 2000, 3000, 4000, 6000, 8000, 9000, 10000, 12000])]
fn scan_merges_soft_boundaries_and_keeps_validity_boundaries(
#[case] nullable: bool,
#[case] expected: Vec<u64>,
) -> VortexResult<()> {
let (segments, layout) = misaligned_struct_layout(1, false, nullable);
let reader = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;
assert_eq!(
SplitBy::default().splits(
reader.as_ref(),
&(0..reader.row_count()),
&[FieldMask::All],
)?,
expected,
);
Ok(())
}

#[fixture]
/// Create a chunked layout with three chunks of primitive arrays.
fn null_struct_layout() -> (Arc<dyn SegmentSource>, LayoutRef) {
Expand Down
36 changes: 36 additions & 0 deletions vortex-layout/src/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,18 @@ impl SplitRange {
/// starts a new run. A run that exactly repeats the previous surviving run — common when sibling
/// columns share chunk boundaries — is dropped as it completes, and the final sort is skipped
/// when only a single run survives (the boundaries are then already ascending).
///
/// A boundary can also be marked soft: one that only some of a struct's fields have. A split there
/// lets the fields that chunk across it decode the same chunk in two splits, so the scan may merge
/// it away (see [`crate::scan::split_by::SplitBy::LayoutSubSplitting`]).
pub struct RowSplits {
splits: Vec<u64>,
/// Start index of the run currently being appended.
run_start: usize,
/// Start index of the surviving run immediately before `run_start`.
prev_run_start: usize,
/// Boundaries marked soft, unsorted and possibly repeated.
soft: Vec<u64>,
}

impl RowSplits {
Expand Down Expand Up @@ -162,15 +168,45 @@ impl RowSplits {
self.splits.reserve(additional);
}

/// Mark boundaries soft: ones that only some fields of a struct have. They must also be
/// pushed.
pub fn mark_soft(&mut self, rows: impl IntoIterator<Item = u64>) {
self.soft.extend(rows);
}

/// Push `other`'s boundaries into this set as a batch, keeping its soft marks.
pub(crate) fn append(&mut self, mut other: RowSplits) {
let soft = std::mem::take(&mut other.soft);
self.extend_ascending(other.into_sorted_deduped());
self.soft.extend(soft);
}

/// Remove soft marks at the supplied sorted boundaries.
pub(crate) fn remove_soft(&mut self, rows: &[u64]) {
debug_assert!(rows.is_sorted());
if !rows.is_empty() {
self.soft.retain(|row| rows.binary_search(row).is_err());
}
}

/// Create a new RowSplits with preallocated "capacity"
pub(crate) fn new_capacity(capacity: usize) -> Self {
Self {
splits: Vec::with_capacity(capacity),
run_start: 0,
prev_run_start: 0,
soft: Vec::new(),
}
}

/// The sorted, deduplicated boundaries, and the sorted soft ones among them.
pub(crate) fn into_sorted_deduped_with_soft(mut self) -> (Vec<u64>, Vec<u64>) {
let mut soft = std::mem::take(&mut self.soft);
soft.sort_unstable();
soft.dedup();
(self.into_sorted_deduped(), soft)
}

pub(crate) fn into_sorted_deduped(mut self) -> Vec<u64> {
let final_run_dropped = self.drop_repeated_run();
// Surviving runs always have a descent between them, so the boundaries are ascending
Expand Down
Loading
Loading