Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 59 additions & 42 deletions vortex-array/src/arrays/varbin/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ use vortex_error::vortex_bail;
use vortex_error::vortex_ensure;
use vortex_error::vortex_ensure_eq;
use vortex_error::vortex_err;
use vortex_mask::Mask;

use crate::ArrayRef;
use crate::ArraySlots;
Expand Down Expand Up @@ -343,22 +344,6 @@ impl VarBinData {
validity: &Validity,
ctx: &mut ExecutionCtx,
) -> VortexResult<()> {
let validate_at = |i: usize, start: usize, end: usize| -> VortexResult<()> {
let string_bytes = bytes.get(start..end).ok_or_else(|| {
vortex_err!(
"offsets {start}..{end} at index {i} are out of order or out of bounds for bytes of length {}",
bytes.len()
)
})?;
simdutf8::basic::from_utf8(string_bytes).map_err(|_| {
#[expect(clippy::unwrap_used)]
// run validation using `compat` package to get more detailed error message
let err = simdutf8::compat::from_utf8(string_bytes).unwrap_err();
vortex_err!("invalid utf-8: {err} at index {i}")
})?;
Ok(())
};

// TODO(joe): update the created VarBin with this decompressed Array.
let primitive_offsets = offsets.clone().execute::<PrimitiveArray>(ctx)?;

Expand All @@ -374,36 +359,68 @@ impl VarBinData {
let all_invalid = validity.definitely_all_null();

match_each_integer_ptype!(primitive_offsets.dtype().as_ptype(), |O| {
let offsets_slice = primitive_offsets.as_slice::<O>();

let last_offset: usize = offsets_slice[offsets_slice.len() - 1].as_();
vortex_ensure!(
last_offset <= bytes.len(),
"Last offset {} exceeds bytes length {}",
last_offset,
bytes.len()
);
Self::validate_utf8_offsets(
offsets,
primitive_offsets.as_slice::<O>(),
bytes,
mask.as_ref(),
all_invalid,
ctx,
)
})
}

fn validate_utf8_offsets<O: AsPrimitive<usize> + PartialOrd>(
offsets: &ArrayRef,
offsets_slice: &[O],
bytes: &[u8],
mask: Option<&Mask>,
all_invalid: bool,
ctx: &mut ExecutionCtx,
) -> VortexResult<()> {
let last_offset: usize = offsets_slice[offsets_slice.len() - 1].as_();
vortex_ensure!(
last_offset <= bytes.len(),
"Last offset {} exceeds bytes length {}",
last_offset,
bytes.len()
);

if offsets_tile_utf8(offsets_slice, bytes) {
return Ok(());
}
if offsets_tile_utf8(offsets_slice, bytes) {
return Ok(());
}

// The fast path proves the offsets are valid. The per-string check below skips null
// rows, so it cannot find bad offsets at those rows.
Self::validate_offsets(offsets, bytes.len(), ctx)?;
// The fast path proves the offsets are valid. The per-string check below skips null
// rows, so it cannot find bad offsets at those rows.
Self::validate_offsets(offsets, bytes.len(), ctx)?;

// Invalid bytes at a null row fail the check above, so check valid strings one by one.
for (i, (start, end)) in offsets_slice
.windows(2)
.map(|o| (o[0].as_(), o[1].as_()))
.enumerate()
{
let valid = mask.as_ref().map_or(!all_invalid, |mask| mask.value(i));
if valid {
validate_at(i, start, end)?;
}
let validate_at = |i: usize, start: usize, end: usize| -> VortexResult<()> {
let string_bytes = bytes.get(start..end).ok_or_else(|| {
vortex_err!(
"offsets {start}..{end} at index {i} are out of order or out of bounds for bytes of length {}",
bytes.len()
)
})?;
simdutf8::basic::from_utf8(string_bytes).map_err(|_| {
#[expect(clippy::unwrap_used)]
// run validation using `compat` package to get more detailed error message
let err = simdutf8::compat::from_utf8(string_bytes).unwrap_err();
vortex_err!("invalid utf-8: {err} at index {i}")
})?;
Ok(())
};

// Invalid bytes at a null row fail the check above, so check valid strings one by one.
for (i, (start, end)) in offsets_slice
.windows(2)
.map(|o| (o[0].as_(), o[1].as_()))
.enumerate()
{
let valid = mask.map_or(!all_invalid, |mask| mask.value(i));
if valid {
validate_at(i, start, end)?;
}
});
}
Ok(())
}

Expand Down
Loading