From c4d23b9b4efe2927cc782166d41a45ba37bc098b Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 18 Nov 2026 14:01:15 -0500 Subject: [PATCH] Move VarBin UTF-8 validation out of type dispatch Signed-off-by: Connor Tsui --- vortex-array/src/arrays/varbin/array.rs | 101 ++++++++++++++---------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 083513389f9..eecb3776513 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -14,6 +14,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_mask::Mask; use crate::ArrayRef; use crate::ArraySlots; @@ -343,22 +344,6 @@ impl VarBinData { validity: &Validity, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - let validate_at = |i: usize, start: usize, end: usize| -> VortexResult<()> { - let string_bytes = bytes.get(start..end).ok_or_else(|| { - vortex_err!( - "offsets {start}..{end} at index {i} are out of order or out of bounds for bytes of length {}", - bytes.len() - ) - })?; - simdutf8::basic::from_utf8(string_bytes).map_err(|_| { - #[expect(clippy::unwrap_used)] - // run validation using `compat` package to get more detailed error message - let err = simdutf8::compat::from_utf8(string_bytes).unwrap_err(); - vortex_err!("invalid utf-8: {err} at index {i}") - })?; - Ok(()) - }; - // TODO(joe): update the created VarBin with this decompressed Array. let primitive_offsets = offsets.clone().execute::(ctx)?; @@ -374,36 +359,68 @@ impl VarBinData { let all_invalid = validity.definitely_all_null(); match_each_integer_ptype!(primitive_offsets.dtype().as_ptype(), |O| { - let offsets_slice = primitive_offsets.as_slice::(); - - let last_offset: usize = offsets_slice[offsets_slice.len() - 1].as_(); - vortex_ensure!( - last_offset <= bytes.len(), - "Last offset {} exceeds bytes length {}", - last_offset, - bytes.len() - ); + Self::validate_utf8_offsets( + offsets, + primitive_offsets.as_slice::(), + bytes, + mask.as_ref(), + all_invalid, + ctx, + ) + }) + } + + fn validate_utf8_offsets + PartialOrd>( + offsets: &ArrayRef, + offsets_slice: &[O], + bytes: &[u8], + mask: Option<&Mask>, + all_invalid: bool, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let last_offset: usize = offsets_slice[offsets_slice.len() - 1].as_(); + vortex_ensure!( + last_offset <= bytes.len(), + "Last offset {} exceeds bytes length {}", + last_offset, + bytes.len() + ); - if offsets_tile_utf8(offsets_slice, bytes) { - return Ok(()); - } + if offsets_tile_utf8(offsets_slice, bytes) { + return Ok(()); + } - // The fast path proves the offsets are valid. The per-string check below skips null - // rows, so it cannot find bad offsets at those rows. - Self::validate_offsets(offsets, bytes.len(), ctx)?; + // The fast path proves the offsets are valid. The per-string check below skips null + // rows, so it cannot find bad offsets at those rows. + Self::validate_offsets(offsets, bytes.len(), ctx)?; - // Invalid bytes at a null row fail the check above, so check valid strings one by one. - for (i, (start, end)) in offsets_slice - .windows(2) - .map(|o| (o[0].as_(), o[1].as_())) - .enumerate() - { - let valid = mask.as_ref().map_or(!all_invalid, |mask| mask.value(i)); - if valid { - validate_at(i, start, end)?; - } + let validate_at = |i: usize, start: usize, end: usize| -> VortexResult<()> { + let string_bytes = bytes.get(start..end).ok_or_else(|| { + vortex_err!( + "offsets {start}..{end} at index {i} are out of order or out of bounds for bytes of length {}", + bytes.len() + ) + })?; + simdutf8::basic::from_utf8(string_bytes).map_err(|_| { + #[expect(clippy::unwrap_used)] + // run validation using `compat` package to get more detailed error message + let err = simdutf8::compat::from_utf8(string_bytes).unwrap_err(); + vortex_err!("invalid utf-8: {err} at index {i}") + })?; + Ok(()) + }; + + // Invalid bytes at a null row fail the check above, so check valid strings one by one. + for (i, (start, end)) in offsets_slice + .windows(2) + .map(|o| (o[0].as_(), o[1].as_())) + .enumerate() + { + let valid = mask.map_or(!all_invalid, |mask| mask.value(i)); + if valid { + validate_at(i, start, end)?; } - }); + } Ok(()) }