From 115318b9107af93ee327f658f7459316ead9d8a8 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Thu, 19 Nov 2026 11:14:57 -0500 Subject: [PATCH] Push DataFusion field reads through extension storage DataFusion can see a struct where Vortex sees an opaque extension value. Expose the storage explicitly when the Arrow field uses storage export, so the scan can project the requested field without bypassing exporters that transform values. Signed-off-by: Connor Tsui --- vortex-arrow/src/executor/mod.rs | 17 +- vortex-arrow/src/session.rs | 186 +++++++++++++++++++ vortex-datafusion/src/convert/exprs/mod.rs | 119 ++++++++---- vortex-datafusion/src/convert/exprs/tests.rs | 169 +++++++++++++++++ vortex-datafusion/src/lib.rs | 62 +++++++ vortex-datafusion/src/v2/tests.rs | 139 ++++++++++++++ 6 files changed, 643 insertions(+), 49 deletions(-) diff --git a/vortex-arrow/src/executor/mod.rs b/vortex-arrow/src/executor/mod.rs index a5e88a498a9..488c46209ed 100644 --- a/vortex-arrow/src/executor/mod.rs +++ b/vortex-arrow/src/executor/mod.rs @@ -69,9 +69,7 @@ use crate::executor::struct_::to_arrow_struct; use crate::executor::temporal::to_arrow_date; use crate::executor::temporal::to_arrow_time; use crate::executor::temporal::to_arrow_timestamp; -use crate::ext_metadata::decode_arrow_ext_metadata; use crate::session::ArrowSessionExt; -use crate::session::ensure_ext_metadata; /// Execution of a Vortex array into an Arrow array, for every type that implements [`IntoArray`]. /// @@ -326,20 +324,9 @@ fn execute_arrow_ext_storage( options: &ArrowExportOptions, ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(importer_dtype) = ctx - .session() + ctx.session() .arrow() - .infer_field_dtype_with_importers(target)? - { - vortex_ensure_eq!( - importer_dtype.as_nullable(), - array.dtype().as_nullable(), - "Arrow field {:?} must describe the source extension type", - target.name() - ); - } else { - ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?; - } + .ensure_ext_storage_metadata(ext_dtype, target)?; let target_dtype = ctx .session() diff --git a/vortex-arrow/src/session.rs b/vortex-arrow/src/session.rs index d0bc576d14d..ca45f4e76ce 100644 --- a/vortex-arrow/src/session.rs +++ b/vortex-arrow/src/session.rs @@ -559,6 +559,96 @@ impl ArrowSession { Ok(field) } + /// Returns whether an export of `ext_dtype` to `field` produces exactly its storage. + /// + /// A caller can use the answer to plan reads into the storage without the array. The export + /// produces the storage only when `field` names the Vortex ID of `ext_dtype` and passes the + /// checks that the [`ArrowSession`] docs describe for such a field. A registered exporter can + /// produce any data for the same field, so a name with an exporter gives `false`. + /// + /// The answer covers only the top level of `field`. Each storage child exports to its own + /// field, which can name another exporter, and [`Self::exports_without_plugins`] checks every + /// depth. + #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")] + pub fn exports_extension_storage(&self, ext_dtype: &ExtDTypeRef, field: &Field) -> bool { + // A dictionary or run-end encoded field holds the storage values behind an encoding, so + // its Arrow data is not the storage, even though it can import as the storage dtype. + if matches!( + field.data_type(), + DataType::Dictionary(..) | DataType::RunEndEncoded(..) + ) { + return false; + } + + let Some(name) = field.extension_type_name() else { + return false; + }; + let id = Id::new(name); + + // The export tries the registered exporters before the storage route. + if !self.exporters(&id).is_empty() { + return false; + } + + // The storage route needs a name that the session resolves to the Vortex ID of + // `ext_dtype`. The name can be an alias of that ID. + if self.known_vortex_id(&id) != Some(ext_dtype.id()) { + return false; + } + + // The export fails unless the field describes `ext_dtype`. + if self.ensure_ext_storage_metadata(ext_dtype, field).is_err() { + return false; + } + + // The export also accepts some Arrow types that import as another dtype, such as an Arrow + // timestamp, which imports as the extension type itself. Only an exact match means that + // the Arrow data has the type of the storage. + self.from_arrow_datatype(field.data_type(), ext_dtype.nullability()) + .is_ok_and(|dtype| &dtype == ext_dtype.storage_dtype()) + } + + /// Returns whether an export to `field` calls no registered exporter at any depth. + /// + /// Such an export uses only the conversions of this crate, so a caller planning reads without + /// the array knows which Arrow data it produces. A field that names an extension type without + /// an exporter still qualifies, because it exports through its Vortex ID or as plain storage. + #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")] + pub fn exports_without_plugins(&self, field: &Field) -> bool { + let names_exporter = field + .extension_type_name() + .is_some_and(|name| !self.exporters(&Id::new(name)).is_empty()); + + !names_exporter + && nested_fields(field.data_type()) + .into_iter() + .all(|child| self.exports_without_plugins(child)) + } + + /// Fails unless `field`, which names the Vortex ID of `ext_dtype`, describes `ext_dtype`. + /// + /// An importer that recognizes `field` reads its metadata in the plugin's format, so the check + /// asks the importer first, as [`Self::infer_field_dtype_with_importers`] explains. Without + /// one, the field must carry the metadata of `ext_dtype`. + pub(crate) fn ensure_ext_storage_metadata( + &self, + ext_dtype: &ExtDTypeRef, + field: &Field, + ) -> VortexResult<()> { + let Some(importer_dtype) = self.infer_field_dtype_with_importers(field)? else { + return ensure_ext_metadata(field, &decode_arrow_ext_metadata(field)?, ext_dtype); + }; + + vortex_ensure_eq!( + importer_dtype.as_nullable(), + DType::Extension(ext_dtype.with_nullability(Nullability::Nullable)), + "Arrow field {:?} must describe the source extension type", + field.name() + ); + + Ok(()) + } + /// Returns the Vortex ID that the extension name `id` resolves to, or `None` if the session does /// not know it. The registry can hold an alias, so the result can differ from `id`. pub(crate) fn known_vortex_id(&self, id: &Id) -> Option { @@ -1116,6 +1206,26 @@ pub(crate) fn ensure_ext_metadata( Ok(()) } +/// Returns the fields nested directly in `data_type`. +/// +/// A dictionary has no field for its values, so it returns the fields nested in the value type. +fn nested_fields(data_type: &DataType) -> Vec<&Field> { + match data_type { + DataType::Struct(fields) => fields.iter().map(Arc::as_ref).collect(), + DataType::Union(fields, _) => fields.iter().map(|(_, field)| field.as_ref()).collect(), + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) + | DataType::RunEndEncoded(_, field) => vec![field.as_ref()], + DataType::Dictionary(_, values) => nested_fields(values), + // Other Arrow types have no nested fields. + _ => Vec::new(), + } +} + /// Extension trait for accessing the [`ArrowSession`] on a Vortex session. pub trait ArrowSessionExt: SessionExt { /// Get the Arrow session, creating it on first use. @@ -1795,4 +1905,80 @@ mod tests { ); Ok(()) } + + const EXPORTER_NAME: &str = "test.exporter"; + static EXPORTER_ID: CachedId = CachedId::new(EXPORTER_NAME); + + /// Declines every array, because the tests only need an exporter registered under its name. + #[derive(Debug)] + struct DecliningExporter; + + impl ArrowExportVTable for DecliningExporter { + fn arrow_ext_id(&self) -> Id { + *EXPORTER_ID + } + + fn vortex_id(&self) -> Id { + *EXPORTER_ID + } + + fn to_arrow_field( + &self, + _name: &str, + _dtype: &DType, + _session: &ArrowSession, + ) -> VortexResult> { + Ok(None) + } + + fn execute_arrow( + &self, + array: ArrayRef, + _target: &Field, + _options: &ArrowExportOptions, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ArrowExport::Unsupported(array)) + } + } + + /// An `Int32` field that names `extension_name`, or a plain one without it. + fn leaf(extension_name: Option<&str>) -> FieldRef { + let mut leaf = Field::new("leaf", DataType::Int32, true); + if let Some(name) = extension_name { + leaf = leaf + .with_metadata([(EXTENSION_TYPE_NAME_KEY.to_string(), name.to_string())].into()); + } + + Arc::new(leaf) + } + + /// Only a leaf that names a registered exporter makes an export call a plugin, at any depth. + #[rstest] + #[case::top_level( |leaf: FieldRef| leaf.as_ref().clone())] + #[case::struct_child( |leaf: FieldRef| Field::new_struct("a", vec![leaf], true))] + #[case::list_item( |leaf: FieldRef| Field::new_list("a", leaf, true))] + #[case::run_end_values(|leaf: FieldRef| { + let run_ends = Arc::new(Field::new("run_ends", DataType::Int32, false)); + Field::new("a", DataType::RunEndEncoded(run_ends, leaf), true) + })] + #[case::dictionary_values(|leaf: FieldRef| { + Field::new_dictionary("a", DataType::Int32, DataType::Struct(vec![leaf].into()), true) + })] + fn test_exports_without_plugins_finds_exporters_at_any_depth( + #[case] wrap: fn(FieldRef) -> Field, + #[values(None, Some("test.unregistered"), Some(EXPORTER_NAME))] leaf_name: Option<&str>, + ) { + let vortex_session = VortexSession::empty(); + let session = vortex_session.arrow(); + session.register_exporter(Arc::new(DecliningExporter)); + + let field = wrap(leaf(leaf_name)); + let leaf_names_exporter = leaf_name == Some(EXPORTER_NAME); + + assert_eq!( + session.exports_without_plugins(&field), + !leaf_names_exporter + ); + } } diff --git a/vortex-datafusion/src/convert/exprs/mod.rs b/vortex-datafusion/src/convert/exprs/mod.rs index 2b0bb323993..6323b2c3c07 100644 --- a/vortex-datafusion/src/convert/exprs/mod.rs +++ b/vortex-datafusion/src/convert/exprs/mod.rs @@ -51,6 +51,7 @@ use vortex::expr::Expression; use vortex::expr::Lambda; use vortex::expr::byte_length; use vortex::expr::cast; +use vortex::expr::ext_storage; use vortex::expr::get_item; use vortex::expr::in_list_of; use vortex::expr::is_not_null; @@ -266,6 +267,8 @@ type Conversion = Result; /// Nested struct field access also uses Vortex semantics: a null parent produces a null child, /// whereas DataFusion extracts the child without applying parent validity. These differences /// are accepted for both filters and projections. +/// Field access through an extension can run natively when Arrow exposes its Vortex storage and no +/// plugin exports the selected field. /// Searched `CASE WHEN` expressions preserve short-circuit evaluation: conditions and /// branch values only execute on the rows that reach them, including fallible branches. /// Other expressions require compatible SQL semantics or remain in DataFusion. @@ -746,40 +749,7 @@ impl DefaultExpressionConverter { } if ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() { - // DataFusion's GetFieldFunc flattens nested field access into a single call - // with multiple field name arguments, e.g. `outer.inner.leaf` becomes - // get_field(Column("outer"), "inner", "leaf"). - let [source, paths @ ..] = scalar_fn.args() else { - return Err( - exec_datafusion_err!("get_field requires a source and field path").into(), - ); - }; - if paths.is_empty() { - return Err(exec_datafusion_err!("get_field requires a field path").into()); - } - let mut source_type = source.data_type(schema)?; - let mut names = Vec::with_capacity(paths.len()); - for path in paths { - let name = path - .downcast_ref::() - .and_then(|literal| literal.value().try_as_str().flatten()) - .ok_or_else(|| { - exec_datafusion_err!("get_field path must be a non-null string literal") - })?; - let DataType::Struct(fields) = &source_type else { - return Err(Unconverted::Unsupported); - }; - let (_, field) = fields.find(name).ok_or_else(|| { - exec_datafusion_err!("get_field references missing field {name}") - })?; - source_type = field.data_type().clone(); - names.push(name); - } - let mut result = self.convert_expr(source, schema, input_dtype)?; - for name in names { - result = get_item(name, result); - } - return Ok(result); + return self.convert_get_field(scalar_fn, schema, input_dtype); } if ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() { @@ -823,6 +793,87 @@ impl DefaultExpressionConverter { Ok(cast(length(input), return_dtype)) } + /// Converts `get_field` to a chain of `get_item` reads. + /// + /// DataFusion can flatten nested access into one call, so `outer.inner.leaf` can arrive as + /// `get_field(outer, "inner", "leaf")`. + /// + /// Vortex extension types are opaque, so `get_item` cannot read into an extension value until + /// `ext_storage` exposes its storage. The converter can add that `ext_storage` without breaking + /// the opacity, because DataFusion already reads the storage: an Arrow extension type is only + /// metadata on its storage field, and `get_field` reads that field like any other struct. Both + /// sides read the same values only when Vortex exports the parent as exactly its storage, which + /// [`exports_extension_storage`] checks. + /// + /// [`exports_extension_storage`]: vortex_arrow::ArrowSession::exports_extension_storage + fn convert_get_field( + &self, + scalar_fn: &ScalarFunctionExpr, + schema: &Schema, + input_dtype: &DType, + ) -> Conversion { + let [source, paths @ ..] = scalar_fn.args() else { + return Err(exec_datafusion_err!("get_field requires a source and field path").into()); + }; + + if paths.is_empty() { + return Err(exec_datafusion_err!("get_field requires a field path").into()); + } + + // Keep the Arrow field of each step, because its metadata decides how Vortex exports it. + let source_field = source.return_field(schema)?; + let mut field = source_field.as_ref(); + let mut steps = Vec::with_capacity(paths.len()); + for path in paths { + let name = path + .downcast_ref::() + .and_then(|literal| literal.value().try_as_str().flatten()) + .ok_or_else(|| { + exec_datafusion_err!("get_field path must be a non-null string literal") + })?; + let DataType::Struct(fields) = field.data_type() else { + return Err(Unconverted::Unsupported); + }; + let (_, child_field) = fields + .find(name) + .ok_or_else(|| exec_datafusion_err!("get_field references missing field {name}"))?; + steps.push((field, name)); + field = child_field.as_ref(); + } + + let mut read = self.convert_expr(source, schema, input_dtype)?; + let mut reads_extension_storage = false; + for (parent_field, name) in steps { + // Check the Vortex dtype instead of the Arrow extension name, because the session + // imports an unknown extension name as a plain struct that `get_item` reads directly. + if let DType::Extension(ext_dtype) = converted_dtype(&read, input_dtype)? { + if !self + .session + .arrow() + .exports_extension_storage(&ext_dtype, parent_field) + { + return Err(Unconverted::Unsupported); + } + + read = ext_storage(read); + reads_extension_storage = true; + } + read = get_item(name, read); + } + + // A plugin exporter decides which Arrow data DataFusion sees for the selected field, so + // Vortex and DataFusion agree on that field only if the export keeps values. Columns and + // plain struct fields already assume this, and the check skips them so that their pushdown + // stays the same. + // TODO(connor): Drop this check once the Arrow plugin contract requires exports to keep + // values. + if reads_extension_storage && !self.session.arrow().exports_without_plugins(field) { + return Err(Unconverted::Unsupported); + } + + Ok(read) + } + /// Converts `array_sum` to `list_sum`. /// /// DataFusion casts the list to `List`, then adds each list's valid elements in diff --git a/vortex-datafusion/src/convert/exprs/tests.rs b/vortex-datafusion/src/convert/exprs/tests.rs index b0ae212c934..4e438af03be 100644 --- a/vortex-datafusion/src/convert/exprs/tests.rs +++ b/vortex-datafusion/src/convert/exprs/tests.rs @@ -54,12 +54,16 @@ use datafusion_physical_expr::projection::ProjectionExpr; use datafusion_physical_plan::expressions as df_expr; use insta::assert_snapshot; use rstest::rstest; +use vortex::array::ArrayRef; use vortex::array::Canonical; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::builtins::ArrayBuiltins; use vortex::dtype::PType; +use vortex::dtype::extension::ExtDTypeRef; +use vortex::dtype::session::DTypeSessionExt; use vortex::encodings::uuid::Uuid; use vortex::error::VortexResult; use vortex::error::vortex_bail; @@ -74,11 +78,16 @@ use vortex::scalar_fn::fns::literal::Literal; use vortex::session::registry::CachedId; use vortex::session::registry::Id; use vortex_array::assert_arrays_eq; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportOptions; +use vortex_arrow::ArrowExportVTable; use vortex_arrow::ArrowImport; use vortex_arrow::ArrowImportVTable; use vortex_arrow::ArrowSession; use super::*; +use crate::common_tests::ANY_STORAGE_ID; +use crate::common_tests::AnyStorage; use crate::common_tests::TestSessionContext; use crate::convert::TryToDataFusion; @@ -2596,6 +2605,166 @@ fn test_native_nested_get_field( Ok(()) } +/// An [`AnyStorage`] point over `Struct`. +fn point_dtype(nullable: bool) -> VortexResult { + let coordinate = DType::Primitive(PType::I32, Nullability::NonNullable); + let storage = DType::struct_( + [("x", coordinate.clone()), ("y", coordinate)], + nullable.into(), + ); + + AnyStorage::dtype(storage) +} + +/// A batch with a `point` column and an `outer` struct around the same points, with the point +/// field that `session` infers. +/// +/// `x` holds 10, 20, 30, and 40. `point` is null at index 1, and `outer` is null at index 2. +fn point_batch(session: &VortexSession) -> anyhow::Result { + let point = session + .arrow() + .to_arrow_field("point", &DType::Extension(point_dtype(true)?))?; + let DataType::Struct(fields) = point.data_type() else { + anyhow::bail!("Expected point storage as a struct, got {point}"); + }; + let points: ArrowArrayRef = Arc::new(StructArray::try_new( + fields.clone(), + vec![ + Arc::new(Int32Array::from(vec![10, 20, 30, 40])), + Arc::new(Int32Array::from(vec![11, 21, 31, 41])), + ], + Some(NullBuffer::from(vec![true, false, true, true])), + )?); + let outer = StructArray::try_new( + vec![point.clone()].into(), + vec![Arc::clone(&points)], + Some(NullBuffer::from(vec![true, true, false, true])), + )?; + let schema = Schema::new(vec![ + point, + Field::new("outer", outer.data_type().clone(), true), + ]); + Ok(RecordBatch::try_new( + Arc::new(schema), + vec![points, Arc::new(outer)], + )?) +} + +/// A registered extension reads its storage through `ext_storage`. The session imports an unknown +/// extension name as plain storage, so the same read is a plain `get_item` with the same values. +#[rstest] +#[case::column( "point", &["x"], false, [Some(10), None, Some(30), Some(40)])] +#[case::flattened("outer", &["point", "x"], false, [Some(10), None, None, Some(40)])] +#[case::chained( "outer", &["point", "x"], true, [Some(10), None, None, Some(40)])] +fn test_get_field_reads_extension_storage( + #[case] column: &str, + #[case] path: &[&str], + #[case] chained: bool, + #[case] expected: [Option; 4], + #[values(false, true)] registered: bool, +) -> anyhow::Result<()> { + let session = VortexSession::default(); + if registered { + session.dtypes().register(AnyStorage); + } + let batch = point_batch(&session)?; + let schema = batch.schema(); + let source = Arc::new(df_expr::Column::new(column, schema.index_of(column)?)); + let expr = if chained { + let (first, rest) = path.split_at(1); + get_field_expr(get_field_expr(source, first, &schema)?, rest, &schema)? + } else { + get_field_expr(source, path, &schema)? + }; + + let converted = DefaultExpressionConverter::new(session.clone()) + .try_convert(&expr, &schema)? + .ok_or_else(|| anyhow::anyhow!("Expected native conversion for {expr}"))?; + let point = if column == "point" { + get_item("point", root()) + } else { + get_item("point", get_item("outer", root())) + }; + let point = if registered { + ext_storage(point) + } else { + point + }; + assert_eq!(converted, get_item("x", point)); + + let input = session.arrow().from_arrow_record_batch(batch, &schema)?; + assert_arrays_eq!( + input.apply(&converted)?, + PrimitiveArray::from_option_iter(expected), + &mut session.create_execution_ctx() + ); + Ok(()) +} + +/// Declines every array, so each export falls through to the Vortex-ID storage export. +/// +/// The converter cannot know that before the export runs, so it treats the plugin as opaque. +#[derive(Debug)] +struct DecliningPointExporter; + +impl ArrowExportVTable for DecliningPointExporter { + fn arrow_ext_id(&self) -> Id { + *ANY_STORAGE_ID + } + + fn vortex_id(&self) -> Id { + *ANY_STORAGE_ID + } + + fn to_arrow_field( + &self, + _name: &str, + _dtype: &DType, + _session: &ArrowSession, + ) -> VortexResult> { + Ok(None) + } + + fn execute_arrow( + &self, + array: ArrayRef, + _target: &Field, + _options: &ArrowExportOptions, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ArrowExport::Unsupported(array)) + } +} + +#[rstest] +#[case::column("point", &["x"])] +#[case::nested("outer", &["point", "x"])] +fn test_get_field_on_plugin_export_stays_in_datafusion( + #[case] column: &str, + #[case] path: &[&str], +) -> anyhow::Result<()> { + let session = VortexSession::default(); + session.dtypes().register(AnyStorage); + let batch = point_batch(&session)?; + session + .arrow() + .register_exporter(Arc::new(DecliningPointExporter)); + + // The plugin leaves the inferred field as it is, so the field alone does not show the plugin. + let schema = batch.schema(); + let point = DType::Extension(point_dtype(true)?); + assert_eq!( + &session.arrow().to_arrow_field("point", &point)?, + schema.field(0) + ); + + let source = Arc::new(df_expr::Column::new(column, schema.index_of(column)?)); + let expr = get_field_expr(source, path, &schema)?; + let converter = DefaultExpressionConverter::new(session); + assert!(converter.try_convert(&expr, &schema)?.is_none()); + Ok(()) +} + #[rstest] fn test_nested_functions_reject_unknown_children( #[values(false, true)] list: bool, diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs index 16acd7abf9f..54790b6eb92 100644 --- a/vortex-datafusion/src/lib.rs +++ b/vortex-datafusion/src/lib.rs @@ -165,10 +165,18 @@ mod common_tests { use object_store::memory::InMemory; use url::Url; use vortex::VortexSessionDefault; + use vortex::dtype::DType; + use vortex::dtype::extension::ExtDType; + use vortex::dtype::extension::ExtDTypeRef; + use vortex::dtype::extension::ExtId; + use vortex::dtype::extension::ExtVTable; + use vortex::error::VortexResult; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; use vortex::io::object_store::ObjectStoreWrite; + use vortex::scalar::ScalarValue; use vortex::session::VortexSession; + use vortex::session::registry::CachedId; use vortex_arrow::ArrowSessionExt; use crate::VortexFormatFactory; @@ -176,6 +184,60 @@ mod common_tests { static VX_SESSION: LazyLock = LazyLock::new(VortexSession::default); + /// The Vortex ID of [`AnyStorage`], which is also its Arrow extension name. + pub static ANY_STORAGE_ID: CachedId = CachedId::new("test.any_storage"); + + /// An extension type over any storage dtype, without Arrow plugins. + /// + /// Its metadata is its storage dtype, which `metadata_from_storage` reads back from the + /// storage, so the serialized metadata is empty. + #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] + pub struct AnyStorage; + + impl AnyStorage { + /// An [`AnyStorage`] dtype over `storage`. + pub fn dtype(storage: DType) -> VortexResult { + Ok(ExtDType::try_from_storage_with_vtable(Self, (), storage)?.erased()) + } + } + + impl ExtVTable for AnyStorage { + type Metadata = DType; + type StoredMetadata = (); + type NativeValue<'a> = &'a ScalarValue; + + fn id(&self) -> ExtId { + *ANY_STORAGE_ID + } + + fn serialize_metadata(&self, _metadata: &DType) -> VortexResult> { + Ok(Vec::new()) + } + + fn deserialize_metadata(&self, _metadata: &[u8]) -> VortexResult<()> { + Ok(()) + } + + fn metadata_from_storage( + &self, + _stored_metadata: (), + storage_dtype: &DType, + ) -> VortexResult { + Ok(storage_dtype.clone()) + } + + fn storage_dtype(metadata: &DType) -> DType { + metadata.clone() + } + + fn unpack_native<'a>( + _ext_dtype: &'a ExtDType, + storage_value: &'a ScalarValue, + ) -> VortexResult<&'a ScalarValue> { + Ok(storage_value) + } + } + pub struct TestSessionContext { pub store: Arc, pub session: SessionContext, diff --git a/vortex-datafusion/src/v2/tests.rs b/vortex-datafusion/src/v2/tests.rs index 8ad23de2b4a..d2b1c5fd84a 100644 --- a/vortex-datafusion/src/v2/tests.rs +++ b/vortex-datafusion/src/v2/tests.rs @@ -7,10 +7,12 @@ use anyhow::anyhow; use arrow_array::ArrayRef; use arrow_array::Int32Array; use arrow_array::RecordBatch; +use arrow_array::StructArray; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; use arrow_schema::DataType; use arrow_schema::Field; +use arrow_schema::Fields; use arrow_schema::Schema; use arrow_schema::extension::Json as ArrowJson; use arrow_schema::extension::Uuid as ArrowUuid; @@ -25,7 +27,9 @@ use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_datasource::source::DataSource; use datafusion_datasource::source::DataSourceExec; use datafusion_expr::Operator; +use datafusion_functions::core::get_field; use datafusion_physical_expr::PhysicalExpr; +use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; use datafusion_physical_expr::projection::ProjectionExpr; use datafusion_physical_expr::projection::ProjectionExprs; @@ -37,13 +41,17 @@ use futures::TryStreamExt; use rstest::rstest; use vortex::VortexSessionDefault; use vortex::buffer::ByteBufferMut; +use vortex::dtype::Nullability; +use vortex::dtype::session::DTypeSessionExt; use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; use vortex::scan::DataSourceRef; use vortex::session::VortexSession; use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ext_metadata::encode_arrow_ext_metadata; use super::VortexDataSource; +use crate::common_tests::AnyStorage; async fn vortex_source() -> anyhow::Result { let batch = @@ -316,6 +324,14 @@ enum ExtensionFilterType { Json, } +/// The struct that holds the value column in the nested extension tests. +#[derive(Clone, Copy, Debug)] +enum Parent { + Plain, + /// An extension type without plugins, so that its export is its storage. + Extension, +} + /// Input values and Arrow metadata shared by the static and dynamic extension filter tests. struct ExtensionFilterFixture { session: VortexSession, @@ -406,6 +422,56 @@ impl ExtensionFilterFixture { async fn source(&self) -> anyhow::Result { vortex_source_from_batch_with_session(self.batch.clone(), &self.session).await } + + /// Moves the value column into a `parent` struct, beside a `key` field that copies `id`. + /// + /// [`Parent::Extension`] also registers [`AnyStorage`] in the fixture session, so the scan + /// imports `parent` as an extension value. + fn nested_batch(&self, parent: Parent) -> anyhow::Result { + let schema = self.batch.schema(); + let fields = Fields::from(vec![ + Field::new("key", DataType::Int32, false), + schema.field(1).clone(), + ]); + let mut parent_field = Field::new_struct("parent", fields.clone(), false); + if matches!(parent, Parent::Extension) { + let storage = self + .session + .arrow() + .from_arrow_datatype(parent_field.data_type(), Nullability::NonNullable)?; + let parent_dtype = AnyStorage::dtype(storage)?; + self.session.dtypes().register(AnyStorage); + parent_field + .metadata_mut() + .extend(encode_arrow_ext_metadata(&parent_dtype)?); + } + + let parents = StructArray::try_new( + fields, + vec![ + Arc::clone(self.batch.column(0)), + Arc::clone(self.batch.column(1)), + ], + None, + )?; + Ok(RecordBatch::try_new( + Arc::new(Schema::new(vec![schema.field(0).clone(), parent_field])), + vec![Arc::clone(self.batch.column(0)), Arc::new(parents)], + )?) + } +} + +/// Reads `parent.` from a batch that [`ExtensionFilterFixture::nested_batch`] built. +fn parent_field_expr(schema: &Schema, name: &str) -> anyhow::Result> { + Ok(Arc::new(ScalarFunctionExpr::try_new( + get_field(), + vec![ + Arc::new(df_expr::Column::new("parent", 1)), + df_expr::lit(name), + ], + schema, + Arc::new(ConfigOptions::new()), + )?)) } /// Collects row identifiers so the tests compare the scan restriction separately from its residual. @@ -579,3 +645,76 @@ async fn test_build_rejects_extension_column_over_plain() -> anyhow::Result<()> Ok(()) } + +/// A comparison on an extension field of a plain struct pushes down like one on a column. Through +/// extension storage, the plugin that exports the field keeps the comparison in DataFusion. +#[rstest] +#[case::plain(Parent::Plain, true)] +#[case::extension(Parent::Extension, false)] +#[tokio::test] +async fn test_nested_extension_comparison_pushdown( + #[case] parent: Parent, + #[case] pushed_down: bool, + #[values(ExtensionFilterType::Uuid, ExtensionFilterType::Json)] + extension_type: ExtensionFilterType, +) -> anyhow::Result<()> { + let fixture = ExtensionFilterFixture::new(extension_type)?; + let batch = fixture.nested_batch(parent)?; + let predicate: Arc = Arc::new(df_expr::BinaryExpr::new( + parent_field_expr(&batch.schema(), "value")?, + Operator::Eq, + fixture.literal(1), + )); + let source = vortex_source_from_batch_with_session(batch, &fixture.session).await?; + + let pushed = + source.try_pushdown_filters(vec![Arc::clone(&predicate)], &ConfigOptions::new())?; + assert_eq!( + matches!(pushed.filters.as_slice(), [PushedDown::Yes]), + pushed_down + ); + + let scan = Arc::new(DataSourceExec::new( + pushed.updated_node.unwrap_or_else(|| Arc::new(source)), + )); + let residual = Arc::new(FilterExec::try_new(predicate, scan)?); + assert_eq!(scan_ids(residual).await?, [2]); + + Ok(()) +} + +/// A plain field of an extension struct reads through its storage in the scan, both in a filter +/// and in a projection. +#[tokio::test] +async fn test_get_field_through_extension_storage_is_pushed_down() -> anyhow::Result<()> { + let fixture = ExtensionFilterFixture::new(ExtensionFilterType::Uuid)?; + let batch = fixture.nested_batch(Parent::Extension)?; + let key = parent_field_expr(&batch.schema(), "key")?; + let source = vortex_source_from_batch_with_session(batch, &fixture.session).await?; + + let filter: Arc = Arc::new(df_expr::BinaryExpr::new( + Arc::clone(&key), + Operator::Eq, + df_expr::lit(2i32), + )); + let pushed = source.try_pushdown_filters(vec![filter], &ConfigOptions::new())?; + assert!(matches!(pushed.filters.as_slice(), [PushedDown::Yes])); + let filtered = pushed + .updated_node + .ok_or_else(|| anyhow!("filter was not pushed"))?; + assert_eq!( + scan_ids(Arc::new(DataSourceExec::new(filtered))).await?, + [2] + ); + + let projection = ProjectionExprs::from(vec![ProjectionExpr::new(key, "key")]); + let projected = source + .try_swapping_with_projection(&projection)? + .ok_or_else(|| anyhow!("projection was not swapped"))?; + assert_eq!( + scan_ids(Arc::new(DataSourceExec::new(projected))).await?, + [1, 2, 3, 4] + ); + + Ok(()) +}