diff --git a/vortex-arrow/src/executor/mod.rs b/vortex-arrow/src/executor/mod.rs index a5e88a498a9..488c46209ed 100644 --- a/vortex-arrow/src/executor/mod.rs +++ b/vortex-arrow/src/executor/mod.rs @@ -69,9 +69,7 @@ use crate::executor::struct_::to_arrow_struct; use crate::executor::temporal::to_arrow_date; use crate::executor::temporal::to_arrow_time; use crate::executor::temporal::to_arrow_timestamp; -use crate::ext_metadata::decode_arrow_ext_metadata; use crate::session::ArrowSessionExt; -use crate::session::ensure_ext_metadata; /// Execution of a Vortex array into an Arrow array, for every type that implements [`IntoArray`]. /// @@ -326,20 +324,9 @@ fn execute_arrow_ext_storage( options: &ArrowExportOptions, ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(importer_dtype) = ctx - .session() + ctx.session() .arrow() - .infer_field_dtype_with_importers(target)? - { - vortex_ensure_eq!( - importer_dtype.as_nullable(), - array.dtype().as_nullable(), - "Arrow field {:?} must describe the source extension type", - target.name() - ); - } else { - ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?; - } + .ensure_ext_storage_metadata(ext_dtype, target)?; let target_dtype = ctx .session() diff --git a/vortex-arrow/src/session.rs b/vortex-arrow/src/session.rs index d0bc576d14d..ca45f4e76ce 100644 --- a/vortex-arrow/src/session.rs +++ b/vortex-arrow/src/session.rs @@ -559,6 +559,96 @@ impl ArrowSession { Ok(field) } + /// Returns whether an export of `ext_dtype` to `field` produces exactly its storage. + /// + /// A caller can use the answer to plan reads into the storage without the array. The export + /// produces the storage only when `field` names the Vortex ID of `ext_dtype` and passes the + /// checks that the [`ArrowSession`] docs describe for such a field. A registered exporter can + /// produce any data for the same field, so a name with an exporter gives `false`. + /// + /// The answer covers only the top level of `field`. Each storage child exports to its own + /// field, which can name another exporter, and [`Self::exports_without_plugins`] checks every + /// depth. + #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")] + pub fn exports_extension_storage(&self, ext_dtype: &ExtDTypeRef, field: &Field) -> bool { + // A dictionary or run-end encoded field holds the storage values behind an encoding, so + // its Arrow data is not the storage, even though it can import as the storage dtype. + if matches!( + field.data_type(), + DataType::Dictionary(..) | DataType::RunEndEncoded(..) + ) { + return false; + } + + let Some(name) = field.extension_type_name() else { + return false; + }; + let id = Id::new(name); + + // The export tries the registered exporters before the storage route. + if !self.exporters(&id).is_empty() { + return false; + } + + // The storage route needs a name that the session resolves to the Vortex ID of + // `ext_dtype`. The name can be an alias of that ID. + if self.known_vortex_id(&id) != Some(ext_dtype.id()) { + return false; + } + + // The export fails unless the field describes `ext_dtype`. + if self.ensure_ext_storage_metadata(ext_dtype, field).is_err() { + return false; + } + + // The export also accepts some Arrow types that import as another dtype, such as an Arrow + // timestamp, which imports as the extension type itself. Only an exact match means that + // the Arrow data has the type of the storage. + self.from_arrow_datatype(field.data_type(), ext_dtype.nullability()) + .is_ok_and(|dtype| &dtype == ext_dtype.storage_dtype()) + } + + /// Returns whether an export to `field` calls no registered exporter at any depth. + /// + /// Such an export uses only the conversions of this crate, so a caller planning reads without + /// the array knows which Arrow data it produces. A field that names an extension type without + /// an exporter still qualifies, because it exports through its Vortex ID or as plain storage. + #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")] + pub fn exports_without_plugins(&self, field: &Field) -> bool { + let names_exporter = field + .extension_type_name() + .is_some_and(|name| !self.exporters(&Id::new(name)).is_empty()); + + !names_exporter + && nested_fields(field.data_type()) + .into_iter() + .all(|child| self.exports_without_plugins(child)) + } + + /// Fails unless `field`, which names the Vortex ID of `ext_dtype`, describes `ext_dtype`. + /// + /// An importer that recognizes `field` reads its metadata in the plugin's format, so the check + /// asks the importer first, as [`Self::infer_field_dtype_with_importers`] explains. Without + /// one, the field must carry the metadata of `ext_dtype`. + pub(crate) fn ensure_ext_storage_metadata( + &self, + ext_dtype: &ExtDTypeRef, + field: &Field, + ) -> VortexResult<()> { + let Some(importer_dtype) = self.infer_field_dtype_with_importers(field)? else { + return ensure_ext_metadata(field, &decode_arrow_ext_metadata(field)?, ext_dtype); + }; + + vortex_ensure_eq!( + importer_dtype.as_nullable(), + DType::Extension(ext_dtype.with_nullability(Nullability::Nullable)), + "Arrow field {:?} must describe the source extension type", + field.name() + ); + + Ok(()) + } + /// Returns the Vortex ID that the extension name `id` resolves to, or `None` if the session does /// not know it. The registry can hold an alias, so the result can differ from `id`. pub(crate) fn known_vortex_id(&self, id: &Id) -> Option { @@ -1116,6 +1206,26 @@ pub(crate) fn ensure_ext_metadata( Ok(()) } +/// Returns the fields nested directly in `data_type`. +/// +/// A dictionary has no field for its values, so it returns the fields nested in the value type. +fn nested_fields(data_type: &DataType) -> Vec<&Field> { + match data_type { + DataType::Struct(fields) => fields.iter().map(Arc::as_ref).collect(), + DataType::Union(fields, _) => fields.iter().map(|(_, field)| field.as_ref()).collect(), + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) + | DataType::RunEndEncoded(_, field) => vec![field.as_ref()], + DataType::Dictionary(_, values) => nested_fields(values), + // Other Arrow types have no nested fields. + _ => Vec::new(), + } +} + /// Extension trait for accessing the [`ArrowSession`] on a Vortex session. pub trait ArrowSessionExt: SessionExt { /// Get the Arrow session, creating it on first use. @@ -1795,4 +1905,80 @@ mod tests { ); Ok(()) } + + const EXPORTER_NAME: &str = "test.exporter"; + static EXPORTER_ID: CachedId = CachedId::new(EXPORTER_NAME); + + /// Declines every array, because the tests only need an exporter registered under its name. + #[derive(Debug)] + struct DecliningExporter; + + impl ArrowExportVTable for DecliningExporter { + fn arrow_ext_id(&self) -> Id { + *EXPORTER_ID + } + + fn vortex_id(&self) -> Id { + *EXPORTER_ID + } + + fn to_arrow_field( + &self, + _name: &str, + _dtype: &DType, + _session: &ArrowSession, + ) -> VortexResult> { + Ok(None) + } + + fn execute_arrow( + &self, + array: ArrayRef, + _target: &Field, + _options: &ArrowExportOptions, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ArrowExport::Unsupported(array)) + } + } + + /// An `Int32` field that names `extension_name`, or a plain one without it. + fn leaf(extension_name: Option<&str>) -> FieldRef { + let mut leaf = Field::new("leaf", DataType::Int32, true); + if let Some(name) = extension_name { + leaf = leaf + .with_metadata([(EXTENSION_TYPE_NAME_KEY.to_string(), name.to_string())].into()); + } + + Arc::new(leaf) + } + + /// Only a leaf that names a registered exporter makes an export call a plugin, at any depth. + #[rstest] + #[case::top_level( |leaf: FieldRef| leaf.as_ref().clone())] + #[case::struct_child( |leaf: FieldRef| Field::new_struct("a", vec![leaf], true))] + #[case::list_item( |leaf: FieldRef| Field::new_list("a", leaf, true))] + #[case::run_end_values(|leaf: FieldRef| { + let run_ends = Arc::new(Field::new("run_ends", DataType::Int32, false)); + Field::new("a", DataType::RunEndEncoded(run_ends, leaf), true) + })] + #[case::dictionary_values(|leaf: FieldRef| { + Field::new_dictionary("a", DataType::Int32, DataType::Struct(vec![leaf].into()), true) + })] + fn test_exports_without_plugins_finds_exporters_at_any_depth( + #[case] wrap: fn(FieldRef) -> Field, + #[values(None, Some("test.unregistered"), Some(EXPORTER_NAME))] leaf_name: Option<&str>, + ) { + let vortex_session = VortexSession::empty(); + let session = vortex_session.arrow(); + session.register_exporter(Arc::new(DecliningExporter)); + + let field = wrap(leaf(leaf_name)); + let leaf_names_exporter = leaf_name == Some(EXPORTER_NAME); + + assert_eq!( + session.exports_without_plugins(&field), + !leaf_names_exporter + ); + } } diff --git a/vortex-datafusion/src/convert/exprs/mod.rs b/vortex-datafusion/src/convert/exprs/mod.rs index 2b0bb323993..6323b2c3c07 100644 --- a/vortex-datafusion/src/convert/exprs/mod.rs +++ b/vortex-datafusion/src/convert/exprs/mod.rs @@ -51,6 +51,7 @@ use vortex::expr::Expression; use vortex::expr::Lambda; use vortex::expr::byte_length; use vortex::expr::cast; +use vortex::expr::ext_storage; use vortex::expr::get_item; use vortex::expr::in_list_of; use vortex::expr::is_not_null; @@ -266,6 +267,8 @@ type Conversion = Result; /// Nested struct field access also uses Vortex semantics: a null parent produces a null child, /// whereas DataFusion extracts the child without applying parent validity. These differences /// are accepted for both filters and projections. +/// Field access through an extension can run natively when Arrow exposes its Vortex storage and no +/// plugin exports the selected field. /// Searched `CASE WHEN` expressions preserve short-circuit evaluation: conditions and /// branch values only execute on the rows that reach them, including fallible branches. /// Other expressions require compatible SQL semantics or remain in DataFusion. @@ -746,40 +749,7 @@ impl DefaultExpressionConverter { } if ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() { - // DataFusion's GetFieldFunc flattens nested field access into a single call - // with multiple field name arguments, e.g. `outer.inner.leaf` becomes - // get_field(Column("outer"), "inner", "leaf"). - let [source, paths @ ..] = scalar_fn.args() else { - return Err( - exec_datafusion_err!("get_field requires a source and field path").into(), - ); - }; - if paths.is_empty() { - return Err(exec_datafusion_err!("get_field requires a field path").into()); - } - let mut source_type = source.data_type(schema)?; - let mut names = Vec::with_capacity(paths.len()); - for path in paths { - let name = path - .downcast_ref::() - .and_then(|literal| literal.value().try_as_str().flatten()) - .ok_or_else(|| { - exec_datafusion_err!("get_field path must be a non-null string literal") - })?; - let DataType::Struct(fields) = &source_type else { - return Err(Unconverted::Unsupported); - }; - let (_, field) = fields.find(name).ok_or_else(|| { - exec_datafusion_err!("get_field references missing field {name}") - })?; - source_type = field.data_type().clone(); - names.push(name); - } - let mut result = self.convert_expr(source, schema, input_dtype)?; - for name in names { - result = get_item(name, result); - } - return Ok(result); + return self.convert_get_field(scalar_fn, schema, input_dtype); } if ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() { @@ -823,6 +793,87 @@ impl DefaultExpressionConverter { Ok(cast(length(input), return_dtype)) } + /// Converts `get_field` to a chain of `get_item` reads. + /// + /// DataFusion can flatten nested access into one call, so `outer.inner.leaf` can arrive as + /// `get_field(outer, "inner", "leaf")`. + /// + /// Vortex extension types are opaque, so `get_item` cannot read into an extension value until + /// `ext_storage` exposes its storage. The converter can add that `ext_storage` without breaking + /// the opacity, because DataFusion already reads the storage: an Arrow extension type is only + /// metadata on its storage field, and `get_field` reads that field like any other struct. Both + /// sides read the same values only when Vortex exports the parent as exactly its storage, which + /// [`exports_extension_storage`] checks. + /// + /// [`exports_extension_storage`]: vortex_arrow::ArrowSession::exports_extension_storage + fn convert_get_field( + &self, + scalar_fn: &ScalarFunctionExpr, + schema: &Schema, + input_dtype: &DType, + ) -> Conversion { + let [source, paths @ ..] = scalar_fn.args() else { + return Err(exec_datafusion_err!("get_field requires a source and field path").into()); + }; + + if paths.is_empty() { + return Err(exec_datafusion_err!("get_field requires a field path").into()); + } + + // Keep the Arrow field of each step, because its metadata decides how Vortex exports it. + let source_field = source.return_field(schema)?; + let mut field = source_field.as_ref(); + let mut steps = Vec::with_capacity(paths.len()); + for path in paths { + let name = path + .downcast_ref::() + .and_then(|literal| literal.value().try_as_str().flatten()) + .ok_or_else(|| { + exec_datafusion_err!("get_field path must be a non-null string literal") + })?; + let DataType::Struct(fields) = field.data_type() else { + return Err(Unconverted::Unsupported); + }; + let (_, child_field) = fields + .find(name) + .ok_or_else(|| exec_datafusion_err!("get_field references missing field {name}"))?; + steps.push((field, name)); + field = child_field.as_ref(); + } + + let mut read = self.convert_expr(source, schema, input_dtype)?; + let mut reads_extension_storage = false; + for (parent_field, name) in steps { + // Check the Vortex dtype instead of the Arrow extension name, because the session + // imports an unknown extension name as a plain struct that `get_item` reads directly. + if let DType::Extension(ext_dtype) = converted_dtype(&read, input_dtype)? { + if !self + .session + .arrow() + .exports_extension_storage(&ext_dtype, parent_field) + { + return Err(Unconverted::Unsupported); + } + + read = ext_storage(read); + reads_extension_storage = true; + } + read = get_item(name, read); + } + + // A plugin exporter decides which Arrow data DataFusion sees for the selected field, so + // Vortex and DataFusion agree on that field only if the export keeps values. Columns and + // plain struct fields already assume this, and the check skips them so that their pushdown + // stays the same. + // TODO(connor): Drop this check once the Arrow plugin contract requires exports to keep + // values. + if reads_extension_storage && !self.session.arrow().exports_without_plugins(field) { + return Err(Unconverted::Unsupported); + } + + Ok(read) + } + /// Converts `array_sum` to `list_sum`. /// /// DataFusion casts the list to `List`, then adds each list's valid elements in diff --git a/vortex-datafusion/src/convert/exprs/tests.rs b/vortex-datafusion/src/convert/exprs/tests.rs index b0ae212c934..4e438af03be 100644 --- a/vortex-datafusion/src/convert/exprs/tests.rs +++ b/vortex-datafusion/src/convert/exprs/tests.rs @@ -54,12 +54,16 @@ use datafusion_physical_expr::projection::ProjectionExpr; use datafusion_physical_plan::expressions as df_expr; use insta::assert_snapshot; use rstest::rstest; +use vortex::array::ArrayRef; use vortex::array::Canonical; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::builtins::ArrayBuiltins; use vortex::dtype::PType; +use vortex::dtype::extension::ExtDTypeRef; +use vortex::dtype::session::DTypeSessionExt; use vortex::encodings::uuid::Uuid; use vortex::error::VortexResult; use vortex::error::vortex_bail; @@ -74,11 +78,16 @@ use vortex::scalar_fn::fns::literal::Literal; use vortex::session::registry::CachedId; use vortex::session::registry::Id; use vortex_array::assert_arrays_eq; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportOptions; +use vortex_arrow::ArrowExportVTable; use vortex_arrow::ArrowImport; use vortex_arrow::ArrowImportVTable; use vortex_arrow::ArrowSession; use super::*; +use crate::common_tests::ANY_STORAGE_ID; +use crate::common_tests::AnyStorage; use crate::common_tests::TestSessionContext; use crate::convert::TryToDataFusion; @@ -2596,6 +2605,166 @@ fn test_native_nested_get_field( Ok(()) } +/// An [`AnyStorage`] point over `Struct`. +fn point_dtype(nullable: bool) -> VortexResult { + let coordinate = DType::Primitive(PType::I32, Nullability::NonNullable); + let storage = DType::struct_( + [("x", coordinate.clone()), ("y", coordinate)], + nullable.into(), + ); + + AnyStorage::dtype(storage) +} + +/// A batch with a `point` column and an `outer` struct around the same points, with the point +/// field that `session` infers. +/// +/// `x` holds 10, 20, 30, and 40. `point` is null at index 1, and `outer` is null at index 2. +fn point_batch(session: &VortexSession) -> anyhow::Result { + let point = session + .arrow() + .to_arrow_field("point", &DType::Extension(point_dtype(true)?))?; + let DataType::Struct(fields) = point.data_type() else { + anyhow::bail!("Expected point storage as a struct, got {point}"); + }; + let points: ArrowArrayRef = Arc::new(StructArray::try_new( + fields.clone(), + vec![ + Arc::new(Int32Array::from(vec![10, 20, 30, 40])), + Arc::new(Int32Array::from(vec![11, 21, 31, 41])), + ], + Some(NullBuffer::from(vec![true, false, true, true])), + )?); + let outer = StructArray::try_new( + vec![point.clone()].into(), + vec![Arc::clone(&points)], + Some(NullBuffer::from(vec![true, true, false, true])), + )?; + let schema = Schema::new(vec![ + point, + Field::new("outer", outer.data_type().clone(), true), + ]); + Ok(RecordBatch::try_new( + Arc::new(schema), + vec![points, Arc::new(outer)], + )?) +} + +/// A registered extension reads its storage through `ext_storage`. The session imports an unknown +/// extension name as plain storage, so the same read is a plain `get_item` with the same values. +#[rstest] +#[case::column( "point", &["x"], false, [Some(10), None, Some(30), Some(40)])] +#[case::flattened("outer", &["point", "x"], false, [Some(10), None, None, Some(40)])] +#[case::chained( "outer", &["point", "x"], true, [Some(10), None, None, Some(40)])] +fn test_get_field_reads_extension_storage( + #[case] column: &str, + #[case] path: &[&str], + #[case] chained: bool, + #[case] expected: [Option; 4], + #[values(false, true)] registered: bool, +) -> anyhow::Result<()> { + let session = VortexSession::default(); + if registered { + session.dtypes().register(AnyStorage); + } + let batch = point_batch(&session)?; + let schema = batch.schema(); + let source = Arc::new(df_expr::Column::new(column, schema.index_of(column)?)); + let expr = if chained { + let (first, rest) = path.split_at(1); + get_field_expr(get_field_expr(source, first, &schema)?, rest, &schema)? + } else { + get_field_expr(source, path, &schema)? + }; + + let converted = DefaultExpressionConverter::new(session.clone()) + .try_convert(&expr, &schema)? + .ok_or_else(|| anyhow::anyhow!("Expected native conversion for {expr}"))?; + let point = if column == "point" { + get_item("point", root()) + } else { + get_item("point", get_item("outer", root())) + }; + let point = if registered { + ext_storage(point) + } else { + point + }; + assert_eq!(converted, get_item("x", point)); + + let input = session.arrow().from_arrow_record_batch(batch, &schema)?; + assert_arrays_eq!( + input.apply(&converted)?, + PrimitiveArray::from_option_iter(expected), + &mut session.create_execution_ctx() + ); + Ok(()) +} + +/// Declines every array, so each export falls through to the Vortex-ID storage export. +/// +/// The converter cannot know that before the export runs, so it treats the plugin as opaque. +#[derive(Debug)] +struct DecliningPointExporter; + +impl ArrowExportVTable for DecliningPointExporter { + fn arrow_ext_id(&self) -> Id { + *ANY_STORAGE_ID + } + + fn vortex_id(&self) -> Id { + *ANY_STORAGE_ID + } + + fn to_arrow_field( + &self, + _name: &str, + _dtype: &DType, + _session: &ArrowSession, + ) -> VortexResult> { + Ok(None) + } + + fn execute_arrow( + &self, + array: ArrayRef, + _target: &Field, + _options: &ArrowExportOptions, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ArrowExport::Unsupported(array)) + } +} + +#[rstest] +#[case::column("point", &["x"])] +#[case::nested("outer", &["point", "x"])] +fn test_get_field_on_plugin_export_stays_in_datafusion( + #[case] column: &str, + #[case] path: &[&str], +) -> anyhow::Result<()> { + let session = VortexSession::default(); + session.dtypes().register(AnyStorage); + let batch = point_batch(&session)?; + session + .arrow() + .register_exporter(Arc::new(DecliningPointExporter)); + + // The plugin leaves the inferred field as it is, so the field alone does not show the plugin. + let schema = batch.schema(); + let point = DType::Extension(point_dtype(true)?); + assert_eq!( + &session.arrow().to_arrow_field("point", &point)?, + schema.field(0) + ); + + let source = Arc::new(df_expr::Column::new(column, schema.index_of(column)?)); + let expr = get_field_expr(source, path, &schema)?; + let converter = DefaultExpressionConverter::new(session); + assert!(converter.try_convert(&expr, &schema)?.is_none()); + Ok(()) +} + #[rstest] fn test_nested_functions_reject_unknown_children( #[values(false, true)] list: bool, diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs index 16acd7abf9f..54790b6eb92 100644 --- a/vortex-datafusion/src/lib.rs +++ b/vortex-datafusion/src/lib.rs @@ -165,10 +165,18 @@ mod common_tests { use object_store::memory::InMemory; use url::Url; use vortex::VortexSessionDefault; + use vortex::dtype::DType; + use vortex::dtype::extension::ExtDType; + use vortex::dtype::extension::ExtDTypeRef; + use vortex::dtype::extension::ExtId; + use vortex::dtype::extension::ExtVTable; + use vortex::error::VortexResult; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; use vortex::io::object_store::ObjectStoreWrite; + use vortex::scalar::ScalarValue; use vortex::session::VortexSession; + use vortex::session::registry::CachedId; use vortex_arrow::ArrowSessionExt; use crate::VortexFormatFactory; @@ -176,6 +184,60 @@ mod common_tests { static VX_SESSION: LazyLock = LazyLock::new(VortexSession::default); + /// The Vortex ID of [`AnyStorage`], which is also its Arrow extension name. + pub static ANY_STORAGE_ID: CachedId = CachedId::new("test.any_storage"); + + /// An extension type over any storage dtype, without Arrow plugins. + /// + /// Its metadata is its storage dtype, which `metadata_from_storage` reads back from the + /// storage, so the serialized metadata is empty. + #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] + pub struct AnyStorage; + + impl AnyStorage { + /// An [`AnyStorage`] dtype over `storage`. + pub fn dtype(storage: DType) -> VortexResult { + Ok(ExtDType::try_from_storage_with_vtable(Self, (), storage)?.erased()) + } + } + + impl ExtVTable for AnyStorage { + type Metadata = DType; + type StoredMetadata = (); + type NativeValue<'a> = &'a ScalarValue; + + fn id(&self) -> ExtId { + *ANY_STORAGE_ID + } + + fn serialize_metadata(&self, _metadata: &DType) -> VortexResult> { + Ok(Vec::new()) + } + + fn deserialize_metadata(&self, _metadata: &[u8]) -> VortexResult<()> { + Ok(()) + } + + fn metadata_from_storage( + &self, + _stored_metadata: (), + storage_dtype: &DType, + ) -> VortexResult { + Ok(storage_dtype.clone()) + } + + fn storage_dtype(metadata: &DType) -> DType { + metadata.clone() + } + + fn unpack_native<'a>( + _ext_dtype: &'a ExtDType, + storage_value: &'a ScalarValue, + ) -> VortexResult<&'a ScalarValue> { + Ok(storage_value) + } + } + pub struct TestSessionContext { pub store: Arc, pub session: SessionContext, diff --git a/vortex-datafusion/src/v2/tests.rs b/vortex-datafusion/src/v2/tests.rs index 8ad23de2b4a..d2b1c5fd84a 100644 --- a/vortex-datafusion/src/v2/tests.rs +++ b/vortex-datafusion/src/v2/tests.rs @@ -7,10 +7,12 @@ use anyhow::anyhow; use arrow_array::ArrayRef; use arrow_array::Int32Array; use arrow_array::RecordBatch; +use arrow_array::StructArray; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; use arrow_schema::DataType; use arrow_schema::Field; +use arrow_schema::Fields; use arrow_schema::Schema; use arrow_schema::extension::Json as ArrowJson; use arrow_schema::extension::Uuid as ArrowUuid; @@ -25,7 +27,9 @@ use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_datasource::source::DataSource; use datafusion_datasource::source::DataSourceExec; use datafusion_expr::Operator; +use datafusion_functions::core::get_field; use datafusion_physical_expr::PhysicalExpr; +use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; use datafusion_physical_expr::projection::ProjectionExpr; use datafusion_physical_expr::projection::ProjectionExprs; @@ -37,13 +41,17 @@ use futures::TryStreamExt; use rstest::rstest; use vortex::VortexSessionDefault; use vortex::buffer::ByteBufferMut; +use vortex::dtype::Nullability; +use vortex::dtype::session::DTypeSessionExt; use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; use vortex::scan::DataSourceRef; use vortex::session::VortexSession; use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ext_metadata::encode_arrow_ext_metadata; use super::VortexDataSource; +use crate::common_tests::AnyStorage; async fn vortex_source() -> anyhow::Result { let batch = @@ -316,6 +324,14 @@ enum ExtensionFilterType { Json, } +/// The struct that holds the value column in the nested extension tests. +#[derive(Clone, Copy, Debug)] +enum Parent { + Plain, + /// An extension type without plugins, so that its export is its storage. + Extension, +} + /// Input values and Arrow metadata shared by the static and dynamic extension filter tests. struct ExtensionFilterFixture { session: VortexSession, @@ -406,6 +422,56 @@ impl ExtensionFilterFixture { async fn source(&self) -> anyhow::Result { vortex_source_from_batch_with_session(self.batch.clone(), &self.session).await } + + /// Moves the value column into a `parent` struct, beside a `key` field that copies `id`. + /// + /// [`Parent::Extension`] also registers [`AnyStorage`] in the fixture session, so the scan + /// imports `parent` as an extension value. + fn nested_batch(&self, parent: Parent) -> anyhow::Result { + let schema = self.batch.schema(); + let fields = Fields::from(vec![ + Field::new("key", DataType::Int32, false), + schema.field(1).clone(), + ]); + let mut parent_field = Field::new_struct("parent", fields.clone(), false); + if matches!(parent, Parent::Extension) { + let storage = self + .session + .arrow() + .from_arrow_datatype(parent_field.data_type(), Nullability::NonNullable)?; + let parent_dtype = AnyStorage::dtype(storage)?; + self.session.dtypes().register(AnyStorage); + parent_field + .metadata_mut() + .extend(encode_arrow_ext_metadata(&parent_dtype)?); + } + + let parents = StructArray::try_new( + fields, + vec![ + Arc::clone(self.batch.column(0)), + Arc::clone(self.batch.column(1)), + ], + None, + )?; + Ok(RecordBatch::try_new( + Arc::new(Schema::new(vec![schema.field(0).clone(), parent_field])), + vec![Arc::clone(self.batch.column(0)), Arc::new(parents)], + )?) + } +} + +/// Reads `parent.` from a batch that [`ExtensionFilterFixture::nested_batch`] built. +fn parent_field_expr(schema: &Schema, name: &str) -> anyhow::Result> { + Ok(Arc::new(ScalarFunctionExpr::try_new( + get_field(), + vec![ + Arc::new(df_expr::Column::new("parent", 1)), + df_expr::lit(name), + ], + schema, + Arc::new(ConfigOptions::new()), + )?)) } /// Collects row identifiers so the tests compare the scan restriction separately from its residual. @@ -579,3 +645,76 @@ async fn test_build_rejects_extension_column_over_plain() -> anyhow::Result<()> Ok(()) } + +/// A comparison on an extension field of a plain struct pushes down like one on a column. Through +/// extension storage, the plugin that exports the field keeps the comparison in DataFusion. +#[rstest] +#[case::plain(Parent::Plain, true)] +#[case::extension(Parent::Extension, false)] +#[tokio::test] +async fn test_nested_extension_comparison_pushdown( + #[case] parent: Parent, + #[case] pushed_down: bool, + #[values(ExtensionFilterType::Uuid, ExtensionFilterType::Json)] + extension_type: ExtensionFilterType, +) -> anyhow::Result<()> { + let fixture = ExtensionFilterFixture::new(extension_type)?; + let batch = fixture.nested_batch(parent)?; + let predicate: Arc = Arc::new(df_expr::BinaryExpr::new( + parent_field_expr(&batch.schema(), "value")?, + Operator::Eq, + fixture.literal(1), + )); + let source = vortex_source_from_batch_with_session(batch, &fixture.session).await?; + + let pushed = + source.try_pushdown_filters(vec![Arc::clone(&predicate)], &ConfigOptions::new())?; + assert_eq!( + matches!(pushed.filters.as_slice(), [PushedDown::Yes]), + pushed_down + ); + + let scan = Arc::new(DataSourceExec::new( + pushed.updated_node.unwrap_or_else(|| Arc::new(source)), + )); + let residual = Arc::new(FilterExec::try_new(predicate, scan)?); + assert_eq!(scan_ids(residual).await?, [2]); + + Ok(()) +} + +/// A plain field of an extension struct reads through its storage in the scan, both in a filter +/// and in a projection. +#[tokio::test] +async fn test_get_field_through_extension_storage_is_pushed_down() -> anyhow::Result<()> { + let fixture = ExtensionFilterFixture::new(ExtensionFilterType::Uuid)?; + let batch = fixture.nested_batch(Parent::Extension)?; + let key = parent_field_expr(&batch.schema(), "key")?; + let source = vortex_source_from_batch_with_session(batch, &fixture.session).await?; + + let filter: Arc = Arc::new(df_expr::BinaryExpr::new( + Arc::clone(&key), + Operator::Eq, + df_expr::lit(2i32), + )); + let pushed = source.try_pushdown_filters(vec![filter], &ConfigOptions::new())?; + assert!(matches!(pushed.filters.as_slice(), [PushedDown::Yes])); + let filtered = pushed + .updated_node + .ok_or_else(|| anyhow!("filter was not pushed"))?; + assert_eq!( + scan_ids(Arc::new(DataSourceExec::new(filtered))).await?, + [2] + ); + + let projection = ProjectionExprs::from(vec![ProjectionExpr::new(key, "key")]); + let projected = source + .try_swapping_with_projection(&projection)? + .ok_or_else(|| anyhow!("projection was not swapped"))?; + assert_eq!( + scan_ids(Arc::new(DataSourceExec::new(projected))).await?, + [1, 2, 3, 4] + ); + + Ok(()) +}