Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 2 additions & 15 deletions vortex-arrow/src/executor/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,7 @@ use crate::executor::struct_::to_arrow_struct;
use crate::executor::temporal::to_arrow_date;
use crate::executor::temporal::to_arrow_time;
use crate::executor::temporal::to_arrow_timestamp;
use crate::ext_metadata::decode_arrow_ext_metadata;
use crate::session::ArrowSessionExt;
use crate::session::ensure_ext_metadata;

/// Execution of a Vortex array into an Arrow array, for every type that implements [`IntoArray`].
///
Expand Down Expand Up @@ -326,20 +324,9 @@ fn execute_arrow_ext_storage(
options: &ArrowExportOptions,
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowArrayRef> {
if let Some(importer_dtype) = ctx
.session()
ctx.session()
.arrow()
.infer_field_dtype_with_importers(target)?
{
vortex_ensure_eq!(
importer_dtype.as_nullable(),
array.dtype().as_nullable(),
"Arrow field {:?} must describe the source extension type",
target.name()
);
} else {
ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?;
}
.ensure_ext_storage_metadata(ext_dtype, target)?;

let target_dtype = ctx
.session()
Expand Down
186 changes: 186 additions & 0 deletions vortex-arrow/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,96 @@ impl ArrowSession {
Ok(field)
}

/// Returns whether an export of `ext_dtype` to `field` produces exactly its storage.
///
/// A caller can use the answer to plan reads into the storage without the array. The export
/// produces the storage only when `field` names the Vortex ID of `ext_dtype` and passes the
/// checks that the [`ArrowSession`] docs describe for such a field. A registered exporter can
/// produce any data for the same field, so a name with an exporter gives `false`.
///
/// The answer covers only the top level of `field`. Each storage child exports to its own
/// field, which can name another exporter, and [`Self::exports_without_plugins`] checks every
/// depth.
#[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
pub fn exports_extension_storage(&self, ext_dtype: &ExtDTypeRef, field: &Field) -> bool {
// A dictionary or run-end encoded field holds the storage values behind an encoding, so
// its Arrow data is not the storage, even though it can import as the storage dtype.
if matches!(
field.data_type(),
DataType::Dictionary(..) | DataType::RunEndEncoded(..)
) {
return false;
}

let Some(name) = field.extension_type_name() else {
return false;
};
let id = Id::new(name);

// The export tries the registered exporters before the storage route.
if !self.exporters(&id).is_empty() {
return false;
}

// The storage route needs a name that the session resolves to the Vortex ID of
// `ext_dtype`. The name can be an alias of that ID.
if self.known_vortex_id(&id) != Some(ext_dtype.id()) {
return false;
}

// The export fails unless the field describes `ext_dtype`.
if self.ensure_ext_storage_metadata(ext_dtype, field).is_err() {
return false;
}

// The export also accepts some Arrow types that import as another dtype, such as an Arrow
// timestamp, which imports as the extension type itself. Only an exact match means that
// the Arrow data has the type of the storage.
self.from_arrow_datatype(field.data_type(), ext_dtype.nullability())
.is_ok_and(|dtype| &dtype == ext_dtype.storage_dtype())
}

/// Returns whether an export to `field` calls no registered exporter at any depth.
///
/// Such an export uses only the conversions of this crate, so a caller planning reads without
/// the array knows which Arrow data it produces. A field that names an extension type without
/// an exporter still qualifies, because it exports through its Vortex ID or as plain storage.
#[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
pub fn exports_without_plugins(&self, field: &Field) -> bool {
let names_exporter = field
.extension_type_name()
.is_some_and(|name| !self.exporters(&Id::new(name)).is_empty());

!names_exporter
&& nested_fields(field.data_type())
.into_iter()
.all(|child| self.exports_without_plugins(child))
}

/// Fails unless `field`, which names the Vortex ID of `ext_dtype`, describes `ext_dtype`.
///
/// An importer that recognizes `field` reads its metadata in the plugin's format, so the check
/// asks the importer first, as [`Self::infer_field_dtype_with_importers`] explains. Without
/// one, the field must carry the metadata of `ext_dtype`.
pub(crate) fn ensure_ext_storage_metadata(
&self,
ext_dtype: &ExtDTypeRef,
field: &Field,
) -> VortexResult<()> {
let Some(importer_dtype) = self.infer_field_dtype_with_importers(field)? else {
return ensure_ext_metadata(field, &decode_arrow_ext_metadata(field)?, ext_dtype);
};

vortex_ensure_eq!(
importer_dtype.as_nullable(),
DType::Extension(ext_dtype.with_nullability(Nullability::Nullable)),
"Arrow field {:?} must describe the source extension type",
field.name()
);

Ok(())
}

/// Returns the Vortex ID that the extension name `id` resolves to, or `None` if the session does
/// not know it. The registry can hold an alias, so the result can differ from `id`.
pub(crate) fn known_vortex_id(&self, id: &Id) -> Option<ExtId> {
Expand Down Expand Up @@ -1116,6 +1206,26 @@ pub(crate) fn ensure_ext_metadata(
Ok(())
}

/// Returns the fields nested directly in `data_type`.
///
/// A dictionary has no field for its values, so it returns the fields nested in the value type.
fn nested_fields(data_type: &DataType) -> Vec<&Field> {
match data_type {
DataType::Struct(fields) => fields.iter().map(Arc::as_ref).collect(),
DataType::Union(fields, _) => fields.iter().map(|(_, field)| field.as_ref()).collect(),
DataType::List(field)
| DataType::LargeList(field)
| DataType::ListView(field)
| DataType::LargeListView(field)
| DataType::FixedSizeList(field, _)
| DataType::Map(field, _)
| DataType::RunEndEncoded(_, field) => vec![field.as_ref()],
DataType::Dictionary(_, values) => nested_fields(values),
// Other Arrow types have no nested fields.
_ => Vec::new(),
}
}

/// Extension trait for accessing the [`ArrowSession`] on a Vortex session.
pub trait ArrowSessionExt: SessionExt {
/// Get the Arrow session, creating it on first use.
Expand Down Expand Up @@ -1795,4 +1905,80 @@ mod tests {
);
Ok(())
}

const EXPORTER_NAME: &str = "test.exporter";
static EXPORTER_ID: CachedId = CachedId::new(EXPORTER_NAME);

/// Declines every array, because the tests only need an exporter registered under its name.
#[derive(Debug)]
struct DecliningExporter;

impl ArrowExportVTable for DecliningExporter {
fn arrow_ext_id(&self) -> Id {
*EXPORTER_ID
}

fn vortex_id(&self) -> Id {
*EXPORTER_ID
}

fn to_arrow_field(
&self,
_name: &str,
_dtype: &DType,
_session: &ArrowSession,
) -> VortexResult<Option<Field>> {
Ok(None)
}

fn execute_arrow(
&self,
array: ArrayRef,
_target: &Field,
_options: &ArrowExportOptions,
_ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowExport> {
Ok(ArrowExport::Unsupported(array))
}
}

/// An `Int32` field that names `extension_name`, or a plain one without it.
fn leaf(extension_name: Option<&str>) -> FieldRef {
let mut leaf = Field::new("leaf", DataType::Int32, true);
if let Some(name) = extension_name {
leaf = leaf
.with_metadata([(EXTENSION_TYPE_NAME_KEY.to_string(), name.to_string())].into());
}

Arc::new(leaf)
}

/// Only a leaf that names a registered exporter makes an export call a plugin, at any depth.
#[rstest]
#[case::top_level( |leaf: FieldRef| leaf.as_ref().clone())]
#[case::struct_child( |leaf: FieldRef| Field::new_struct("a", vec![leaf], true))]
#[case::list_item( |leaf: FieldRef| Field::new_list("a", leaf, true))]
#[case::run_end_values(|leaf: FieldRef| {
let run_ends = Arc::new(Field::new("run_ends", DataType::Int32, false));
Field::new("a", DataType::RunEndEncoded(run_ends, leaf), true)
})]
#[case::dictionary_values(|leaf: FieldRef| {
Field::new_dictionary("a", DataType::Int32, DataType::Struct(vec![leaf].into()), true)
})]
fn test_exports_without_plugins_finds_exporters_at_any_depth(
#[case] wrap: fn(FieldRef) -> Field,
#[values(None, Some("test.unregistered"), Some(EXPORTER_NAME))] leaf_name: Option<&str>,
) {
let vortex_session = VortexSession::empty();
let session = vortex_session.arrow();
session.register_exporter(Arc::new(DecliningExporter));

let field = wrap(leaf(leaf_name));
let leaf_names_exporter = leaf_name == Some(EXPORTER_NAME);

assert_eq!(
session.exports_without_plugins(&field),
!leaf_names_exporter
);
}
}
119 changes: 85 additions & 34 deletions vortex-datafusion/src/convert/exprs/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ use vortex::expr::Expression;
use vortex::expr::Lambda;
use vortex::expr::byte_length;
use vortex::expr::cast;
use vortex::expr::ext_storage;
use vortex::expr::get_item;
use vortex::expr::in_list_of;
use vortex::expr::is_not_null;
Expand Down Expand Up @@ -266,6 +267,8 @@ type Conversion<T> = Result<T, Unconverted>;
/// Nested struct field access also uses Vortex semantics: a null parent produces a null child,
/// whereas DataFusion extracts the child without applying parent validity. These differences
/// are accepted for both filters and projections.
/// Field access through an extension can run natively when Arrow exposes its Vortex storage and no
/// plugin exports the selected field.
/// Searched `CASE WHEN` expressions preserve short-circuit evaluation: conditions and
/// branch values only execute on the rows that reach them, including fallible branches.
/// Other expressions require compatible SQL semantics or remain in DataFusion.
Expand Down Expand Up @@ -746,40 +749,7 @@ impl DefaultExpressionConverter {
}

if ScalarFunctionExpr::try_downcast_func::<GetFieldFunc>(scalar_fn).is_some() {
// DataFusion's GetFieldFunc flattens nested field access into a single call
// with multiple field name arguments, e.g. `outer.inner.leaf` becomes
// get_field(Column("outer"), "inner", "leaf").
let [source, paths @ ..] = scalar_fn.args() else {
return Err(
exec_datafusion_err!("get_field requires a source and field path").into(),
);
};
if paths.is_empty() {
return Err(exec_datafusion_err!("get_field requires a field path").into());
}
let mut source_type = source.data_type(schema)?;
let mut names = Vec::with_capacity(paths.len());
for path in paths {
let name = path
.downcast_ref::<df_expr::Literal>()
.and_then(|literal| literal.value().try_as_str().flatten())
.ok_or_else(|| {
exec_datafusion_err!("get_field path must be a non-null string literal")
})?;
let DataType::Struct(fields) = &source_type else {
return Err(Unconverted::Unsupported);
};
let (_, field) = fields.find(name).ok_or_else(|| {
exec_datafusion_err!("get_field references missing field {name}")
})?;
source_type = field.data_type().clone();
names.push(name);
}
let mut result = self.convert_expr(source, schema, input_dtype)?;
for name in names {
result = get_item(name, result);
}
return Ok(result);
return self.convert_get_field(scalar_fn, schema, input_dtype);
}

if ScalarFunctionExpr::try_downcast_func::<ArraySum>(scalar_fn).is_some() {
Expand Down Expand Up @@ -823,6 +793,87 @@ impl DefaultExpressionConverter {
Ok(cast(length(input), return_dtype))
}

/// Converts `get_field` to a chain of `get_item` reads.
///
/// DataFusion can flatten nested access into one call, so `outer.inner.leaf` can arrive as
/// `get_field(outer, "inner", "leaf")`.
///
/// Vortex extension types are opaque, so `get_item` cannot read into an extension value until
/// `ext_storage` exposes its storage. The converter can add that `ext_storage` without breaking
/// the opacity, because DataFusion already reads the storage: an Arrow extension type is only
/// metadata on its storage field, and `get_field` reads that field like any other struct. Both
/// sides read the same values only when Vortex exports the parent as exactly its storage, which
/// [`exports_extension_storage`] checks.
///
/// [`exports_extension_storage`]: vortex_arrow::ArrowSession::exports_extension_storage
fn convert_get_field(
&self,
scalar_fn: &ScalarFunctionExpr,
schema: &Schema,
input_dtype: &DType,
) -> Conversion<Expression> {
let [source, paths @ ..] = scalar_fn.args() else {
return Err(exec_datafusion_err!("get_field requires a source and field path").into());
};

if paths.is_empty() {
return Err(exec_datafusion_err!("get_field requires a field path").into());
}

// Keep the Arrow field of each step, because its metadata decides how Vortex exports it.
let source_field = source.return_field(schema)?;
let mut field = source_field.as_ref();
let mut steps = Vec::with_capacity(paths.len());
for path in paths {
let name = path
.downcast_ref::<df_expr::Literal>()
.and_then(|literal| literal.value().try_as_str().flatten())
.ok_or_else(|| {
exec_datafusion_err!("get_field path must be a non-null string literal")
})?;
let DataType::Struct(fields) = field.data_type() else {
return Err(Unconverted::Unsupported);
};
let (_, child_field) = fields
.find(name)
.ok_or_else(|| exec_datafusion_err!("get_field references missing field {name}"))?;
steps.push((field, name));
field = child_field.as_ref();
}

let mut read = self.convert_expr(source, schema, input_dtype)?;
let mut reads_extension_storage = false;
for (parent_field, name) in steps {
// Check the Vortex dtype instead of the Arrow extension name, because the session
// imports an unknown extension name as a plain struct that `get_item` reads directly.
if let DType::Extension(ext_dtype) = converted_dtype(&read, input_dtype)? {
if !self
.session
.arrow()
.exports_extension_storage(&ext_dtype, parent_field)
{
return Err(Unconverted::Unsupported);
}

read = ext_storage(read);
reads_extension_storage = true;
}
read = get_item(name, read);
}
Comment on lines +844 to +862

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And the extension check and ext_storage insertion here is new.


// A plugin exporter decides which Arrow data DataFusion sees for the selected field, so
// Vortex and DataFusion agree on that field only if the export keeps values. Columns and
// plain struct fields already assume this, and the check skips them so that their pushdown
// stays the same.
// TODO(connor): Drop this check once the Arrow plugin contract requires exports to keep
// values.
if reads_extension_storage && !self.session.arrow().exports_without_plugins(field) {
return Err(Unconverted::Unsupported);
}

Ok(read)
}

/// Converts `array_sum` to `list_sum`.
///
/// DataFusion casts the list to `List<Float64>`, then adds each list's valid elements in
Expand Down
Loading
Loading