Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions vortex-datafusion/src/persistent/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -215,9 +215,10 @@ extensions_options! {
pub footer_initial_read_size_bytes: usize, default = DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES
/// Whether to enable projection pushdown into the underlying Vortex scan.
///
/// When enabled, projection expressions may be partially evaluated during
/// the scan. When disabled, Vortex reads only the referenced columns and
/// all expressions are evaluated after the scan.
/// When enabled, the scan evaluates projections whose expressions all convert
/// to Vortex, and DataFusion evaluates any other projection after the scan.
/// When disabled, Vortex reads only the referenced columns and all expressions
/// are evaluated after the scan.
///
/// Disabled by default.
pub projection_pushdown: bool, default = false
Expand Down
31 changes: 24 additions & 7 deletions vortex-datafusion/src/persistent/opener.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use std::sync::Weak;
use arrow_array::RecordBatchOptions;
use arrow_schema::DataType;
use arrow_schema::Field;
use arrow_schema::Schema;
use arrow_schema::SchemaRef;
use datafusion_common::DataFusionError;
use datafusion_common::Result as DFResult;
Expand Down Expand Up @@ -315,9 +316,6 @@ impl FileOpener for VortexOpener {
let projection =
projection.try_map_exprs(|p| simplifier.simplify(expr_adapter.rewrite(p)?))?;

// TODO: support lambda pushdown.
let projection_pushdown = projection_pushdown && !contains_lambda(&projection)?;

// Split the adapted filter into conjuncts Vortex evaluates natively and residual
// conjuncts DataFusion evaluates on the raw scan output.
let mut native_filters = Vec::new();
Expand Down Expand Up @@ -373,14 +371,14 @@ impl FileOpener for VortexOpener {
scan_reference_schema,
leftover_projection: projection,
}
} else if projection_pushdown {
expr_converter.split_projection(
} else {
process_projection(
expr_converter.as_ref(),
projection,
&this_file_schema,
output_schema.as_ref(),
projection_pushdown,
)?
} else {
expr_converter.no_pushdown_projection(projection, &this_file_schema)?
};

// The schema of the stream returned from the vortex scan.
Expand Down Expand Up @@ -680,6 +678,25 @@ impl NaturalSplits {
}
}

/// Splits `projection` into the Vortex scan projection and the DataFusion leftover.
///
/// [`VortexSource`](super::VortexSource) makes the same split against the table schema when
/// planning, and accepts only projections the scan evaluates entirely.
pub(super) fn process_projection(
expr_converter: &dyn ExpressionConverter,
projection: ProjectionExprs,
input_schema: &Schema,
output_schema: &Schema,
projection_pushdown: bool,
) -> DFResult<ProcessedProjection> {
// TODO: support lambda pushdown.
if projection_pushdown && !contains_lambda(&projection)? {
expr_converter.split_projection(projection, input_schema, output_schema)
} else {
expr_converter.no_pushdown_projection(projection, input_schema)
}
}

/// Whether any expression in `projection` contains a lambda.
fn contains_lambda(projection: &ProjectionExprs) -> DFResult<bool> {
for projection_expr in projection.iter() {
Expand Down
79 changes: 70 additions & 9 deletions vortex-datafusion/src/persistent/source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use datafusion_physical_expr::LexOrdering;
use datafusion_physical_expr::PhysicalExprRef;
use datafusion_physical_expr::PhysicalSortExpr;
use datafusion_physical_expr::conjunction;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
Expand All @@ -43,9 +44,11 @@ use vortex_utils::aliases::dash_map::DashMap;

use super::opener::NaturalSplits;
use super::opener::VortexOpener;
use super::opener::process_projection;
use crate::VortexTableOptions;
use crate::convert::exprs::DefaultExpressionConverter;
use crate::convert::exprs::ExpressionConverter;
use crate::convert::exprs::ProcessedProjection;
use crate::persistent::reader::DefaultVortexReaderFactory;
use crate::persistent::reader::VortexReaderFactory;

Expand Down Expand Up @@ -137,13 +140,15 @@ use crate::persistent::reader::VortexReaderFactory;
/// - `vortex_predicate`, which contains accepted exact filters. After per-file
/// adaptation, these run either natively or as DataFusion residual filters.
///
/// Projection handling depends on
/// [`VortexTableOptions::projection_pushdown`]:
/// `VortexSource` accepts a projection only when the scan evaluates all of it, so
/// DataFusion keeps any other projection in a `ProjectionExec` above the scan. What
/// the scan evaluates depends on [`VortexTableOptions::projection_pushdown`]:
///
/// - when disabled, `VortexSource` still prunes unreferenced top-level columns,
/// but DataFusion applies the full projection after the scan,
/// - when enabled, the default converter evaluates fully supported projections
/// natively. Otherwise, DataFusion evaluates the full projection over raw columns.
/// - when disabled, the scan only selects top-level columns,
/// - when enabled, the scan also evaluates projections the converter fully supports.
///
/// A file whose schema differs from the table schema can still need DataFusion to
/// evaluate part of an accepted projection, which then happens inside the scan.
///
/// Predicate handling depends on [`VortexTableOptions::predicate_pushdown`]:
///
Expand Down Expand Up @@ -563,8 +568,31 @@ impl FileSource for VortexSource {
&self,
projection: &ProjectionExprs,
) -> DFResult<Option<Arc<dyn FileSource>>> {
let projection = self.projection.try_merge(projection)?;

// Accept only projections the scan evaluates entirely, so DataFusion keeps anything else
// in its own `ProjectionExec`. Files whose schema differs from the table's are split
// again when opened, and may still evaluate part of the projection inside the scan.
let table_schema = self.table_schema.table_schema();
let ProcessedProjection {
leftover_projection,
..
} = process_projection(
self.expression_converter.as_ref(),
projection.clone(),
table_schema,
&projection.project_schema(table_schema)?,
self.options.projection_pushdown,
)?;
if !leftover_projection
.iter()
.all(|leftover| leftover.expr.is::<Column>())
{
return Ok(None);
}

let mut source = self.clone();
source.projection = self.projection.try_merge(projection)?;
source.projection = projection;
Ok(Some(Arc::new(source)))
}

Expand Down Expand Up @@ -604,12 +632,12 @@ mod tests {
use datafusion_functions::string::octet_length::OctetLengthFunc;
use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::expressions as df_expr;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::projection::ProjectionExpr;
use object_store::memory::InMemory;
use rstest::rstest;
use vortex::VortexSessionDefault;

use super::*;
use crate::convert::exprs::ProcessedProjection;

struct TrackingExpressionConverter {
inner: DefaultExpressionConverter,
Expand Down Expand Up @@ -774,4 +802,37 @@ mod tests {
assert!(updated_source.vortex_predicate.is_none());
Ok(())
}

/// `None` projects column `b`; `Some(op)` projects `a op 2`. Vortex converts `+` but not `%`.
#[rstest]
#[case::columns(&[None], false, true)]
#[case::columns_with_pushdown(&[None], true, true)]
#[case::convertible_without_pushdown(&[Some(Operator::Plus)], false, false)]
#[case::convertible(&[None, Some(Operator::Plus)], true, true)]
#[case::unconvertible(&[Some(Operator::Modulo)], true, false)]
#[case::mixed(&[Some(Operator::Plus), Some(Operator::Modulo)], true, false)]
fn try_pushdown_projection_accepts_only_what_the_scan_evaluates(
#[case] exprs: &[Option<Operator>],
#[case] projection_pushdown: bool,
#[case] accepted: bool,
) -> anyhow::Result<()> {
let source =
sort_test_source(sort_test_schema()).with_projection_pushdown(projection_pushdown);
let projection = ProjectionExprs::new(exprs.iter().enumerate().map(|(i, op)| {
let expr: PhysicalExprRef = match op {
None => Arc::new(Column::new("b", 1)),
Some(op) => Arc::new(df_expr::BinaryExpr::new(
Arc::new(Column::new("a", 0)),
*op,
Arc::new(df_expr::Literal::new(ScalarValue::Int32(Some(2)))),
)),
};
ProjectionExpr::new(expr, format!("e{i}"))
}));

let pushed = source.try_pushdown_projection(&projection)?;

assert_eq!(pushed.is_some(), accepted);
Ok(())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[sum(hits.ResolutionWidth), count(hits.ResolutionWidth)]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.ResolutionWidth), count(hits.ResolutionWidth)]
05)--------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/hits.vortex:<slt:ignore>], [${WORK_DIR}/hits.vortex:<slt:ignore>], [${WORK_DIR}/hits.vortex:<slt:ignore>], [${WORK_DIR}/hits.vortex:<slt:ignore>]]}, projection=[CAST(ResolutionWidth@20 AS Int64) as __common_expr_2], file_type=vortex
05)--------ProjectionExec: expr=[CAST(ResolutionWidth@0 AS Int64) as __common_expr_2]
06)----------DataSourceExec: file_groups={4 groups: [[${WORK_DIR}/hits.vortex:<slt:ignore>], [${WORK_DIR}/hits.vortex:<slt:ignore>], [${WORK_DIR}/hits.vortex:<slt:ignore>], [${WORK_DIR}/hits.vortex:<slt:ignore>]]}, projection=[ResolutionWidth], file_type=vortex
7 changes: 5 additions & 2 deletions vortex-sqllogictest/slt/datafusion/decimal_to_integer.slt
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,18 @@ COPY (SELECT id, CAST(value AS DECIMAL(21,1)) AS value FROM (VALUES
statement ok
CREATE VIEW decimals AS SELECT * FROM '${WORK_DIR}/decimal_to_integer.vortex';

# EXPLAIN verifies that the cast executes inside the Vortex scan.
# Vortex does not convert decimal-to-integer casts, so EXPLAIN shows the cast in a
# DataFusion ProjectionExec above the scan.
query TT
EXPLAIN SELECT CAST(value AS BIGINT) AS integer_value FROM decimals;
----
logical_plan
01)Projection: CAST(decimals.value AS Int64) AS integer_value
02)--SubqueryAlias: decimals
03)----TableScan: ${WORK_DIR}/decimal_to_integer.vortex projection=[value]
physical_plan DataSourceExec: file_groups={1 group: [[${WORK_DIR}/decimal_to_integer.vortex]]}, projection=[CAST(value@1 AS Int64) as integer_value], file_type=vortex
physical_plan
01)ProjectionExec: expr=[CAST(value@0 AS Int64) as integer_value]
02)--DataSourceExec: file_groups={1 group: [[${WORK_DIR}/decimal_to_integer.vortex]]}, projection=[value], file_type=vortex

# Fractions truncate toward zero; large integers remain exact and nulls survive.
query I
Expand Down
Loading