Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions vortex-duckdb/src/e2e_test/spatial_pushdown_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,13 @@ fn explain_plan(conn: &Connection, query: &str) -> String {
plan
}

/// Assert that filtering `table` with `filter` counts `expected` points and leaves no `FILTER`
/// operator in the plan, i.e. the predicate ran inside the scan.
/// Assert that filtering `table` with `filter` counts `expected` points and that the Vortex scan
/// lists the predicate. DuckDB keeps its own copy of the filter in the plan for its estimates.
fn assert_pushed(conn: &Connection, table: &str, filter: &str, expected: i64) {
let query = format!("SELECT count(*) FROM {table} WHERE {filter}");
assert_eq!(query_i64(conn, &query), expected);
let plan = explain_plan(conn, &query);
assert!(!plan.contains("FILTER"), "filter was not pushed:\n{plan}");
assert!(plan.contains("Filters"), "filter was not pushed:\n{plan}");
}

/// Every lowered spatial filter pushes on a direct file scan.
Expand Down
2 changes: 1 addition & 1 deletion vortex-duckdb/src/e2e_test/vortex_scan_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ fn assert_membership_filter_pushed(
}
}
assert!(plan.contains("Read Vortex"), "missing Vortex scan:\n{plan}");
assert!(!plan.contains("FILTER"), "filter was not pushed:\n{plan}");
assert!(plan.contains("Filters"), "filter was not pushed:\n{plan}");

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't change the string

Ok(())
}

Expand Down
2 changes: 0 additions & 2 deletions vortex-duckdb/src/file_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;

use futures::FutureExt;
Expand Down Expand Up @@ -199,7 +198,6 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR
first_file_row_count: file.reader().row_count(),
filters: vec![],
columns,
has_non_optional_filter: AtomicBool::new(false),
aggregates: vec![],
no_footer_caches: false,
})
Expand Down
15 changes: 0 additions & 15 deletions vortex-duckdb/src/projection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,7 @@ use vortex_utils::aliases::hash_set::HashSet;

use crate::convert::try_from_table_filter;
use crate::convert::try_from_virtual_column_filter;
use crate::duckdb::ExpressionClass;
use crate::duckdb::LogicalType;
use crate::duckdb::TableFilterClass;
use crate::duckdb::TableFilterSetRef;
use crate::table_function::ColumnAggregate;

Expand Down Expand Up @@ -192,7 +190,6 @@ pub struct Filter {
pub filter: Option<BoundExpression>,
pub row_selection: Selection,
pub row_range: Option<Range<u64>>,
pub has_non_optional_filter: bool,
}

fn push_filter_expr(filter_exprs: &mut Vec<Expression>, expr: &Expression) {
Expand All @@ -211,23 +208,12 @@ impl Filter {
additional_filters: &[Expression],
dtype: &DType,
) -> VortexResult<Self> {
let mut has_non_optional_filter = false;

let mut table_filter_exprs = Vec::new();
if let Some(filter) = table_filter_set {
for (idx, ex) in filter.into_iter().filter(|(idx, _)| {
let idx_u: usize = idx.as_();
!is_virtual_column(column_ids[idx_u])
}) {
has_non_optional_filter |= match ex.as_class() {
TableFilterClass::Optional(_) => false,
TableFilterClass::ExpressionRef(expr) => match expr.as_class() {
Some(ExpressionClass::BoundFunction(bfn)) => bfn.optional().is_none(),
_ => true,
},
_ => true,
};

let idx_u: usize = idx.as_();
let col_idx: usize = column_ids[idx_u].as_();
let name = &column_fields.get(col_idx).vortex_expect("exists").name;
Expand Down Expand Up @@ -260,7 +246,6 @@ impl Filter {
filter,
row_selection,
row_range,
has_non_optional_filter,
};
Ok(out)
}
Expand Down
104 changes: 55 additions & 49 deletions vortex-duckdb/src/table_function.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use std::cmp::max;
use std::fmt::Formatter;
use std::fmt::{self};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;

Expand Down Expand Up @@ -90,9 +89,6 @@ pub(crate) struct BindState {
pub first_file_row_count: u64,
pub filters: Vec<Expression>,
pub columns: Vec<DuckdbField>,
// There exists at least one non-optional table filter or at least one
// complex filter is pushed down.
pub has_non_optional_filter: AtomicBool,
// Non-empty iff this scan is aggregate
pub aggregates: Vec<ColumnAggregate>,
// Any of files present in scan lack a footer
Expand All @@ -107,7 +103,6 @@ impl BindState {
first_file_row_count: 0,
filters: vec![],
columns: vec![],
has_non_optional_filter: AtomicBool::new(false),
aggregates: vec![],
no_footer_caches: true,
}
Expand All @@ -121,9 +116,6 @@ impl Clone for BindState {
first_file_row_count: self.first_file_row_count,
filters: vec![],
columns: self.columns.clone(),
has_non_optional_filter: AtomicBool::new(
self.has_non_optional_filter.load(Ordering::Relaxed),
),
aggregates: self.aggregates.clone(),
no_footer_caches: false,
}
Expand All @@ -149,8 +141,7 @@ impl fmt::Debug for BindState {
const COUNT_STAR_KIND: &str = "count_star";
#[derive(Clone, PartialEq, prost::Message)]
struct SerializedBindState {
#[prost(bool, tag = "1")]
has_non_optional_filter: bool,
// Tag 1 held the removed `has_non_optional_filter` flag. Do not reuse it.
#[prost(message, repeated, tag = "2")]
filters: Vec<expr_pb::Expr>,
#[prost(uint64, repeated, tag = "3")]
Expand Down Expand Up @@ -237,7 +228,7 @@ pub(crate) enum ColumnAggregate {
CountStar,
}

#[derive(Debug)]
#[derive(Debug, PartialEq, Eq)]
pub enum Cardinality {
/// The exact number of rows.
Exact(u64),
Expand Down Expand Up @@ -331,11 +322,6 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult<GlobalState> {
&bind_data.filters,
&bind_data.dtype,
)?;
if filter.has_non_optional_filter {
bind_data
.has_non_optional_filter
.store(true, Ordering::Relaxed);
}

debug!(
%projection,
Expand Down Expand Up @@ -470,6 +456,14 @@ fn aggregate_output_value(scalar: Scalar, expected: &LogicalTypeRef) -> VortexRe
}
}

/// Copies a DuckDB filter into the scan, so the scan can prune and filter
/// with it.
///
/// Always returns `false`, so DuckDB also keeps the filter in its plan. The
/// join order optimizer estimates the selectivity only of filters that it
/// can see. If the scan took the filter away, DuckDB would treat the
/// filtered scan as unfiltered, and it could pick a join order that makes
/// large intermediate results, as in TPC-DS query 64.
pub fn pushdown_complex_filter(
bind_data: &mut BindState,
expr: &ExpressionRef,
Expand All @@ -481,15 +475,13 @@ pub fn pushdown_complex_filter(
return Ok(false);
};

// Only table filters may be optional, any complex filter is
// non-optional by definition.
bind_data
.has_non_optional_filter
.store(true, Ordering::Relaxed);
// DuckDB keeps the filter, so a later optimizer pass can offer it again.
if !bind_data.filters.contains(&expr) {
debug!(%expr, "pushed down expression");
bind_data.filters.push(expr);
}

debug!(%expr, "pushed down expression");
bind_data.filters.push(expr);
Ok(true)
Ok(false)
}

pub fn pushdown_projection_expression(
Expand Down Expand Up @@ -634,34 +626,27 @@ fn try_push_projection_aggregate(
}))
}

/// Duckdb requires post-filter cardinality estimates, otherwise join planner
/// may flip join sides which is a huge regression for some queries i.e. 1000x
/// for tpcds 85.
/// Reports the row count of the scan before filters.
///
/// See duckdb/src/optimizer/join_order/relation_statistics_helper.cpp
/// DuckDB applies the selectivity of the filters itself, because every filter
/// stays in its plan (see [`pushdown_complex_filter`]). DuckDB also uses this
/// count as the upper limit for distinct values of a join key, so a
/// post-filter count here makes join estimates too high.
///
/// As we don't report distinct values (same as Parquet), the only heuristic
/// duckdb uses is a 0.2 filter if there is any non-optional filter. We mimic it
/// here.
const DEFAULT_SELECTIVITY: f64 = 0.2;
/// See duckdb/src/optimizer/relation_statistics/relation_statistics_scan.cpp
pub fn cardinality(bind_data: &BindState, file_count: u64) -> Cardinality {
// If we're doing an aggregate scan, we don't change output cardinality to
// 1 as we want duckdb to do our aggregation in parallel. That may look
// counterintuitive in the plan, though.
let has_non_optional_filter = bind_data.has_non_optional_filter.load(Ordering::Relaxed);
let total = bind_data
.first_file_row_count
.saturating_mul(max(file_count, 1));
if !has_non_optional_filter {
return if file_count <= 1 {
Cardinality::Exact(total)
} else {
Cardinality::Estimate(total)
};

if file_count <= 1 {
Cardinality::Exact(total)
} else {
Cardinality::Estimate(total)
}
let post_cardinality = total as f64 * DEFAULT_SELECTIVITY;
let post_cardinality: u64 = post_cardinality.as_();
Cardinality::Estimate(max(1, post_cardinality))
}

pub fn to_string(bind_data: &BindState, map: &mut DuckdbStringMapRef) {
Expand Down Expand Up @@ -708,10 +693,7 @@ pub fn to_string(bind_data: &BindState, map: &mut DuckdbStringMapRef) {
}

pub fn serialize(bind_data: &BindState) -> VortexResult<Vec<u8>> {
let mut state = SerializedBindState {
has_non_optional_filter: bind_data.has_non_optional_filter.load(Ordering::Relaxed),
..Default::default()
};
let mut state = SerializedBindState::default();
for expr in &bind_data.filters {
state.filters.push(expr.serialize_proto()?);
}
Expand Down Expand Up @@ -748,9 +730,6 @@ pub fn deserialize(
"bind state length mismatch"
);

bind_data
.has_non_optional_filter
.store(state.has_non_optional_filter, Ordering::Relaxed);
bind_data.filters = state
.filters
.iter()
Expand Down Expand Up @@ -788,3 +767,30 @@ pub fn deserialize(
}
Ok(())
}

#[cfg(test)]
mod tests {
use rstest::rstest;
use vortex::expr::col;
use vortex::expr::gt;
use vortex::expr::lit;

use super::BindState;
use super::Cardinality;
use super::cardinality;

/// Pushed filters stay in DuckDB's plan, so the scan reports its row count before filters.
#[rstest]
#[case::one_file(1, Cardinality::Exact(100))]
#[case::many_files(3, Cardinality::Estimate(300))]
fn cardinality_ignores_pushed_filters(
#[case] file_count: u64,
#[case] expected: Cardinality,
) {
let mut bind_data = BindState::empty();
bind_data.first_file_row_count = 100;
bind_data.filters.push(gt(col("x"), lit(1i32)));

assert_eq!(cardinality(&bind_data, file_count), expected);
}
}
Loading