diff --git a/vortex-duckdb/src/e2e_test/spatial_pushdown_test.rs b/vortex-duckdb/src/e2e_test/spatial_pushdown_test.rs index 0bed547c516..9ff0567f789 100644 --- a/vortex-duckdb/src/e2e_test/spatial_pushdown_test.rs +++ b/vortex-duckdb/src/e2e_test/spatial_pushdown_test.rs @@ -117,13 +117,13 @@ fn explain_plan(conn: &Connection, query: &str) -> String { plan } -/// Assert that filtering `table` with `filter` counts `expected` points and leaves no `FILTER` -/// operator in the plan, i.e. the predicate ran inside the scan. +/// Assert that filtering `table` with `filter` counts `expected` points and that the Vortex scan +/// lists the predicate. DuckDB keeps its own copy of the filter in the plan for its estimates. fn assert_pushed(conn: &Connection, table: &str, filter: &str, expected: i64) { let query = format!("SELECT count(*) FROM {table} WHERE {filter}"); assert_eq!(query_i64(conn, &query), expected); let plan = explain_plan(conn, &query); - assert!(!plan.contains("FILTER"), "filter was not pushed:\n{plan}"); + assert!(plan.contains("Filters"), "filter was not pushed:\n{plan}"); } /// Every lowered spatial filter pushes on a direct file scan. diff --git a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs index d804d168123..c0b2f82eade 100644 --- a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs +++ b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs @@ -305,7 +305,7 @@ fn assert_membership_filter_pushed( } } assert!(plan.contains("Read Vortex"), "missing Vortex scan:\n{plan}"); - assert!(!plan.contains("FILTER"), "filter was not pushed:\n{plan}"); + assert!(plan.contains("Filters"), "filter was not pushed:\n{plan}"); Ok(()) } diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index d10c7b39809..e667e89a3ab 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use std::sync::OnceLock; -use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use futures::FutureExt; @@ -199,7 +198,6 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR first_file_row_count: file.reader().row_count(), filters: vec![], columns, - has_non_optional_filter: AtomicBool::new(false), aggregates: vec![], no_footer_caches: false, }) diff --git a/vortex-duckdb/src/projection.rs b/vortex-duckdb/src/projection.rs index beec2cc74db..54ec96743b4 100644 --- a/vortex-duckdb/src/projection.rs +++ b/vortex-duckdb/src/projection.rs @@ -25,9 +25,7 @@ use vortex_utils::aliases::hash_set::HashSet; use crate::convert::try_from_table_filter; use crate::convert::try_from_virtual_column_filter; -use crate::duckdb::ExpressionClass; use crate::duckdb::LogicalType; -use crate::duckdb::TableFilterClass; use crate::duckdb::TableFilterSetRef; use crate::table_function::ColumnAggregate; @@ -192,7 +190,6 @@ pub struct Filter { pub filter: Option, pub row_selection: Selection, pub row_range: Option>, - pub has_non_optional_filter: bool, } fn push_filter_expr(filter_exprs: &mut Vec, expr: &Expression) { @@ -211,23 +208,12 @@ impl Filter { additional_filters: &[Expression], dtype: &DType, ) -> VortexResult { - let mut has_non_optional_filter = false; - let mut table_filter_exprs = Vec::new(); if let Some(filter) = table_filter_set { for (idx, ex) in filter.into_iter().filter(|(idx, _)| { let idx_u: usize = idx.as_(); !is_virtual_column(column_ids[idx_u]) }) { - has_non_optional_filter |= match ex.as_class() { - TableFilterClass::Optional(_) => false, - TableFilterClass::ExpressionRef(expr) => match expr.as_class() { - Some(ExpressionClass::BoundFunction(bfn)) => bfn.optional().is_none(), - _ => true, - }, - _ => true, - }; - let idx_u: usize = idx.as_(); let col_idx: usize = column_ids[idx_u].as_(); let name = &column_fields.get(col_idx).vortex_expect("exists").name; @@ -260,7 +246,6 @@ impl Filter { filter, row_selection, row_range, - has_non_optional_filter, }; Ok(out) } diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index dd0a9ffc5f1..377ccf5cfd5 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -5,7 +5,6 @@ use std::cmp::max; use std::fmt::Formatter; use std::fmt::{self}; use std::sync::Arc; -use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; @@ -90,9 +89,6 @@ pub(crate) struct BindState { pub first_file_row_count: u64, pub filters: Vec, pub columns: Vec, - // There exists at least one non-optional table filter or at least one - // complex filter is pushed down. - pub has_non_optional_filter: AtomicBool, // Non-empty iff this scan is aggregate pub aggregates: Vec, // Any of files present in scan lack a footer @@ -107,7 +103,6 @@ impl BindState { first_file_row_count: 0, filters: vec![], columns: vec![], - has_non_optional_filter: AtomicBool::new(false), aggregates: vec![], no_footer_caches: true, } @@ -121,9 +116,6 @@ impl Clone for BindState { first_file_row_count: self.first_file_row_count, filters: vec![], columns: self.columns.clone(), - has_non_optional_filter: AtomicBool::new( - self.has_non_optional_filter.load(Ordering::Relaxed), - ), aggregates: self.aggregates.clone(), no_footer_caches: false, } @@ -149,8 +141,7 @@ impl fmt::Debug for BindState { const COUNT_STAR_KIND: &str = "count_star"; #[derive(Clone, PartialEq, prost::Message)] struct SerializedBindState { - #[prost(bool, tag = "1")] - has_non_optional_filter: bool, + // Tag 1 held the removed `has_non_optional_filter` flag. Do not reuse it. #[prost(message, repeated, tag = "2")] filters: Vec, #[prost(uint64, repeated, tag = "3")] @@ -237,7 +228,7 @@ pub(crate) enum ColumnAggregate { CountStar, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub enum Cardinality { /// The exact number of rows. Exact(u64), @@ -331,11 +322,6 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { &bind_data.filters, &bind_data.dtype, )?; - if filter.has_non_optional_filter { - bind_data - .has_non_optional_filter - .store(true, Ordering::Relaxed); - } debug!( %projection, @@ -470,6 +456,14 @@ fn aggregate_output_value(scalar: Scalar, expected: &LogicalTypeRef) -> VortexRe } } +/// Copies a DuckDB filter into the scan, so the scan can prune and filter +/// with it. +/// +/// Always returns `false`, so DuckDB also keeps the filter in its plan. The +/// join order optimizer estimates the selectivity only of filters that it +/// can see. If the scan took the filter away, DuckDB would treat the +/// filtered scan as unfiltered, and it could pick a join order that makes +/// large intermediate results, as in TPC-DS query 64. pub fn pushdown_complex_filter( bind_data: &mut BindState, expr: &ExpressionRef, @@ -481,15 +475,13 @@ pub fn pushdown_complex_filter( return Ok(false); }; - // Only table filters may be optional, any complex filter is - // non-optional by definition. - bind_data - .has_non_optional_filter - .store(true, Ordering::Relaxed); + // DuckDB keeps the filter, so a later optimizer pass can offer it again. + if !bind_data.filters.contains(&expr) { + debug!(%expr, "pushed down expression"); + bind_data.filters.push(expr); + } - debug!(%expr, "pushed down expression"); - bind_data.filters.push(expr); - Ok(true) + Ok(false) } pub fn pushdown_projection_expression( @@ -634,34 +626,27 @@ fn try_push_projection_aggregate( })) } -/// Duckdb requires post-filter cardinality estimates, otherwise join planner -/// may flip join sides which is a huge regression for some queries i.e. 1000x -/// for tpcds 85. +/// Reports the row count of the scan before filters. /// -/// See duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +/// DuckDB applies the selectivity of the filters itself, because every filter +/// stays in its plan (see [`pushdown_complex_filter`]). DuckDB also uses this +/// count as the upper limit for distinct values of a join key, so a +/// post-filter count here makes join estimates too high. /// -/// As we don't report distinct values (same as Parquet), the only heuristic -/// duckdb uses is a 0.2 filter if there is any non-optional filter. We mimic it -/// here. -const DEFAULT_SELECTIVITY: f64 = 0.2; +/// See duckdb/src/optimizer/relation_statistics/relation_statistics_scan.cpp pub fn cardinality(bind_data: &BindState, file_count: u64) -> Cardinality { // If we're doing an aggregate scan, we don't change output cardinality to // 1 as we want duckdb to do our aggregation in parallel. That may look // counterintuitive in the plan, though. - let has_non_optional_filter = bind_data.has_non_optional_filter.load(Ordering::Relaxed); let total = bind_data .first_file_row_count .saturating_mul(max(file_count, 1)); - if !has_non_optional_filter { - return if file_count <= 1 { - Cardinality::Exact(total) - } else { - Cardinality::Estimate(total) - }; + + if file_count <= 1 { + Cardinality::Exact(total) + } else { + Cardinality::Estimate(total) } - let post_cardinality = total as f64 * DEFAULT_SELECTIVITY; - let post_cardinality: u64 = post_cardinality.as_(); - Cardinality::Estimate(max(1, post_cardinality)) } pub fn to_string(bind_data: &BindState, map: &mut DuckdbStringMapRef) { @@ -708,10 +693,7 @@ pub fn to_string(bind_data: &BindState, map: &mut DuckdbStringMapRef) { } pub fn serialize(bind_data: &BindState) -> VortexResult> { - let mut state = SerializedBindState { - has_non_optional_filter: bind_data.has_non_optional_filter.load(Ordering::Relaxed), - ..Default::default() - }; + let mut state = SerializedBindState::default(); for expr in &bind_data.filters { state.filters.push(expr.serialize_proto()?); } @@ -748,9 +730,6 @@ pub fn deserialize( "bind state length mismatch" ); - bind_data - .has_non_optional_filter - .store(state.has_non_optional_filter, Ordering::Relaxed); bind_data.filters = state .filters .iter() @@ -788,3 +767,30 @@ pub fn deserialize( } Ok(()) } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex::expr::col; + use vortex::expr::gt; + use vortex::expr::lit; + + use super::BindState; + use super::Cardinality; + use super::cardinality; + + /// Pushed filters stay in DuckDB's plan, so the scan reports its row count before filters. + #[rstest] + #[case::one_file(1, Cardinality::Exact(100))] + #[case::many_files(3, Cardinality::Estimate(300))] + fn cardinality_ignores_pushed_filters( + #[case] file_count: u64, + #[case] expected: Cardinality, + ) { + let mut bind_data = BindState::empty(); + bind_data.first_file_row_count = 100; + bind_data.filters.push(gt(col("x"), lit(1i32))); + + assert_eq!(cardinality(&bind_data, file_count), expected); + } +}