diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 48bf8a391d5c3..b3daff1e9fba2 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -847,10 +847,13 @@ impl Interval { let zero = ScalarValue::new_zero(&dt)?; // We want 0 to be approachable from both negative and positive sides. let zero_point = match &dt { - DataType::Float32 | DataType::Float64 => Self::new(zero.clone(), zero), - _ => Self::new(prev_value(zero.clone()), next_value(zero)), + DataType::Float32 | DataType::Float64 => { + Self::new(zero.clone(), zero.clone()) + } + _ => Self::new(prev_value(zero.clone()), next_value(zero.clone())), }; + // The helpers treat intervals with a finite upper bound <= 0 as non-positive. // Exit early with an unbounded interval if zero is strictly inside the // right hand side: if rhs_ref.contains(&zero_point)? == Self::TRUE && !dt.is_unsigned_integer() { @@ -861,19 +864,9 @@ impl Interval { else if lhs_ref.contains(&zero_point)? == Self::TRUE && !dt.is_unsigned_integer() { - Ok(div_helper_lhs_zero_inclusive( - &dt, - lhs_ref, - rhs_ref, - &zero_point, - )) + Ok(div_helper_lhs_zero_inclusive(&dt, lhs_ref, rhs_ref, &zero)) } else { - Ok(div_helper_zero_exclusive( - &dt, - lhs_ref, - rhs_ref, - &zero_point, - )) + Ok(div_helper_zero_exclusive(&dt, lhs_ref, rhs_ref, &zero)) } } @@ -1640,10 +1633,10 @@ fn div_helper_lhs_zero_inclusive( dt: &DataType, lhs: &Interval, rhs: &Interval, - zero_point: &Interval, + zero: &ScalarValue, ) -> Interval { // With the following interval bounds, there is no possibility to create an invalid interval. - if rhs.upper <= zero_point.lower && !rhs.upper.is_null() { + if rhs.upper <= *zero && !rhs.upper.is_null() { // <-------=====0=====-------> // <--======----0------------> let lower = div_bounds::(dt, &lhs.upper, &rhs.upper); @@ -1659,12 +1652,12 @@ fn div_helper_lhs_zero_inclusive( } /// Divides the left-hand side interval by the right-hand side interval when -/// neither interval contains zero. +/// neither interval straddles zero (zero endpoints are allowed). /// /// This function takes in two intervals (`lhs` and `rhs`) as arguments and /// returns their quotient (whose data type is known to be `dt`). It is -/// specifically designed to handle intervals that do not contain zero within -/// their ranges. Returns an error if the division of bounds fails. +/// specifically designed to handle intervals that do not straddle zero. +/// Returns an error if the division of bounds fails. /// /// ``` text /// Left-hand side: <--======----0------------> @@ -1693,11 +1686,11 @@ fn div_helper_zero_exclusive( dt: &DataType, lhs: &Interval, rhs: &Interval, - zero_point: &Interval, + zero: &ScalarValue, ) -> Interval { let (lower, upper) = match ( - lhs.upper <= zero_point.lower && !lhs.upper.is_null(), - rhs.upper <= zero_point.lower && !rhs.upper.is_null(), + lhs.upper <= *zero && !lhs.upper.is_null(), + rhs.upper <= *zero && !rhs.upper.is_null(), ) { // With the following interval bounds, there is no possibility to create an invalid interval. (true, true) => ( @@ -3674,9 +3667,123 @@ mod tests { Ok(()) } + #[test] + fn test_div_small_integer_intervals_contain_runtime_results() -> Result<()> { + // Include zero at either endpoint, inside the range, and as a singleton. + for lo in -3_i64..=3 { + for hi in lo..=3 { + for rlo in -3_i64..=3 { + for rhi in rlo..=3 { + let lhs = Interval::make(Some(lo), Some(hi))?; + let rhs = Interval::make(Some(rlo), Some(rhi))?; + let result = lhs.div(&rhs)?; + for a in lo..=hi { + for b in rlo..=rhi { + if b != 0 { + assert!( + result.contains_value(ScalarValue::Int64( + Some(a / b) + ))?, + "{lhs:?} / {rhs:?} = {result:?} excludes {a} / {b}", + ); + } + } + } + } + } + } + } + Ok(()) + } + #[test] fn test_div() -> Result<()> { let cases = vec![ + ( + Interval::make(Some(-3_i64), Some(0_i64))?, + Interval::make(Some(-2_i64), Some(-1_i64))?, + Interval::make(Some(0_i64), Some(3_i64))?, + ), + ( + Interval::make(Some(-3_i64), Some(0_i64))?, + Interval::make(Some(1_i64), Some(2_i64))?, + Interval::make(Some(-3_i64), Some(0_i64))?, + ), + ( + Interval::make(Some(0_i64), Some(3_i64))?, + Interval::make(Some(-2_i64), Some(-1_i64))?, + Interval::make(Some(-3_i64), Some(0_i64))?, + ), + ( + Interval::make(Some(0_i64), Some(3_i64))?, + Interval::make(Some(1_i64), Some(2_i64))?, + Interval::make(Some(0_i64), Some(3_i64))?, + ), + ( + Interval::make(Some(-3_i64), Some(-1_i64))?, + Interval::make(Some(-3_i64), Some(0_i64))?, + Interval::make(Some(0_i64), None)?, + ), + ( + Interval::make(Some(1_i64), Some(3_i64))?, + Interval::make(Some(-3_i64), Some(0_i64))?, + Interval::make(None, Some(0_i64))?, + ), + ( + Interval::make(Some(-3_i64), Some(-1_i64))?, + Interval::make(Some(0_i64), Some(3_i64))?, + Interval::make(None, Some(0_i64))?, + ), + ( + Interval::make(Some(1_i64), Some(3_i64))?, + Interval::make(Some(0_i64), Some(3_i64))?, + Interval::make(Some(0_i64), None)?, + ), + ( + Interval::make(Some(-6_i64), Some(6_i64))?, + Interval::make(Some(-3_i64), Some(0_i64))?, + Interval::make::(None, None)?, + ), + ( + Interval::make(Some(-6_i64), Some(6_i64))?, + Interval::make(Some(0_i64), Some(3_i64))?, + Interval::make::(None, None)?, + ), + ( + Interval::make(Some(0_i64), Some(0_i64))?, + Interval::make(Some(0_i64), Some(0_i64))?, + Interval::make::(None, None)?, + ), + ( + Interval::make(Some(0_u32), Some(0_u32))?, + Interval::make(Some(0_u32), Some(5_u32))?, + Interval::make(Some(0_u32), Some(0_u32))?, + ), + ( + Interval::make(Some(0_u32), Some(0_u32))?, + Interval::make(Some(0_u32), Some(0_u32))?, + Interval::make::(None, None)?, + ), + ( + Interval::make(Some(0_u32), Some(5_u32))?, + Interval::make(Some(0_u32), Some(5_u32))?, + Interval::make(Some(0_u32), None)?, + ), + ( + Interval::make(Some(1_u32), Some(5_u32))?, + Interval::make(Some(0_u32), Some(5_u32))?, + Interval::make(Some(0_u32), None)?, + ), + ( + Interval::make(Some(0_u64), Some(0_u64))?, + Interval::make(Some(0_u64), Some(5_u64))?, + Interval::make(Some(0_u64), Some(0_u64))?, + ), + ( + Interval::make(Some(0_u64), Some(0_u64))?, + Interval::make(Some(0_u64), Some(0_u64))?, + Interval::make::(None, None)?, + ), ( Interval::make(Some(100_i64), Some(200_i64))?, Interval::make(Some(1_i64), Some(2_i64))?, @@ -3854,7 +3961,7 @@ mod tests { ), ]; for case in cases { - let result = case.0.div(case.1)?; + let result = case.0.div(&case.1)?; if case.0.data_type().is_floating() { assert!( result.lower().is_null() && case.2.lower().is_null() @@ -3865,7 +3972,7 @@ mod tests { || result.upper().ge(case.2.upper()) ); } else { - assert_eq!(result, case.2); + assert_eq!(result, case.2, "{:?} / {:?}", case.0, case.1); } } diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt index 2a8d1747518d2..84383413fa8cd 100644 --- a/datafusion/sqllogictest/test_files/parquet_statistics.slt +++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt @@ -222,6 +222,37 @@ physical_plan statement ok DROP TABLE mul_wrap; +###### +# Division with a zero numerator endpoint must not infer an empty filter +###### + +statement ok +COPY ( + SELECT * FROM (VALUES (-3, -1), (-2, -1), (0, -2)) AS t(a, b) +) +TO 'test_files/scratch/parquet_statistics/div_zero_endpoint.parquet' STORED AS PARQUET; + +statement ok +CREATE EXTERNAL TABLE div_zero_endpoint STORED AS PARQUET +LOCATION 'test_files/scratch/parquet_statistics/div_zero_endpoint.parquet'; + +# [-3,0] / [-2,-1] must include 2 and 3, rather than being narrowed to [0,1]. +query III +SELECT a, b, a / b AS q FROM div_zero_endpoint WHERE a / b > 1 ORDER BY a; +---- +-3 -1 3 +-2 -1 2 + +query TT +EXPLAIN SELECT a, b FROM div_zero_endpoint WHERE a / b > 1; +---- +physical_plan +01)FilterExec: a@0 / b@1 > 1, statistics=[Rows=Inexact(3), Bytes=Inexact(48), [(Col[0]: Min=Inexact(Int64(-3)) Max=Inexact(Int64(0)) Null=Inexact(0) ScanBytes=Inexact(24)),(Col[1]: Min=Inexact(Int64(-2)) Max=Inexact(Int64(-1)) Null=Inexact(0) ScanBytes=Inexact(24))]] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/div_zero_endpoint.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 / b@1 > 1, statistics=[Rows=Inexact(3), Bytes=Inexact(48), [(Col[0]: Min=Inexact(Int64(-3)) Max=Inexact(Int64(0)) Null=Inexact(0) ScanBytes=Inexact(24)),(Col[1]: Min=Inexact(Int64(-2)) Max=Inexact(Int64(-1)) Null=Inexact(0) ScanBytes=Inexact(24))]] + +statement ok +DROP TABLE div_zero_endpoint; + ###### # Aggregate statistics through casts ######