From 23215f8ecacc2ac14da386dcf229d2b4a7b699bc Mon Sep 17 00:00:00 2001 From: Patrick More Date: Wed, 5 May 2021 16:59:36 -0700 Subject: [PATCH 1/9] Added large list optimizations for InList using HashSet. Added short circuiting extreme values check on list. --- .../src/physical_plan/expressions/in_list.rs | 465 ++++++++++++++---- 1 file changed, 363 insertions(+), 102 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index 41f111006ea2a..c9b785d002968 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -17,14 +17,13 @@ //! InList expression -use std::any::Any; use std::sync::Arc; +use std::{any::Any, collections::HashSet}; -use arrow::array::GenericStringArray; -use arrow::array::{ - ArrayRef, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array, - Int64Array, Int8Array, StringOffsetSizeTrait, UInt16Array, UInt32Array, UInt64Array, - UInt8Array, +use arrow::array::{ArrayRef, BooleanArray, StringOffsetSizeTrait}; +use arrow::{ + array::{Array, GenericStringArray, PrimitiveArray}, + datatypes::ArrowPrimitiveType, }; use arrow::{ datatypes::{DataType, Schema}, @@ -43,59 +42,193 @@ pub struct InListExpr { negated: bool, } -macro_rules! make_contains { - ($ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $ARRAY_TYPE:ident) => {{ - let array = $ARRAY.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap(); +use std::iter::Iterator; +//This function computes the in list boolean array over an iterator +//the checks for the behavior for the negated and the contains_null are lifted outside the loops to try to avoid branches within the loop +fn compute_contains_array< + V, + I: Iterator>, + F: FnMut(Option) -> Option, +>( + it: I, + negated: bool, + contains_null: bool, + contains: F, +) -> BooleanArray { + let it_contains = it.map(contains); + match (negated, contains_null) { + (true, true) => it_contains + .map(|x| match x { + Some(true) => Some(false), + Some(false) | None => None, + }) + .collect::(), + (true, false) => it_contains.map(|x| x.map(|x| !x)).collect::(), + (false, true) => it_contains + .map(|x| match x { + Some(true) => Some(true), + Some(false) | None => None, + }) + .collect::(), + (false, false) => it_contains.collect::(), + } +} +//Collects the scalar values from a specified discriminant into a vector, marking if there are any null values. +fn collect_scalar_value Option>( + vals: Vec, + contains_null: &mut bool, + disc: std::mem::Discriminant, + scalar_handler: F, +) -> Vec { + vals.iter() + .flat_map(|expr| match expr { + ColumnarValue::Scalar(s) => match s { + sc if std::mem::discriminant(s) == disc => { + scalar_handler(sc, contains_null) + } + ScalarValue::Utf8(None) => { + *contains_null = true; + None + } + datatype => unimplemented!("Unexpected type {} for InList", datatype), + }, + ColumnarValue::Array(_) => { + unimplemented!("InList does not yet support nested columns.") + } + }) + .collect::>() +} - let mut contains_null = false; - let values = $LIST_VALUES - .iter() - .flat_map(|expr| match expr { - ColumnarValue::Scalar(s) => match s { - ScalarValue::$SCALAR_VALUE(Some(v)) => Some(*v), - ScalarValue::$SCALAR_VALUE(None) => { - contains_null = true; - None - } - ScalarValue::Utf8(None) => { - contains_null = true; - None - } - datatype => unimplemented!("Unexpected type {} for InList", datatype), - }, - ColumnarValue::Array(_) => { - unimplemented!("InList does not yet support nested columns.") +//Checks for values in the array of the specified ScalarValue discriminant for partially ordered that can be totally ordered types like f32 and f64. +#[allow(clippy::unnecessary_wraps, clippy::too_many_arguments)] +fn contains_partial_ord_type( + array: Arc, + list_values: Vec, + negated: bool, + disc: std::mem::Discriminant, + linear_search_upper_bound: usize, + binary_search_upper_bound: usize, + make_ord_val: V, + scalar_handler: F, +) -> Result +where + A: ArrowPrimitiveType, + V: Fn(N) -> O + Copy, + O: Ord + std::hash::Hash + Eq, + F: Fn(&ScalarValue, &mut bool) -> Option, +{ + let arr = array + .as_ref() + .as_any() + .downcast_ref::>() + .unwrap(); + let mut contains_null = false; + let handler_wrapper = move |s: &ScalarValue, cont_null: &mut bool| -> Option { + let partial_ord_val = scalar_handler(s, cont_null); + partial_ord_val.map(make_ord_val) + }; + let mut values = + collect_scalar_value(list_values, &mut contains_null, disc, handler_wrapper); + let num_values = values.len(); + let contains_arr = if num_values == 0 { + compute_contains_array(arr.iter(), negated, contains_null, |val| { + Some(val.is_none()) + }) + } else if num_values <= linear_search_upper_bound { + values.sort_unstable(); + compute_contains_array(arr.iter(), negated, contains_null, |val| { + val.map(|x| { + let ord_val = make_ord_val(x); + if *values.first().unwrap() > ord_val || *values.last().unwrap() < ord_val + { + return false; } + values.contains(&ord_val) }) - .collect::>(); + }) + } else if num_values <= binary_search_upper_bound { + values.sort_unstable(); + compute_contains_array(arr.iter(), negated, contains_null, |val| { + val.map(|x| { + let ord_val = make_ord_val(x); + if *values.first().unwrap() > ord_val || *values.last().unwrap() < ord_val + { + return false; + } + values.binary_search(&ord_val).is_ok() + }) + }) + } else { + let set = values.into_iter().collect::>(); // HashSet::with_capacity(values.len()); + compute_contains_array(arr.iter(), negated, contains_null, |val| { + val.map(|x| set.contains(&make_ord_val(x))) + }) + }; + let col_val = ColumnarValue::Array(Arc::new(contains_arr)); + Ok(col_val) +} - Ok(ColumnarValue::Array(Arc::new( - array - .iter() - .map(|x| { - let contains = x.map(|x| values.contains(&x)); - match contains { - Some(true) => { - if $NEGATED { - Some(false) - } else { - Some(true) - } - } - Some(false) => { - if contains_null { - None - } else if $NEGATED { - Some(true) - } else { - Some(false) - } - } - None => None, - } - }) - .collect::(), - ))) + +//Checks if the values are in the array after downcasting it to the specified Primitive type +fn contains_ord_types( + array: Arc, + list_values: Vec, + negated: bool, + disc: std::mem::Discriminant, + linear_search_upper_bound: usize, + binary_search_upper_bound: usize, + scalar_handler: F, +) -> Result +where + N: Ord + std::hash::Hash, + A: ArrowPrimitiveType, + F: Fn(&ScalarValue, &mut bool) -> Option, +{ + let make_ord_val = |partial_ord_val: N| -> N { partial_ord_val }; + contains_partial_ord_type::( + array, + list_values, + negated, + disc, + linear_search_upper_bound, + binary_search_upper_bound, + make_ord_val, + scalar_handler, + ) +} + +//The last two literals are the linear search upper bound and the binary search upper bound before the strategy switches to a HashSet. +//If a single literal is given then it is used for both search bounds and the binary search will never be performed. + +macro_rules! make_ord_contains { + ($PRIM_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal) => {{ + make_ord_contains!( + $PRIM_TYPE, + $ARRAY, + $LIST_VALUES, + $NEGATED, + $SCALAR_VALUE, + $LINEAR_SEARCH_BOUND, + $LINEAR_SEARCH_BOUND + ) + }}; + ($PRIM_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal, $BINARY_SEARCH_BOUND:literal) => {{ + contains_ord_types::<$PRIM_TYPE, <$PRIM_TYPE as ArrowPrimitiveType>::Native, _>( + $ARRAY, + $LIST_VALUES, + $NEGATED, + std::mem::discriminant(&ScalarValue::$SCALAR_VALUE(None)), + $LINEAR_SEARCH_BOUND, + $BINARY_SEARCH_BOUND, + |s, contains_null| match s { + ScalarValue::$SCALAR_VALUE(Some(v)) => Some(*v), + ScalarValue::$SCALAR_VALUE(None) => { + *contains_null = true; + None + } + _ => unreachable!(), + }, + ) }}; } @@ -134,7 +267,6 @@ impl InListExpr { &self, array: ArrayRef, list_values: Vec, - negated: bool, ) -> Result { let array = array .as_any() @@ -142,7 +274,8 @@ impl InListExpr { .unwrap(); let mut contains_null = false; - let values = list_values + + let mut values = list_values .iter() .flat_map(|expr| match expr { ColumnarValue::Scalar(s) => match s { @@ -162,35 +295,32 @@ impl InListExpr { unimplemented!("InList does not yet support nested columns.") } }) - .collect::>(); - - Ok(ColumnarValue::Array(Arc::new( - array - .iter() - .map(|x| { - let contains = x.map(|x| values.contains(&x)); - match contains { - Some(true) => { - if negated { - Some(false) - } else { - Some(true) - } - } - Some(false) => { - if contains_null { - None - } else if negated { - Some(true) - } else { - Some(false) - } - } - None => None, - } + .collect::>(); + + const LINEAR_SEARCH_UPPER_BOUND: usize = 8; + let contains_arr = match values.len() { + 0 => { + if !contains_null { + return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(false)))); + } + compute_contains_array(array.iter(), self.negated, contains_null, |x| { + x.map(|x| values.contains(&x)) + }) + } + 1..=LINEAR_SEARCH_UPPER_BOUND => { + values.sort_unstable(); + compute_contains_array(array.iter(), self.negated, contains_null, |x| { + x.map(|x| values.contains(&x)) }) - .collect::(), - ))) + } + _ => { + let set = values.into_iter().collect::>(); + compute_contains_array(array.iter(), self.negated, contains_null, |x| { + x.map(|s| set.contains(s)) + }) + } + }; + Ok(ColumnarValue::Array(Arc::new(contains_arr))) } } @@ -231,45 +361,172 @@ impl PhysicalExpr for InListExpr { ColumnarValue::Array(array) => array, ColumnarValue::Scalar(scalar) => scalar.to_array(), }; - + use arrow::datatypes::{ + Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, + UInt16Type, UInt32Type, UInt64Type, UInt8Type, + }; match value_data_type { DataType::Float32 => { - make_contains!(array, list_values, self.negated, Float32, Float32Array) + let disc = std::mem::discriminant(&ScalarValue::Float32(None)); + contains_partial_ord_type::( + array, + list_values, + self.negated, + disc, + 8, + 16, + ordered_float::OrderedFloat::from, + |s, contains_null| match s { + ScalarValue::Float32(Some(v)) => Some(*v), + ScalarValue::Float32(None) => { + *contains_null = true; + None + } + _ => unreachable!(), + }, + ) } DataType::Float64 => { - make_contains!(array, list_values, self.negated, Float64, Float64Array) + let disc = std::mem::discriminant(&ScalarValue::Float64(None)); + contains_partial_ord_type::( + array, + list_values, + self.negated, + disc, + 8, + 16, + ordered_float::OrderedFloat::from, + |s, contains_null| match s { + ScalarValue::Float64(Some(v)) => Some(*v), + ScalarValue::Float64(None) => { + *contains_null = true; + None + } + _ => unreachable!(), + }, + ) } DataType::Int16 => { - make_contains!(array, list_values, self.negated, Int16, Int16Array) + make_ord_contains!( + Int16Type, + array, + list_values, + self.negated, + Int16, + 256 + ) } DataType::Int32 => { - make_contains!(array, list_values, self.negated, Int32, Int32Array) + make_ord_contains!( + Int32Type, + array, + list_values, + self.negated, + Int32, + 256 + ) } DataType::Int64 => { - make_contains!(array, list_values, self.negated, Int64, Int64Array) + make_ord_contains!( + Int64Type, + array, + list_values, + self.negated, + Int64, + 128 + ) } DataType::Int8 => { - make_contains!(array, list_values, self.negated, Int8, Int8Array) + make_ord_contains!(Int8Type, array, list_values, self.negated, Int8, 256) } DataType::UInt16 => { - make_contains!(array, list_values, self.negated, UInt16, UInt16Array) + make_ord_contains!( + UInt16Type, + array, + list_values, + self.negated, + UInt16, + 256 + ) } DataType::UInt32 => { - make_contains!(array, list_values, self.negated, UInt32, UInt32Array) + make_ord_contains!( + UInt32Type, + array, + list_values, + self.negated, + UInt32, + 256 + ) } DataType::UInt64 => { - make_contains!(array, list_values, self.negated, UInt64, UInt64Array) + make_ord_contains!( + UInt64Type, + array, + list_values, + self.negated, + UInt64, + 128 + ) } DataType::UInt8 => { - make_contains!(array, list_values, self.negated, UInt8, UInt8Array) + make_ord_contains!( + UInt8Type, + array, + list_values, + self.negated, + UInt8, + 256 + ) } DataType::Boolean => { - make_contains!(array, list_values, self.negated, Boolean, BooleanArray) - } - DataType::Utf8 => self.compare_utf8::(array, list_values, self.negated), - DataType::LargeUtf8 => { - self.compare_utf8::(array, list_values, self.negated) + let mut contains_nulls = false; + let bool_disc = std::mem::discriminant(&ScalarValue::Boolean(None)); + let values = collect_scalar_value( + list_values, + &mut contains_nulls, + bool_disc, + |s, contains_null| match s { + ScalarValue::Boolean(Some(v)) => Some(*v), + ScalarValue::Boolean(None) => { + *contains_null = true; + None + } + _ => unreachable!(), + }, + ); + + if values.is_empty() { + return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(false)))); + } + let mut found_true = false; + let mut found_false = false; + let exhaustively_true = values.iter().any(|b| { + match b { + true => { + found_true = true; + } + false => { + found_false = true; + } + } + found_true && found_false + }); + if exhaustively_true { + return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(true)))); + } + let match_bool = found_true; + let bool_iter = array.as_any().downcast_ref::().unwrap(); + let contains_arr = compute_contains_array( + bool_iter.iter(), + self.negated, + contains_nulls, + |o| o.map(|b| b == match_bool), + ); + Ok(ColumnarValue::Array(Arc::new(contains_arr))) } + DataType::Utf8 => self.compare_utf8::(array, list_values), + DataType::LargeUtf8 => self.compare_utf8::(array, list_values), datatype => { unimplemented!("InList does not support datatype {:?}.", datatype) } @@ -288,7 +545,11 @@ pub fn in_list( #[cfg(test)] mod tests { - use arrow::{array::StringArray, datatypes::Field}; + use arrow::{ + array::StringArray, + array::{Float64Array, Int64Array}, + datatypes::Field, + }; use super::*; use crate::error::Result; From d37d5a0a96a0cb074479e746761c589b6370a79a Mon Sep 17 00:00:00 2001 From: Patrick More Date: Wed, 5 May 2021 19:31:16 -0700 Subject: [PATCH 2/9] Corrected behavior when list was all null values --- .../src/physical_plan/expressions/in_list.rs | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index c9b785d002968..46d8823d24055 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -99,7 +99,7 @@ fn collect_scalar_value Option>( .collect::>() } -//Checks for values in the array of the specified ScalarValue discriminant for partially ordered that can be totally ordered types like f32 and f64. +//Checks for values in the array of the specified ScalarValue discriminant for partially ordered that can be totally ordered types like f32 and f64. #[allow(clippy::unnecessary_wraps, clippy::too_many_arguments)] fn contains_partial_ord_type( array: Arc, @@ -130,17 +130,18 @@ where let mut values = collect_scalar_value(list_values, &mut contains_null, disc, handler_wrapper); let num_values = values.len(); - let contains_arr = if num_values == 0 { - compute_contains_array(arr.iter(), negated, contains_null, |val| { - Some(val.is_none()) - }) - } else if num_values <= linear_search_upper_bound { + let contains_arr = if num_values <= linear_search_upper_bound { values.sort_unstable(); compute_contains_array(arr.iter(), negated, contains_null, |val| { val.map(|x| { let ord_val = make_ord_val(x); - if *values.first().unwrap() > ord_val || *values.last().unwrap() < ord_val - { + let out_of_bounds = match (values.first(), values.last()) { + (Some(lowest), Some(highest)) => { + *lowest > ord_val || *highest < ord_val + } + _ => false, + }; + if out_of_bounds { return false; } values.contains(&ord_val) @@ -168,7 +169,6 @@ where Ok(col_val) } - //Checks if the values are in the array after downcasting it to the specified Primitive type fn contains_ord_types( array: Arc, @@ -299,15 +299,7 @@ impl InListExpr { const LINEAR_SEARCH_UPPER_BOUND: usize = 8; let contains_arr = match values.len() { - 0 => { - if !contains_null { - return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(false)))); - } - compute_contains_array(array.iter(), self.negated, contains_null, |x| { - x.map(|x| values.contains(&x)) - }) - } - 1..=LINEAR_SEARCH_UPPER_BOUND => { + 0..=LINEAR_SEARCH_UPPER_BOUND => { values.sort_unstable(); compute_contains_array(array.iter(), self.negated, contains_null, |x| { x.map(|x| values.contains(&x)) @@ -373,8 +365,8 @@ impl PhysicalExpr for InListExpr { list_values, self.negated, disc, - 8, - 16, + 256, + 256, ordered_float::OrderedFloat::from, |s, contains_null| match s { ScalarValue::Float32(Some(v)) => Some(*v), @@ -393,8 +385,8 @@ impl PhysicalExpr for InListExpr { list_values, self.negated, disc, - 8, - 16, + 128, + 128, ordered_float::OrderedFloat::from, |s, contains_null| match s { ScalarValue::Float64(Some(v)) => Some(*v), @@ -713,7 +705,12 @@ mod tests { lit(ScalarValue::Utf8(None)), ]; in_list!(batch, list, &true, vec![Some(false), None]); - + ////Expression "a in true (NULL, true)" + //let list=vec![ + // lit(ScalarValue::Utf8(None)), + //]; + ////let batch = BooleanArray::from(vec![Some(true), None]); + //in_list!(batch, list, &false, vec![Some(false), None]); Ok(()) } } From 368a54093b594f7349c981a88279f9d02a50d347 Mon Sep 17 00:00:00 2001 From: Patrick More Date: Wed, 5 May 2021 19:32:29 -0700 Subject: [PATCH 3/9] Removed incorrect test case --- datafusion/src/physical_plan/expressions/in_list.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index 46d8823d24055..bea7378adebd3 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -705,12 +705,6 @@ mod tests { lit(ScalarValue::Utf8(None)), ]; in_list!(batch, list, &true, vec![Some(false), None]); - ////Expression "a in true (NULL, true)" - //let list=vec![ - // lit(ScalarValue::Utf8(None)), - //]; - ////let batch = BooleanArray::from(vec![Some(true), None]); - //in_list!(batch, list, &false, vec![Some(false), None]); Ok(()) } } From 63a0c450d6d4cb6fa48485122adfd688a9af4d4f Mon Sep 17 00:00:00 2001 From: Patrick More Date: Thu, 6 May 2021 11:22:58 -0700 Subject: [PATCH 4/9] Reworked InList contains functions to be more readable. Added more tests to boolean InList due to abundance of special cases in implementation. --- .../src/physical_plan/expressions/in_list.rs | 276 +++++++++++------- 1 file changed, 169 insertions(+), 107 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index bea7378adebd3..dfb9693e3996b 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -17,8 +17,8 @@ //! InList expression -use std::sync::Arc; use std::{any::Any, collections::HashSet}; +use std::{hash::Hash, sync::Arc}; use arrow::array::{ArrayRef, BooleanArray, StringOffsetSizeTrait}; use arrow::{ @@ -99,101 +99,114 @@ fn collect_scalar_value Option>( .collect::>() } -//Checks for values in the array of the specified ScalarValue discriminant for partially ordered that can be totally ordered types like f32 and f64. -#[allow(clippy::unnecessary_wraps, clippy::too_many_arguments)] -fn contains_partial_ord_type( - array: Arc, - list_values: Vec, +#[allow(clippy::unnecessary_wraps)] +fn contains_ord_types_it( + it: I, + mut list_values: Vec, negated: bool, - disc: std::mem::Discriminant, - linear_search_upper_bound: usize, - binary_search_upper_bound: usize, - make_ord_val: V, - scalar_handler: F, + contains_null: bool, + method_bounds: (usize, usize), ) -> Result where - A: ArrowPrimitiveType, - V: Fn(N) -> O + Copy, - O: Ord + std::hash::Hash + Eq, - F: Fn(&ScalarValue, &mut bool) -> Option, + I: Iterator>, + O: Ord + Hash + Eq, { - let arr = array - .as_ref() - .as_any() - .downcast_ref::>() - .unwrap(); - let mut contains_null = false; - let handler_wrapper = move |s: &ScalarValue, cont_null: &mut bool| -> Option { - let partial_ord_val = scalar_handler(s, cont_null); - partial_ord_val.map(make_ord_val) - }; - let mut values = - collect_scalar_value(list_values, &mut contains_null, disc, handler_wrapper); - let num_values = values.len(); - let contains_arr = if num_values <= linear_search_upper_bound { - values.sort_unstable(); - compute_contains_array(arr.iter(), negated, contains_null, |val| { + let num_values = list_values.len(); + let contains_arr = if num_values <= method_bounds.0 { + list_values.sort_unstable(); + compute_contains_array(it, negated, contains_null, |val| { val.map(|x| { - let ord_val = make_ord_val(x); - let out_of_bounds = match (values.first(), values.last()) { - (Some(lowest), Some(highest)) => { - *lowest > ord_val || *highest < ord_val - } + let out_of_bounds = match (list_values.first(), list_values.last()) { + (Some(lowest), Some(highest)) => *lowest > x || *highest < x, _ => false, }; if out_of_bounds { return false; } - values.contains(&ord_val) + list_values.contains(&x) }) }) - } else if num_values <= binary_search_upper_bound { - values.sort_unstable(); - compute_contains_array(arr.iter(), negated, contains_null, |val| { + } else if num_values <= method_bounds.1 { + list_values.sort_unstable(); + compute_contains_array(it, negated, contains_null, |val| { val.map(|x| { - let ord_val = make_ord_val(x); - if *values.first().unwrap() > ord_val || *values.last().unwrap() < ord_val - { + let out_of_bounds = match (list_values.first(), list_values.last()) { + (Some(lowest), Some(highest)) => *lowest > x || *highest < x, + _ => false, + }; + if out_of_bounds { return false; } - values.binary_search(&ord_val).is_ok() + list_values.binary_search(&x).is_ok() }) }) } else { - let set = values.into_iter().collect::>(); // HashSet::with_capacity(values.len()); - compute_contains_array(arr.iter(), negated, contains_null, |val| { - val.map(|x| set.contains(&make_ord_val(x))) + let set = list_values.into_iter().collect::>(); // HashSet::with_capacity(values.len()); + compute_contains_array(it, negated, contains_null, |val| { + val.map(|x| set.contains(&x)) }) }; let col_val = ColumnarValue::Array(Arc::new(contains_arr)); Ok(col_val) } -//Checks if the values are in the array after downcasting it to the specified Primitive type -fn contains_ord_types( +fn contains_ord_types( + array: Arc, + list_values: Vec, + negated: bool, + disc: std::mem::Discriminant, + method_bounds: (usize, usize), + scalar_handler: F, +) -> Result +where + A: ArrowPrimitiveType, + A::Native: Ord + Hash + Eq, + F: Fn(&ScalarValue, &mut bool) -> Option, +{ + let arr = array + .as_ref() + .as_any() + .downcast_ref::>() + .unwrap(); + let mut contains_null = false; + let values = + collect_scalar_value(list_values, &mut contains_null, disc, scalar_handler); + contains_ord_types_it(arr.iter(), values, negated, contains_null, method_bounds) +} + +//Checks for values in the array of the specified ScalarValue discriminant for partially ordered that can be totally ordered types like f32 and f64. +fn contains_partial_ord_type( array: Arc, list_values: Vec, negated: bool, disc: std::mem::Discriminant, - linear_search_upper_bound: usize, - binary_search_upper_bound: usize, + method_bounds: (usize, usize), + make_ord_val: V, scalar_handler: F, ) -> Result where - N: Ord + std::hash::Hash, - A: ArrowPrimitiveType, + A: ArrowPrimitiveType, + V: Copy + Fn(A::Native) -> O, + O: Ord + Hash + Eq, F: Fn(&ScalarValue, &mut bool) -> Option, { - let make_ord_val = |partial_ord_val: N| -> N { partial_ord_val }; - contains_partial_ord_type::( - array, - list_values, + let arr = array + .as_ref() + .as_any() + .downcast_ref::>() + .unwrap(); + let mut contains_null = false; + let handler_wrapper = |s: &ScalarValue, cont_null: &mut bool| { + scalar_handler(s, cont_null).map(make_ord_val) + }; + let values = + collect_scalar_value(list_values, &mut contains_null, disc, handler_wrapper); + contains_ord_types_it( + arr.iter().map(|v| v.map(make_ord_val)), + values, negated, - disc, - linear_search_upper_bound, - binary_search_upper_bound, - make_ord_val, - scalar_handler, + contains_null, + method_bounds, ) } @@ -212,14 +225,13 @@ macro_rules! make_ord_contains { $LINEAR_SEARCH_BOUND ) }}; - ($PRIM_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal, $BINARY_SEARCH_BOUND:literal) => {{ - contains_ord_types::<$PRIM_TYPE, <$PRIM_TYPE as ArrowPrimitiveType>::Native, _>( + ($ARRAY_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal, $BINARY_SEARCH_BOUND:literal) => {{ + contains_ord_types::<$ARRAY_TYPE, _>( $ARRAY, $LIST_VALUES, $NEGATED, std::mem::discriminant(&ScalarValue::$SCALAR_VALUE(None)), - $LINEAR_SEARCH_BOUND, - $BINARY_SEARCH_BOUND, + ($LINEAR_SEARCH_BOUND, $BINARY_SEARCH_BOUND), |s, contains_null| match s { ScalarValue::$SCALAR_VALUE(Some(v)) => Some(*v), ScalarValue::$SCALAR_VALUE(None) => { @@ -274,7 +286,6 @@ impl InListExpr { .unwrap(); let mut contains_null = false; - let mut values = list_values .iter() .flat_map(|expr| match expr { @@ -296,7 +307,8 @@ impl InListExpr { } }) .collect::>(); - + //TODO: determine better bound for when to switch to hashset + // Possibly based off of the number of characters the values in the list have combined with the total length of the list? const LINEAR_SEARCH_UPPER_BOUND: usize = 8; let contains_arr = match values.len() { 0..=LINEAR_SEARCH_UPPER_BOUND => { @@ -360,13 +372,12 @@ impl PhysicalExpr for InListExpr { match value_data_type { DataType::Float32 => { let disc = std::mem::discriminant(&ScalarValue::Float32(None)); - contains_partial_ord_type::( + contains_partial_ord_type::( array, list_values, self.negated, disc, - 256, - 256, + (256, 256), ordered_float::OrderedFloat::from, |s, contains_null| match s { ScalarValue::Float32(Some(v)) => Some(*v), @@ -380,13 +391,12 @@ impl PhysicalExpr for InListExpr { } DataType::Float64 => { let disc = std::mem::discriminant(&ScalarValue::Float64(None)); - contains_partial_ord_type::( + contains_partial_ord_type::( array, list_values, self.negated, disc, - 128, - 128, + (128, 128), ordered_float::OrderedFloat::from, |s, contains_null| match s { ScalarValue::Float64(Some(v)) => Some(*v), @@ -398,6 +408,9 @@ impl PhysicalExpr for InListExpr { }, ) } + DataType::Int8 => { + make_ord_contains!(Int8Type, array, list_values, self.negated, Int8, 256) + } DataType::Int16 => { make_ord_contains!( Int16Type, @@ -428,8 +441,15 @@ impl PhysicalExpr for InListExpr { 128 ) } - DataType::Int8 => { - make_ord_contains!(Int8Type, array, list_values, self.negated, Int8, 256) + DataType::UInt8 => { + make_ord_contains!( + UInt8Type, + array, + list_values, + self.negated, + UInt8, + 256 + ) } DataType::UInt16 => { make_ord_contains!( @@ -461,16 +481,7 @@ impl PhysicalExpr for InListExpr { 128 ) } - DataType::UInt8 => { - make_ord_contains!( - UInt8Type, - array, - list_values, - self.negated, - UInt8, - 256 - ) - } + DataType::Boolean => { let mut contains_nulls = false; let bool_disc = std::mem::discriminant(&ScalarValue::Boolean(None)); @@ -487,13 +498,11 @@ impl PhysicalExpr for InListExpr { _ => unreachable!(), }, ); - - if values.is_empty() { - return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(false)))); - } + let bool_arr = array.as_any().downcast_ref::().unwrap(); let mut found_true = false; let mut found_false = false; - let exhaustively_true = values.iter().any(|b| { + //Check if the list contains both true and false values + values.iter().any(|b| { match b { true => { found_true = true; @@ -504,17 +513,37 @@ impl PhysicalExpr for InListExpr { } found_true && found_false }); - if exhaustively_true { - return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(true)))); - } - let match_bool = found_true; - let bool_iter = array.as_any().downcast_ref::().unwrap(); - let contains_arr = compute_contains_array( - bool_iter.iter(), - self.negated, - contains_nulls, - |o| o.map(|b| b == match_bool), - ); + + let contains_arr = match (found_true, found_false, contains_nulls) { + (true, true, false) => { + //If the list contains both true and false but no nulls then fill the array with !negated + let fill_value = !self.negated; + bool_arr + .iter() + .map(|o| o.map(|_| fill_value)) + .collect::() + } + (true, true, true) => { + //If the list contains both true and false and a null value then fill the array with !negated and map false to null + let fill_value = !self.negated; + bool_arr + .iter() + .map(|o| match o { + Some(true) => Some(fill_value), + Some(false) | None => None, + }) + .collect() + } + _ => { + //Otherwise calculate normally + compute_contains_array( + bool_arr.iter(), + self.negated, + contains_nulls, + |o| o.map(|b| (found_false && !b) || (b && found_true)), + ) + } + }; Ok(ColumnarValue::Array(Arc::new(contains_arr))) } DataType::Utf8 => self.compare_utf8::(array, list_values), @@ -681,30 +710,63 @@ mod tests { #[test] fn in_list_bool() -> Result<()> { let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]); - let a = BooleanArray::from(vec![Some(true), None]); + let a = BooleanArray::from(vec![Some(true), None, Some(false)]); let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)])?; // expression: "a in (true)" let list = vec![lit(ScalarValue::Boolean(Some(true)))]; - in_list!(batch, list, &false, vec![Some(true), None]); + in_list!(batch, list, &false, vec![Some(true), None, Some(false)]); // expression: "a not in (true)" let list = vec![lit(ScalarValue::Boolean(Some(true)))]; - in_list!(batch, list, &true, vec![Some(false), None]); + in_list!(batch, list, &true, vec![Some(false), None, Some(true)]); // expression: "a in (true, NULL)" let list = vec![ lit(ScalarValue::Boolean(Some(true))), lit(ScalarValue::Utf8(None)), ]; - in_list!(batch, list, &false, vec![Some(true), None]); + in_list!(batch, list, &false, vec![Some(true), None, None]); // expression: "a not in (true, NULL)" let list = vec![ lit(ScalarValue::Boolean(Some(true))), lit(ScalarValue::Utf8(None)), ]; - in_list!(batch, list, &true, vec![Some(false), None]); + in_list!(batch, list, &true, vec![Some(false), None, None]); + + // expression: "a in (true, false)" + let list = vec![ + lit(ScalarValue::Boolean(Some(true))), + lit(ScalarValue::Boolean(Some(false))), + ]; + in_list!(batch, list, &false, vec![Some(true), None, Some(true)]); + + + //expression: "a not in (true, false)" + let list = vec![ + lit(ScalarValue::Boolean(Some(true))), + lit(ScalarValue::Boolean(Some(false))), + ]; + in_list!(batch, list, &true, vec![Some(false), None, Some(false)]); + + + // expression: "a in (true, false, NULL)" + let list = vec![ + lit(ScalarValue::Boolean(Some(true))), + lit(ScalarValue::Boolean(Some(false))), + lit(ScalarValue::Utf8(None)) + ]; + in_list!(batch, list, &false, vec![Some(true), None, None]); + + + //expression: "a not in (true, false, NULL)" + let list = vec![ + lit(ScalarValue::Boolean(Some(true))), + lit(ScalarValue::Boolean(Some(false))), + lit(ScalarValue::Utf8(None)) + ]; + in_list!(batch, list, &true, vec![Some(false), None, None]); Ok(()) } } From 73511bb6b2e8da54d92970fc34f7f26ed5d03a79 Mon Sep 17 00:00:00 2001 From: Patrick More Date: Thu, 6 May 2021 11:43:16 -0700 Subject: [PATCH 5/9] Simplified Boolean implementation as more complicated version offered no advantages. --- .../src/physical_plan/expressions/in_list.rs | 45 ++++--------------- 1 file changed, 8 insertions(+), 37 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index dfb9693e3996b..33e962c565de6 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -483,16 +483,16 @@ impl PhysicalExpr for InListExpr { } DataType::Boolean => { - let mut contains_nulls = false; + let mut contains_null = false; let bool_disc = std::mem::discriminant(&ScalarValue::Boolean(None)); let values = collect_scalar_value( list_values, - &mut contains_nulls, + &mut contains_null, bool_disc, - |s, contains_null| match s { + |s, cont_null| match s { ScalarValue::Boolean(Some(v)) => Some(*v), ScalarValue::Boolean(None) => { - *contains_null = true; + *cont_null = true; None } _ => unreachable!(), @@ -513,37 +513,8 @@ impl PhysicalExpr for InListExpr { } found_true && found_false }); - - let contains_arr = match (found_true, found_false, contains_nulls) { - (true, true, false) => { - //If the list contains both true and false but no nulls then fill the array with !negated - let fill_value = !self.negated; - bool_arr - .iter() - .map(|o| o.map(|_| fill_value)) - .collect::() - } - (true, true, true) => { - //If the list contains both true and false and a null value then fill the array with !negated and map false to null - let fill_value = !self.negated; - bool_arr - .iter() - .map(|o| match o { - Some(true) => Some(fill_value), - Some(false) | None => None, - }) - .collect() - } - _ => { - //Otherwise calculate normally - compute_contains_array( - bool_arr.iter(), - self.negated, - contains_nulls, - |o| o.map(|b| (found_false && !b) || (b && found_true)), - ) - } - }; + + let contains_arr = compute_contains_array(bool_arr.iter(), self.negated, contains_null, |o| o.map(|b| (found_true && b) || (found_false && !b))); Ok(ColumnarValue::Array(Arc::new(contains_arr))) } DataType::Utf8 => self.compare_utf8::(array, list_values), @@ -757,7 +728,7 @@ mod tests { lit(ScalarValue::Boolean(Some(false))), lit(ScalarValue::Utf8(None)) ]; - in_list!(batch, list, &false, vec![Some(true), None, None]); + in_list!(batch, list, &false, vec![Some(true), None, Some(true)]); //expression: "a not in (true, false, NULL)" @@ -766,7 +737,7 @@ mod tests { lit(ScalarValue::Boolean(Some(false))), lit(ScalarValue::Utf8(None)) ]; - in_list!(batch, list, &true, vec![Some(false), None, None]); + in_list!(batch, list, &true, vec![Some(false), None, Some(false)]); Ok(()) } } From 5c436319c006a2ada5be5ec273cf56f781b4369e Mon Sep 17 00:00:00 2001 From: Patrick More Date: Thu, 6 May 2021 11:44:42 -0700 Subject: [PATCH 6/9] ran cargo fmt --- .../src/physical_plan/expressions/in_list.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index 33e962c565de6..a24ffde677af4 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -513,8 +513,13 @@ impl PhysicalExpr for InListExpr { } found_true && found_false }); - - let contains_arr = compute_contains_array(bool_arr.iter(), self.negated, contains_null, |o| o.map(|b| (found_true && b) || (found_false && !b))); + + let contains_arr = compute_contains_array( + bool_arr.iter(), + self.negated, + contains_null, + |o| o.map(|b| (found_true && b) || (found_false && !b)), + ); Ok(ColumnarValue::Array(Arc::new(contains_arr))) } DataType::Utf8 => self.compare_utf8::(array, list_values), @@ -713,7 +718,6 @@ mod tests { ]; in_list!(batch, list, &false, vec![Some(true), None, Some(true)]); - //expression: "a not in (true, false)" let list = vec![ lit(ScalarValue::Boolean(Some(true))), @@ -721,21 +725,19 @@ mod tests { ]; in_list!(batch, list, &true, vec![Some(false), None, Some(false)]); - // expression: "a in (true, false, NULL)" let list = vec![ lit(ScalarValue::Boolean(Some(true))), lit(ScalarValue::Boolean(Some(false))), - lit(ScalarValue::Utf8(None)) + lit(ScalarValue::Utf8(None)), ]; in_list!(batch, list, &false, vec![Some(true), None, Some(true)]); - //expression: "a not in (true, false, NULL)" let list = vec![ lit(ScalarValue::Boolean(Some(true))), lit(ScalarValue::Boolean(Some(false))), - lit(ScalarValue::Utf8(None)) + lit(ScalarValue::Utf8(None)), ]; in_list!(batch, list, &true, vec![Some(false), None, Some(false)]); Ok(()) From 4984b9938b99af8b3383e2eae22af1db98696f0a Mon Sep 17 00:00:00 2001 From: Patrick More Date: Thu, 6 May 2021 11:51:14 -0700 Subject: [PATCH 7/9] Removed unused binary search for InList checking --- .../src/physical_plan/expressions/in_list.rs | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index a24ffde677af4..a6f068f93657a 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -105,14 +105,14 @@ fn contains_ord_types_it( mut list_values: Vec, negated: bool, contains_null: bool, - method_bounds: (usize, usize), + linear_search_bound: usize, ) -> Result where I: Iterator>, O: Ord + Hash + Eq, { let num_values = list_values.len(); - let contains_arr = if num_values <= method_bounds.0 { + let contains_arr = if num_values <= linear_search_bound { list_values.sort_unstable(); compute_contains_array(it, negated, contains_null, |val| { val.map(|x| { @@ -126,20 +126,6 @@ where list_values.contains(&x) }) }) - } else if num_values <= method_bounds.1 { - list_values.sort_unstable(); - compute_contains_array(it, negated, contains_null, |val| { - val.map(|x| { - let out_of_bounds = match (list_values.first(), list_values.last()) { - (Some(lowest), Some(highest)) => *lowest > x || *highest < x, - _ => false, - }; - if out_of_bounds { - return false; - } - list_values.binary_search(&x).is_ok() - }) - }) } else { let set = list_values.into_iter().collect::>(); // HashSet::with_capacity(values.len()); compute_contains_array(it, negated, contains_null, |val| { @@ -155,7 +141,7 @@ fn contains_ord_types( list_values: Vec, negated: bool, disc: std::mem::Discriminant, - method_bounds: (usize, usize), + linear_search_bound: usize, scalar_handler: F, ) -> Result where @@ -171,7 +157,13 @@ where let mut contains_null = false; let values = collect_scalar_value(list_values, &mut contains_null, disc, scalar_handler); - contains_ord_types_it(arr.iter(), values, negated, contains_null, method_bounds) + contains_ord_types_it( + arr.iter(), + values, + negated, + contains_null, + linear_search_bound, + ) } //Checks for values in the array of the specified ScalarValue discriminant for partially ordered that can be totally ordered types like f32 and f64. @@ -180,7 +172,7 @@ fn contains_partial_ord_type( list_values: Vec, negated: bool, disc: std::mem::Discriminant, - method_bounds: (usize, usize), + linear_search_bound: usize, make_ord_val: V, scalar_handler: F, ) -> Result @@ -206,7 +198,7 @@ where values, negated, contains_null, - method_bounds, + linear_search_bound, ) } @@ -214,24 +206,13 @@ where //If a single literal is given then it is used for both search bounds and the binary search will never be performed. macro_rules! make_ord_contains { - ($PRIM_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal) => {{ - make_ord_contains!( - $PRIM_TYPE, - $ARRAY, - $LIST_VALUES, - $NEGATED, - $SCALAR_VALUE, - $LINEAR_SEARCH_BOUND, - $LINEAR_SEARCH_BOUND - ) - }}; - ($ARRAY_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal, $BINARY_SEARCH_BOUND:literal) => {{ + ($ARRAY_TYPE:ty, $ARRAY:expr, $LIST_VALUES:expr, $NEGATED:expr, $SCALAR_VALUE:ident, $LINEAR_SEARCH_BOUND:literal) => {{ contains_ord_types::<$ARRAY_TYPE, _>( $ARRAY, $LIST_VALUES, $NEGATED, std::mem::discriminant(&ScalarValue::$SCALAR_VALUE(None)), - ($LINEAR_SEARCH_BOUND, $BINARY_SEARCH_BOUND), + $LINEAR_SEARCH_BOUND, |s, contains_null| match s { ScalarValue::$SCALAR_VALUE(Some(v)) => Some(*v), ScalarValue::$SCALAR_VALUE(None) => { @@ -377,7 +358,7 @@ impl PhysicalExpr for InListExpr { list_values, self.negated, disc, - (256, 256), + 256, ordered_float::OrderedFloat::from, |s, contains_null| match s { ScalarValue::Float32(Some(v)) => Some(*v), @@ -396,7 +377,7 @@ impl PhysicalExpr for InListExpr { list_values, self.negated, disc, - (128, 128), + 128, ordered_float::OrderedFloat::from, |s, contains_null| match s { ScalarValue::Float64(Some(v)) => Some(*v), From 89202bb0ad60d0e019c30deaf705230de2b0ec6d Mon Sep 17 00:00:00 2001 From: Patrick More Date: Thu, 6 May 2021 12:07:53 -0700 Subject: [PATCH 8/9] Fixed short circuiting behavior when checking for value in Vec --- .../src/physical_plan/expressions/in_list.rs | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index a6f068f93657a..5046c68b82305 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -99,6 +99,28 @@ fn collect_scalar_value Option>( .collect::>() } +fn contains_sorted(data: &[E], item: &E)->bool{ + if data.is_empty(){ + return false; + } + //At this point slice is not empty so it will never panic + let lowest = data.first().unwrap(); + let highest = data.last().unwrap(); + + if item < lowest || highest < item{ + return false + } + + for elem in data.iter(){ + if *elem == *item{ + return true; + }else if *elem > *item{ + return false; + } + } + false +} + #[allow(clippy::unnecessary_wraps)] fn contains_ord_types_it( it: I, @@ -112,19 +134,14 @@ where O: Ord + Hash + Eq, { let num_values = list_values.len(); - let contains_arr = if num_values <= linear_search_bound { + let contains_arr = if num_values == 0{ + compute_contains_array(it, negated, contains_null, |val| { + val.map(|_x| false) + }) + } else if num_values <= linear_search_bound { list_values.sort_unstable(); compute_contains_array(it, negated, contains_null, |val| { - val.map(|x| { - let out_of_bounds = match (list_values.first(), list_values.last()) { - (Some(lowest), Some(highest)) => *lowest > x || *highest < x, - _ => false, - }; - if out_of_bounds { - return false; - } - list_values.contains(&x) - }) + val.map(|x| contains_sorted(list_values.as_slice(), &x)) }) } else { let set = list_values.into_iter().collect::>(); // HashSet::with_capacity(values.len()); @@ -292,10 +309,15 @@ impl InListExpr { // Possibly based off of the number of characters the values in the list have combined with the total length of the list? const LINEAR_SEARCH_UPPER_BOUND: usize = 8; let contains_arr = match values.len() { - 0..=LINEAR_SEARCH_UPPER_BOUND => { + 0 =>{ + compute_contains_array(array.iter(), self.negated, contains_null, |x| { + x.map(|_x| false) + }) + } + 1..=LINEAR_SEARCH_UPPER_BOUND => { values.sort_unstable(); compute_contains_array(array.iter(), self.negated, contains_null, |x| { - x.map(|x| values.contains(&x)) + x.map(|x| contains_sorted(values.as_slice(), &x)) }) } _ => { From c052c14a5cfb1772f253d7e2dbfa58f08213d225 Mon Sep 17 00:00:00 2001 From: Patrick More Date: Thu, 6 May 2021 12:13:51 -0700 Subject: [PATCH 9/9] Fixed clippy warnings and ran cargo fmt --- .../src/physical_plan/expressions/in_list.rs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/in_list.rs b/datafusion/src/physical_plan/expressions/in_list.rs index 5046c68b82305..cf8ef2d26af6a 100644 --- a/datafusion/src/physical_plan/expressions/in_list.rs +++ b/datafusion/src/physical_plan/expressions/in_list.rs @@ -99,28 +99,32 @@ fn collect_scalar_value Option>( .collect::>() } -fn contains_sorted(data: &[E], item: &E)->bool{ - if data.is_empty(){ +#[allow(clippy::comparison_chain)] +//Allow comparision chain because sometimes the match solution can be slower due to the +//cmp call not getting inlined. +fn contains_sorted(data: &[E], item: &E) -> bool { + if data.is_empty() { return false; } //At this point slice is not empty so it will never panic let lowest = data.first().unwrap(); let highest = data.last().unwrap(); - if item < lowest || highest < item{ - return false + if item < lowest || highest < item { + return false; } - for elem in data.iter(){ - if *elem == *item{ + for elem in data.iter() { + if *elem == *item { return true; - }else if *elem > *item{ + } else if *elem > *item { return false; } } false } +//Allow unnecessary_wraps to avoid wrapping with result every time this is invoked since this is a fairly specialized function #[allow(clippy::unnecessary_wraps)] fn contains_ord_types_it( it: I, @@ -134,14 +138,12 @@ where O: Ord + Hash + Eq, { let num_values = list_values.len(); - let contains_arr = if num_values == 0{ - compute_contains_array(it, negated, contains_null, |val| { - val.map(|_x| false) - }) + let contains_arr = if num_values == 0 { + compute_contains_array(it, negated, contains_null, |val| val.map(|_x| false)) } else if num_values <= linear_search_bound { list_values.sort_unstable(); compute_contains_array(it, negated, contains_null, |val| { - val.map(|x| contains_sorted(list_values.as_slice(), &x)) + val.map(|x| contains_sorted(list_values.as_slice(), &x)) }) } else { let set = list_values.into_iter().collect::>(); // HashSet::with_capacity(values.len()); @@ -309,11 +311,9 @@ impl InListExpr { // Possibly based off of the number of characters the values in the list have combined with the total length of the list? const LINEAR_SEARCH_UPPER_BOUND: usize = 8; let contains_arr = match values.len() { - 0 =>{ - compute_contains_array(array.iter(), self.negated, contains_null, |x| { - x.map(|_x| false) - }) - } + 0 => compute_contains_array(array.iter(), self.negated, contains_null, |x| { + x.map(|_x| false) + }), 1..=LINEAR_SEARCH_UPPER_BOUND => { values.sort_unstable(); compute_contains_array(array.iter(), self.negated, contains_null, |x| {