diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 404e2a03501d..3701c7d60228 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -414,6 +414,7 @@ arrow_add_object_library(ARROW_ARRAY array/data.cc array/diff.cc array/statistics.cc + array/statistics_option.cc array/util.cc array/validate.cc) @@ -668,6 +669,7 @@ endif() set(ARROW_TESTING_SRCS io/test_common.cc + array/statistics_test_util.cc ipc/test_common.cc testing/fixed_width_test_util.cc testing/generator.cc diff --git a/cpp/src/arrow/array/statistics.cc b/cpp/src/arrow/array/statistics.cc index b661c9fbaffe..e952d01afb4a 100644 --- a/cpp/src/arrow/array/statistics.cc +++ b/cpp/src/arrow/array/statistics.cc @@ -15,7 +15,404 @@ // specific language governing permissions and limitations // under the License. -// This empty .cc file is for embedding not inlined symbols in -// arrow::ArrayStatistics into libarrow. - #include "arrow/array/statistics.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array/array_base.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_dict.h" +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_union.h" +#include "arrow/array/data.h" +#include "arrow/c/abi.h" +#include "arrow/memory_pool.h" +#include "arrow/result.h" +#include "arrow/scalar.h" +#include "arrow/status.h" +#include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/util/macros.h" +#include "arrow/visit_type_inline.h" + +namespace arrow { + +Result> ArrayStatistics::ValueToArrowType( + const std::optional& value, + const std::shared_ptr& array_type) { + if (!value.has_value()) { + return null(); + } + + struct Visitor { + const std::shared_ptr& array_type; + + Result> operator()(const bool&) { return boolean(); } + Result> operator()(const int64_t&) { return int64(); } + Result> operator()(const uint64_t&) { return uint64(); } + Result> operator()(const double&) { return float64(); } + Result> operator()(const std::shared_ptr& value) { + return value->type; + } + Result> operator()(const std::string&) { + switch (array_type->id()) { + case Type::STRING: + case Type::BINARY: + case Type::FIXED_SIZE_BINARY: + case Type::LARGE_STRING: + case Type::LARGE_BINARY: + case Type::STRING_VIEW: + case Type::BINARY_VIEW: + return array_type; + default: + return Status::Invalid("type should be specified for string"); + } + } + } visitor{array_type}; + return std::visit(visitor, value.value()); +} +namespace { +bool ValueTypeEquality(const std::optional& first, + const std::optional& second, + const EqualOptions& options) { + if (first == second) { + return true; + } + if (!first || !second) { + return false; + } + + return std::visit( + [&options](auto& v1, auto& v2) { + if constexpr (std::is_same_v, + std::shared_ptr> && + std::is_same_v, + std::shared_ptr>) { + if (!v1 || !v2) { + // both empty shared_ptr case is handled in std::optional and return true + return false; + } + return v1->Equals(*v2, options); + } + return false; + }, + first.value(), second.value()); +} +} // namespace +bool ArrayStatistics::Equals(const ArrayStatistics& other, + const EqualOptions& options) const { + return null_count == other.null_count && distinct_count == other.distinct_count && + is_min_exact == other.is_min_exact && is_max_exact == other.is_max_exact && + ValueTypeEquality(this->max, other.max, options) && + ValueTypeEquality(this->min, other.min, options); +} +namespace internal { +// Perform a depth-first search (DFS) on the given array data to extract +// all nested array data up to the specified maximum nesting depth. +Result ExtractColumnsToArrayData( + const std::shared_ptr& array_data, const int32_t& max_nesting_depth) { + ArrayDataVector traverse; + ArrayDataVector result; + + int32_t remaining_nested_depth = max_nesting_depth; + if (remaining_nested_depth <= 0) { + return Status::Invalid("Max recursion depth reached"); + } + result.push_back(array_data); + if (!array_data->child_data.empty()) { + // depth delimiter + traverse.emplace_back(nullptr); + traverse.insert(traverse.end(), array_data->child_data.crbegin(), + array_data->child_data.crend()); + remaining_nested_depth--; + } + + while (!traverse.empty()) { + if (ARROW_PREDICT_FALSE(remaining_nested_depth <= 0)) { + return Status::Invalid("Max recursion depth reached"); + } + + auto child_data = traverse.back(); + traverse.pop_back(); + // Check whether the traversal of a depth is finished or not. + if (!child_data) { + remaining_nested_depth++; + continue; + } + result.push_back(child_data); + if (!child_data->child_data.empty()) { + // depth delimiter + traverse.emplace_back(nullptr); + traverse.insert(traverse.end(), child_data->child_data.crbegin(), + child_data->child_data.crend()); + remaining_nested_depth--; + } + } + return result; +} +namespace { +struct EnumeratedStatistics { + int nth_statistics = 0; + bool start_new_column = false; + std::optional nth_column = std::nullopt; + const char* key = nullptr; + std::shared_ptr type = nullptr; + ArrayStatistics::ValueType value = false; +}; +using OnStatistics = + std::function; + +Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, + std::optional num_rows, OnStatistics on_statistics) { + // TODO implement Statistic Schema attribute in C++(GH-45639) + EnumeratedStatistics statistics; + statistics.nth_statistics = 0; + statistics.start_new_column = true; + statistics.nth_column = std::nullopt; + + statistics.key = ARROW_STATISTICS_KEY_ROW_COUNT_EXACT; + statistics.type = int64(); + statistics.value = ArrayStatistics::ValueType{int64_t{0}}; + if (num_rows.has_value()) { + // handle Record Batch + statistics.value = ArrayStatistics::ValueType{num_rows.value()}; + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } else if (!extracted_array_data.empty()) { + // handle parent array + statistics.value = + ArrayStatistics::ValueType{int64_t{extracted_array_data[0]->length}}; + statistics.nth_column = int32_t{0}; + RETURN_NOT_OK(on_statistics(statistics)); + } + + auto num_fields = static_cast(extracted_array_data.size()); + for (int64_t nth_column = 0; nth_column < num_fields; ++nth_column) { + const auto& type = extracted_array_data[nth_column]->type; + auto column_statistics = extracted_array_data[nth_column]->statistics; + if (!column_statistics) { + continue; + } + statistics.start_new_column = true; + statistics.nth_column = static_cast(nth_column); + if (nth_column == 0 && !num_rows.has_value()) { + // handle parent array + statistics.start_new_column = false; + } + + if (column_statistics->null_count.has_value()) { + statistics.nth_statistics++; + statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT; + statistics.type = int64(); + statistics.value = column_statistics->null_count.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + + if (column_statistics->distinct_count.has_value()) { + statistics.nth_statistics++; + statistics.key = ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT; + statistics.type = int64(); + statistics.value = column_statistics->distinct_count.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + + if (column_statistics->min.has_value()) { + statistics.nth_statistics++; + if (column_statistics->is_min_exact) { + statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_EXACT; + } else { + statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE; + } + ARROW_ASSIGN_OR_RAISE(statistics.type, column_statistics->MinArrowType(type)); + statistics.value = column_statistics->min.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + + if (column_statistics->max.has_value()) { + statistics.nth_statistics++; + if (column_statistics->is_max_exact) { + statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_EXACT; + } else { + statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE; + } + ARROW_ASSIGN_OR_RAISE(statistics.type, column_statistics->MaxArrowType(type)); + statistics.value = column_statistics->max.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + } + return Status::OK(); +} + +struct StringVisitorBuilder { + template ::BuilderType> + enable_if_t::value || + is_binary_view_like_type::value || + std::is_same_v, + Status> + Visit(const DataType&, ArrayBuilder* raw_builder, const std::string& value) { + auto* builder = static_cast(raw_builder); + return builder->Append(value); + } + Status Visit(const DataType& type, ArrayBuilder* raw_builder, + const std::string& value) { + return Status::Invalid("The type is ", type.ToString()); + } +}; +} // namespace + +Result> MakeStatisticsArray( + MemoryPool* memory_pool, const ArrayDataVector& extracted_array_data_vector, + std::optional num_rows) { + // Statistics schema: + // struct< + // column: int32, + // statistics: map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + // > + + // Statistics schema doesn't define static dense union type for + // values. Each statistics schema have a dense union type that has + // needled value types. The following block collects these types. + + std::vector> values_types; + std::vector values_type_indexes; + RETURN_NOT_OK(EnumerateStatistics( + extracted_array_data_vector, num_rows, [&](const EnumeratedStatistics& statistics) { + int8_t i = 0; + for (const auto& field : values_types) { + if (field->type()->Equals(statistics.type)) { + break; + } + i++; + } + if (i == static_cast(values_types.size())) { + values_types.push_back(field(statistics.type->name(), statistics.type)); + } + values_type_indexes.push_back(i); + return Status::OK(); + })); + + // statistics.key: dictionary + auto keys_type = dictionary(int32(), utf8(), false); + // statistics.items: dense_union<...all needed types...> + auto values_type = dense_union(values_types); + // struct< + // column: int32, + // statistics: map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + // > + auto statistics_type = + struct_({field("column", int32()), + field("statistics", map(keys_type, values_type, false))}); + + std::vector> field_builders; + // columns: int32 + auto columns_builder = std::make_shared(memory_pool); + field_builders.push_back(std::static_pointer_cast(columns_builder)); + // statistics.key: dictionary + auto keys_builder = std::make_shared(); + // statistics.items: dense_union<...all needed types...> + std::vector> values_builders; + for (const auto& values_type : values_types) { + std::unique_ptr values_builder; + RETURN_NOT_OK(MakeBuilder(memory_pool, values_type->type(), &values_builder)); + values_builders.push_back(std::shared_ptr(std::move(values_builder))); + } + auto items_builder = std::make_shared( + memory_pool, std::move(values_builders), values_type); + // statistics: + // map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + auto values_builder = std::make_shared( + memory_pool, std::static_pointer_cast(keys_builder), + std::static_pointer_cast(items_builder)); + field_builders.push_back(std::static_pointer_cast(values_builder)); + // struct< + // column: int32, + // statistics: map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + // > + StructBuilder builder(statistics_type, memory_pool, std::move(field_builders)); + + // Append statistics. + RETURN_NOT_OK(EnumerateStatistics( + extracted_array_data_vector, num_rows, [&](const EnumeratedStatistics& statistics) { + if (statistics.start_new_column) { + RETURN_NOT_OK(builder.Append()); + if (statistics.nth_column.has_value()) { + RETURN_NOT_OK(columns_builder->Append(statistics.nth_column.value())); + } else { + RETURN_NOT_OK(columns_builder->AppendNull()); + } + RETURN_NOT_OK(values_builder->Append()); + } + RETURN_NOT_OK(keys_builder->Append(statistics.key, + static_cast(strlen(statistics.key)))); + const auto values_type_index = values_type_indexes[statistics.nth_statistics]; + RETURN_NOT_OK(items_builder->Append(values_type_index)); + struct Visitor { + ArrayBuilder* builder; + DataType* type; + Status operator()(const bool& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const int64_t& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const uint64_t& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const double& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const std::string& value) { + auto string_visitor_builder = StringVisitorBuilder(); + return VisitTypeInline(*type, &string_visitor_builder, builder, value); + } + Status operator()(const std::shared_ptr& value) { + return builder->AppendScalar(*value); + } + } visitor; + visitor.builder = values_builders[values_type_index].get(); + visitor.type = statistics.type.get(); + RETURN_NOT_OK(std::visit(visitor, statistics.value)); + return Status::OK(); + })); + + return builder.Finish(); +} + +} // namespace internal +} // namespace arrow diff --git a/cpp/src/arrow/array/statistics.h b/cpp/src/arrow/array/statistics.h index 6ccd2f4766e6..75c6a18070bc 100644 --- a/cpp/src/arrow/array/statistics.h +++ b/cpp/src/arrow/array/statistics.h @@ -18,11 +18,14 @@ #pragma once #include +#include #include #include #include -#include "arrow/type.h" +#include "arrow/compare.h" +#include "arrow/result.h" +#include "arrow/type_fwd.h" #include "arrow/util/visibility.h" namespace arrow { @@ -37,37 +40,11 @@ struct ARROW_EXPORT ArrayStatistics { /// \brief The type for maximum and minimum values. If the target /// value exists, one of them is used. `std::nullopt` is used /// otherwise. - using ValueType = std::variant; + using ValueType = + std::variant>; - static const std::shared_ptr& ValueToArrowType( - const std::optional& value, - const std::shared_ptr& array_type) { - if (!value.has_value()) { - return null(); - } - - struct Visitor { - const std::shared_ptr& array_type; - - const std::shared_ptr& operator()(const bool&) { return boolean(); } - const std::shared_ptr& operator()(const int64_t&) { return int64(); } - const std::shared_ptr& operator()(const uint64_t&) { return uint64(); } - const std::shared_ptr& operator()(const double&) { return float64(); } - const std::shared_ptr& operator()(const std::string&) { - switch (array_type->id()) { - case Type::STRING: - case Type::BINARY: - case Type::FIXED_SIZE_BINARY: - case Type::LARGE_STRING: - case Type::LARGE_BINARY: - return array_type; - default: - return utf8(); - } - } - } visitor{array_type}; - return std::visit(visitor, value.value()); - } + static Result> ValueToArrowType( + const std::optional& value, const std::shared_ptr& array_type); /// \brief The number of null values, may not be set std::optional null_count = std::nullopt; @@ -92,7 +69,7 @@ struct ARROW_EXPORT ArrayStatistics { /// \return \ref arrow::null if the minimum value is `std::nullopt`, /// Arrow type based on \ref ValueType of the \ref min /// otherwise. - const std::shared_ptr& MinArrowType( + Result> MinArrowType( const std::shared_ptr& array_type) { return ValueToArrowType(min, array_type); } @@ -117,7 +94,7 @@ struct ARROW_EXPORT ArrayStatistics { /// \return \ref arrow::null if the maximum value is `std::nullopt`, /// Arrow type based on \ref ValueType of the \ref max /// otherwise. - const std::shared_ptr& MaxArrowType( + Result> MaxArrowType( const std::shared_ptr& array_type) { return ValueToArrowType(max, array_type); } @@ -126,11 +103,13 @@ struct ARROW_EXPORT ArrayStatistics { bool is_max_exact = false; /// \brief Check two statistics for equality - bool Equals(const ArrayStatistics& other) const { - return null_count == other.null_count && distinct_count == other.distinct_count && - min == other.min && is_min_exact == other.is_min_exact && max == other.max && - is_max_exact == other.is_max_exact; - } + /// + /// \param[in] other the ArrayStatistics to compare with + /// \param[in] options the options for equality comparisons of Scalars + /// + /// \return true if statistics are equal + bool Equals(const ArrayStatistics& other, + const EqualOptions& options = EqualOptions::Defaults()) const; /// \brief Check two statistics for equality bool operator==(const ArrayStatistics& other) const { return Equals(other); } @@ -139,4 +118,36 @@ struct ARROW_EXPORT ArrayStatistics { bool operator!=(const ArrayStatistics& other) const { return !Equals(other); } }; +namespace internal { +/// \brief Extract all nested arrays from an array as \ref ArrayDataVector. +/// +/// Returns all nested arrays within the given array, up to the specified +/// maximum nesting depth, as a vector of ArrayData. +/// +/// \param[in] array_data The input array_data from which nested array_data will be +/// extracted. +/// \param[in] max_nesting_depth The maximum depth of nested arrays to extract. +/// +/// \return A vector of \ref ArrayData . +ARROW_EXPORT +Result ExtractColumnsToArrayData( + const std::shared_ptr& array_data, const int32_t& max_nesting_depth); +/// \brief Collect Statistics From ArrayDataVector and turns into an Array +/// +/// \param[in] memory_pool the memory pool to allocate memory from +/// \param extracted_array_data_vector The input ArrayData from which statistics array is +/// extracted +/// +/// \param num_rows The input specifies the number of rows an object(which uses for +/// record batch), if it is set to std::nullopt(which uses for array), +/// the number of rows is deduced from the +/// length of the first array +/// +/// \return A statistics array. +ARROW_EXPORT +Result> MakeStatisticsArray( + MemoryPool* memory_pool, const ArrayDataVector& extracted_array_data_vector, + std::optional num_rows = std::nullopt); +} // namespace internal + } // namespace arrow diff --git a/cpp/src/arrow/array/statistics_option.cc b/cpp/src/arrow/array/statistics_option.cc new file mode 100644 index 000000000000..f6bd5b5697c9 --- /dev/null +++ b/cpp/src/arrow/array/statistics_option.cc @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/array/statistics_option.h" diff --git a/cpp/src/arrow/array/statistics_option.h b/cpp/src/arrow/array/statistics_option.h new file mode 100644 index 000000000000..f117f9c3fe41 --- /dev/null +++ b/cpp/src/arrow/array/statistics_option.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +// ARROW-109: We set this number arbitrarily to help catch user mistakes. For +// deeply nested schemas, it is expected the user will indicate explicitly the +// maximum allowed recursion depth +constexpr int32_t kMaxNestingDepth = 64; + +/// \brief Options for creating statistics array +struct ARROW_EXPORT StatisticsArrayTOptions { + /// \brief The memory pool to use for allocations made during creating a statistics + /// arrays + MemoryPool* memory_pool = default_memory_pool(); + + /// \brief The maximum permitted schema nesting depth. + int32_t max_recursion_depth = kMaxNestingDepth; + + static StatisticsArrayTOptions Default() { return {}; } +}; +} // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test.cc b/cpp/src/arrow/array/statistics_test.cc index cf15a5d38297..ba7e9ee4ae26 100644 --- a/cpp/src/arrow/array/statistics_test.cc +++ b/cpp/src/arrow/array/statistics_test.cc @@ -14,10 +14,14 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#include #include #include "arrow/array/statistics.h" +#include "arrow/compare.h" +#include "arrow/scalar.h" +#include "arrow/testing/gtest_util.h" namespace arrow { @@ -66,7 +70,6 @@ TEST(ArrayStatisticsTest, TestEquality) { ArrayStatistics statistics2; ASSERT_EQ(statistics1, statistics2); - statistics1.null_count = 29; ASSERT_NE(statistics1, statistics2); statistics2.null_count = 29; @@ -91,11 +94,45 @@ TEST(ArrayStatisticsTest, TestEquality) { ASSERT_NE(statistics1, statistics2); statistics2.max = static_cast(-29); ASSERT_EQ(statistics1, statistics2); + statistics2.max = static_cast(2); + ASSERT_NE(statistics1, statistics2); + + statistics1.max = std::nullopt; + statistics2.max = std::nullopt; + // the state of both of them are std::nullopt + ASSERT_EQ(statistics1.max, statistics2.max); + // the state of one of them is std::nullopt + statistics1.max = std::shared_ptr(); + ASSERT_NE(statistics1, statistics2); + // the state of both of them are empty shared_ptr + statistics2.max = std::shared_ptr(); + ASSERT_EQ(statistics1, statistics2); + ASSERT_OK_AND_ASSIGN(statistics1.max, MakeScalar(int64(), 5)); + // the state of one of them is empty shared_ptr + ASSERT_NE(statistics1, statistics2); + // the state of one of them has different type + statistics2.max = static_cast(10); + ASSERT_NE(statistics1.max, statistics2.max); + ASSERT_OK_AND_ASSIGN(statistics2.max, MakeScalar(int64(), 5)); + // the state of both of them are equal + ASSERT_EQ(statistics1, statistics2); statistics1.is_max_exact = true; ASSERT_NE(statistics1, statistics2); statistics2.is_max_exact = true; ASSERT_EQ(statistics1, statistics2); } +TEST(ArrayStatisticsTest, EqualOptions) { + auto list_nan_1 = std::make_shared( + ArrayFromJSON(float32(), R"([0,null,NaN])"), list(float32())); + auto list_nan_2 = std::make_shared( + ArrayFromJSON(float32(), R"([0,null,NaN])"), list(float32())); + ArrayStatistics statistics1; + ArrayStatistics statistics2; + statistics1.max = list_nan_1; + statistics2.max = list_nan_2; + ASSERT_FALSE(statistics1.Equals(statistics2)); + ASSERT_TRUE(statistics1.Equals(statistics2, EqualOptions::Defaults().nans_equal(true))); +} } // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test_util.cc b/cpp/src/arrow/array/statistics_test_util.cc new file mode 100644 index 000000000000..695b21c076b7 --- /dev/null +++ b/cpp/src/arrow/array/statistics_test_util.cc @@ -0,0 +1,316 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/array/statistics_test_util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array/array_base.h" +#include "arrow/array/array_dict.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_base.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/array/data.h" +#include "arrow/array/statistics.h" +#include "arrow/result.h" +#include "arrow/scalar.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/visit_type_inline.h" + +namespace arrow { +namespace { +template ::value || + is_number_type::value>> +Result> BuildArray( + const std::vector::CType>& values) { + using BuilderType = typename TypeTraits::BuilderType; + BuilderType builder; + for (const auto& value : values) { + ARROW_RETURN_NOT_OK(builder.Append(value)); + } + return builder.Finish(); +} + +struct StringVisitorBuilder { + template ::BuilderType> + enable_if_t::value, Status> Visit( + const DataType&, ArrayBuilder* raw_builder, + const std::vector& strings) { + auto* builder = static_cast(raw_builder); + ARROW_RETURN_NOT_OK(builder->AppendValues(strings)); + return Status::OK(); + } + + template ::BuilderType> + enable_if_t::value || + std::is_same_v, + Status> + Visit(const DataType&, ArrayBuilder* raw_builder, + const std::vector& strings) { + auto* builder = static_cast(raw_builder); + for (const auto& string : strings) { + ARROW_RETURN_NOT_OK(builder->Append(string)); + } + return Status::OK(); + } + + Status Visit(const DataType& type, ArrayBuilder*, const std::vector&) { + return Status::Invalid("the type is ", type.ToString()); + } +}; + +Result> BuildArray(const std::vector& values, + const std::shared_ptr& type) { + std::unique_ptr builder; + ARROW_RETURN_NOT_OK(MakeBuilder(default_memory_pool(), type, &builder)); + StringVisitorBuilder visitor; + ARROW_RETURN_NOT_OK(VisitTypeInline(*type, &visitor, builder.get(), values)); + return builder->Finish(); +} + +template +std::vector StatisticsValuesToRawValues( + const std::vector& values) { + std::vector raw_values; + for (const auto& value : values) { + raw_values.push_back(std::get(value)); + } + return raw_values; +} + +template ::value>> +Result> BuildArray(const std::vector& values, + const std::shared_ptr& string_type) { + struct Builder { + const std::vector& values_; + const std::shared_ptr& type; + explicit Builder(const std::vector& values, + const std::shared_ptr& type) + : values_(values), type(type) {} + + Result> operator()(const bool&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const int64_t&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const uint64_t&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const double&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const std::string&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values, type); + } + Result> operator()(const std::shared_ptr& scalar) { + auto values = StatisticsValuesToRawValues>(values_); + ARROW_ASSIGN_OR_RAISE(auto builder, MakeBuilder(scalar->type)); + ARROW_RETURN_NOT_OK(builder->Reserve(values.size())); + ARROW_RETURN_NOT_OK(builder->AppendScalars(values)); + return builder->Finish(); + } + } builder(values, string_type); + return std::visit(builder, values[0]); +} +} // namespace + +Result> detail::MakeMockStatisticsArray( + const std::string& columns_json, + const std::vector>& nested_statistics_keys, + const std::vector>& nested_statistics_values, + const std::vector>& string_types) { + const auto& columns_type = int32(); + auto columns_array = ArrayFromJSON(columns_type, columns_json); + const auto n_columns = columns_array->length(); + + // nested_statistics_keys: + // { + // {"ARROW:row_count:exact", "ARROW:null_count:exact"}, + // {"ARROW:max_value:exact"}, + // {"ARROW:max_value:exact", "ARROW:distinct_count:exact"}, + // } + // nested_statistics_values: + // { + // {int64_t{29}, int64_t{1}}, + // {double{2.9}}, + // {double{-2.9}, int64_t{2}}, + // } + // -> + // keys_dictionary: + // { + // "ARROW:row_count:exact", + // "ARROW:null_count:exact", + // "ARROW:max_value:exact", + // "ARROW:distinct_count:exact", + // } + // keys_indices: {0, 1, 2, 2, 3} + // values_types: {int64(), float64()} + // values_type_codes: {0, 1} + // values_values[0]: {int64_t{29}, int64_t{1}, int64_t{2}} + // values_values[1]: {double{2.9}, double{-2.9}} + // values_value_type_ids: {0, 0, 1, 1, 0} + // values_value_offsets: {0, 1, 0, 1, 2} + // statistics_offsets: {0, 2, 3, 5, 5} + std::vector keys_dictionary; + std::vector keys_indices; + std::vector> values_types; + std::vector values_type_codes; + std::vector> values_values; + std::vector values_value_type_ids; + std::vector values_value_offsets; + std::vector statistics_offsets; + + int32_t offset = 0; + std::vector values_value_offset_counters; + for (size_t i = 0; i < nested_statistics_keys.size(); ++i) { + const auto& statistics_keys = nested_statistics_keys[i]; + const auto& statistics_values = nested_statistics_values[i]; + std::shared_ptr string_type = utf8(); + if (!string_types.empty() && string_types[i]) { + string_type = string_types[i]; + } + + statistics_offsets.push_back(offset); + for (size_t j = 0; j < statistics_keys.size(); ++j) { + const auto& key = statistics_keys[j]; + const auto& value = statistics_values[j]; + ++offset; + + int32_t key_index = 0; + for (; key_index < static_cast(keys_dictionary.size()); ++key_index) { + if (keys_dictionary[key_index] == key) { + break; + } + } + if (key_index == static_cast(keys_dictionary.size())) { + keys_dictionary.push_back(key); + } + keys_indices.push_back(key_index); + + ARROW_ASSIGN_OR_RAISE(auto values_type, + ArrayStatistics::ValueToArrowType(value, string_type)); + int8_t values_type_code = 0; + for (; values_type_code < static_cast(values_types.size()); + ++values_type_code) { + if (values_types[values_type_code]->Equals(values_type)) { + break; + } + } + if (values_type_code == static_cast(values_types.size())) { + values_types.push_back(values_type); + values_type_codes.push_back(values_type_code); + values_values.emplace_back(); + values_value_offset_counters.push_back(0); + } + values_values[values_type_code].push_back(value); + values_value_type_ids.push_back(values_type_code); + values_value_offsets.push_back(values_value_offset_counters[values_type_code]++); + } + } + statistics_offsets.push_back(offset); + + auto keys_type = dictionary(int32(), utf8(), false); + std::vector> values_fields; + for (const auto& type : values_types) { + values_fields.push_back(field(type->name(), type)); + } + auto values_type = dense_union(values_fields); + auto statistics_type = map(keys_type, values_type, false); + auto struct_type = + struct_({field("column", columns_type), field("statistics", statistics_type)}); + + ARROW_ASSIGN_OR_RAISE(auto keys_indices_array, BuildArray(keys_indices)); + ARROW_ASSIGN_OR_RAISE(auto keys_dictionary_array, BuildArray(keys_dictionary, utf8())); + ARROW_ASSIGN_OR_RAISE( + auto keys_array, + DictionaryArray::FromArrays(keys_type, keys_indices_array, keys_dictionary_array)); + + std::vector> values_arrays; + for (uint32_t i = 0; i < values_values.size(); ++i) { + const auto& values = values_values[i]; + std::shared_ptr string_type = utf8(); + if (!string_types.empty() && string_types[i]) { + string_type = string_types[i]; + } + + ARROW_ASSIGN_OR_RAISE(auto values_array, + BuildArray(values, string_type)); + values_arrays.push_back(values_array); + } + ARROW_ASSIGN_OR_RAISE(auto values_value_type_ids_array, + BuildArray(values_value_type_ids)); + ARROW_ASSIGN_OR_RAISE(auto values_value_offsets_array, + BuildArray(values_value_offsets)); + auto values_array = std::make_shared( + values_type, values_value_offsets_array->length(), values_arrays, + values_value_type_ids_array->data()->buffers[1], + values_value_offsets_array->data()->buffers[1]); + ARROW_ASSIGN_OR_RAISE(auto statistics_offsets_array, + BuildArray(statistics_offsets)); + ARROW_ASSIGN_OR_RAISE(auto statistics_array, + MapArray::FromArrays(statistics_type, statistics_offsets_array, + keys_array, values_array)); + std::vector> struct_arrays = {std::move(columns_array), + std::move(statistics_array)}; + return std::make_shared(struct_type, n_columns, struct_arrays); +} + +Result> MakeNestedStruct(const int32_t depth) { + // each struct occupy two depth + if (depth < 2) { + return Status::Invalid("depth must be greater thant 1"); + } + auto parent_struct_type = + struct_({field("a", int64()), field("b", int32()), field("c", int16())}); + auto parent_struct = + ArrayFromJSON(parent_struct_type, + R"([{"a":1,"b":4,"c":7},{"a":2,"b":5,"c":8},{"a":3,"b":6,"c":9}])"); + auto child_struct = + ArrayFromJSON(parent_struct_type, + R"([{"a":1,"b":4,"c":7},{"a":2,"b":5,"c":8},{"a":3,"b":6,"c":9}])"); + + for (int i = 0; i < depth - 2; ++i) { + ARROW_ASSIGN_OR_RAISE( + parent_struct, StructArray::Make({parent_struct, child_struct}, + {field("struct", parent_struct->type()), + field("child_struct", child_struct->type())})); + } + return parent_struct; +} +} // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test_util.h b/cpp/src/arrow/array/statistics_test_util.h new file mode 100644 index 000000000000..b17eccd5ce77 --- /dev/null +++ b/cpp/src/arrow/array/statistics_test_util.h @@ -0,0 +1,78 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/array/statistics.h" +#include "arrow/array/statistics_option.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/visibility.h" +#include "arrow/type_fwd.h" + +namespace arrow { + +namespace detail { +ARROW_TESTING_EXPORT Result> MakeMockStatisticsArray( + const std::string& columns_json, + const std::vector>& nested_statistics_keys, + const std::vector>& nested_statistics_values, + const std::vector>& string_type = {}); +} // namespace detail + +/// \brief Create a nested struct array with a specified depth. +/// +/// \param[in] depth The depth of the nested structs. Must be greater than 1. +/// +/// \return A parent_struct which has depth -1 child struct +ARROW_TESTING_EXPORT +Result> MakeNestedStruct(int32_t depth); + +template +class TestStatisticsArray { + public: + void CheckStatStatisticsArray( + const std::shared_ptr& value, const std::string& columns_json, + const std::vector>& nested_statistics_keys, + const std::vector>& + nested_statistics_values, + const std::vector>& string_type = {}) const { + ASSERT_OK_AND_ASSIGN(auto statistics_array, value->MakeStatisticsArray(options_)); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + detail::MakeMockStatisticsArray(columns_json, nested_statistics_keys, + nested_statistics_values, string_type)); + AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + } + + void CheckDepth(const std::shared_ptr& value, bool is_error) const { + if (!is_error) { + ASSERT_OK(value->MakeStatisticsArray(options_).status()); + } else { + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + value->MakeStatisticsArray(options_).status()); + } + } + + StatisticsArrayTOptions options_; +}; +} // namespace arrow diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index 9622fe8322cc..bd12a0dbcfd5 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -21,19 +21,17 @@ #include #include #include +#include #include #include #include #include #include "arrow/array.h" -#include "arrow/array/builder_binary.h" -#include "arrow/array/builder_dict.h" -#include "arrow/array/builder_nested.h" -#include "arrow/array/builder_union.h" #include "arrow/array/concatenate.h" +#include "arrow/array/statistics.h" +#include "arrow/array/statistics_option.h" #include "arrow/array/validate.h" -#include "arrow/c/abi.h" #include "arrow/pretty_print.h" #include "arrow/status.h" #include "arrow/table.h" @@ -476,220 +474,19 @@ Result> RecordBatch::ViewOrCopyTo( return Make(schema_, num_rows(), std::move(copied_columns)); } -namespace { -struct EnumeratedStatistics { - int nth_statistics = 0; - bool start_new_column = false; - std::optional nth_column = std::nullopt; - const char* key = nullptr; - std::shared_ptr type = nullptr; - ArrayStatistics::ValueType value = false; -}; -using OnStatistics = - std::function; -Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_statistics) { - EnumeratedStatistics statistics; - statistics.nth_statistics = 0; - statistics.start_new_column = true; - statistics.nth_column = std::nullopt; - - statistics.key = ARROW_STATISTICS_KEY_ROW_COUNT_EXACT; - statistics.type = int64(); - statistics.value = record_batch.num_rows(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - - const auto& schema = record_batch.schema(); - const auto num_fields = schema->num_fields(); - for (int nth_column = 0; nth_column < num_fields; ++nth_column) { - const auto& field = schema->field(nth_column); - auto column_statistics = record_batch.column(nth_column)->statistics(); - if (!column_statistics) { - continue; - } - - statistics.start_new_column = true; - statistics.nth_column = nth_column; - if (column_statistics->null_count.has_value()) { - statistics.nth_statistics++; - statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT; - statistics.type = int64(); - statistics.value = column_statistics->null_count.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - - if (column_statistics->distinct_count.has_value()) { - statistics.nth_statistics++; - statistics.key = ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT; - statistics.type = int64(); - statistics.value = column_statistics->distinct_count.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - - if (column_statistics->min.has_value()) { - statistics.nth_statistics++; - if (column_statistics->is_min_exact) { - statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_EXACT; - } else { - statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE; - } - statistics.type = column_statistics->MinArrowType(field->type()); - statistics.value = column_statistics->min.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - - if (column_statistics->max.has_value()) { - statistics.nth_statistics++; - if (column_statistics->is_max_exact) { - statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_EXACT; - } else { - statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE; - } - statistics.type = column_statistics->MaxArrowType(field->type()); - statistics.value = column_statistics->max.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - } - return Status::OK(); -} -} // namespace - Result> RecordBatch::MakeStatisticsArray( - MemoryPool* memory_pool) const { - // Statistics schema: - // struct< - // column: int32, - // statistics: map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - // > - - // Statistics schema doesn't define static dense union type for - // values. Each statistics schema have a dense union type that has - // needled value types. The following block collects these types. - std::vector> values_types; - std::vector values_type_indexes; - RETURN_NOT_OK(EnumerateStatistics(*this, [&](const EnumeratedStatistics& statistics) { - int8_t i = 0; - for (const auto& field : values_types) { - if (field->type()->id() == statistics.type->id()) { - break; - } - i++; - } - if (i == static_cast(values_types.size())) { - values_types.push_back(field(statistics.type->name(), statistics.type)); - } - values_type_indexes.push_back(i); - return Status::OK(); - })); - - // statistics.key: dictionary - auto keys_type = dictionary(int32(), utf8(), false); - // statistics.items: dense_union<...all needed types...> - auto values_type = dense_union(values_types); - // struct< - // column: int32, - // statistics: map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - // > - auto statistics_type = - struct_({field("column", int32()), - field("statistics", map(keys_type, values_type, false))}); - - std::vector> field_builders; - // columns: int32 - auto columns_builder = std::make_shared(memory_pool); - field_builders.push_back(std::static_pointer_cast(columns_builder)); - // statistics.key: dictionary - auto keys_builder = std::make_shared(); - // statistics.items: dense_union<...all needed types...> - std::vector> values_builders; - for (const auto& values_type : values_types) { - std::unique_ptr values_builder; - RETURN_NOT_OK(MakeBuilder(memory_pool, values_type->type(), &values_builder)); - values_builders.push_back(std::shared_ptr(std::move(values_builder))); - } - auto items_builder = std::make_shared( - memory_pool, std::move(values_builders), values_type); - // statistics: - // map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - auto values_builder = std::make_shared( - memory_pool, std::static_pointer_cast(keys_builder), - std::static_pointer_cast(items_builder)); - field_builders.push_back(std::static_pointer_cast(values_builder)); - // struct< - // column: int32, - // statistics: map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - // > - StructBuilder builder(statistics_type, memory_pool, std::move(field_builders)); - - // Append statistics. - RETURN_NOT_OK(EnumerateStatistics(*this, [&](const EnumeratedStatistics& statistics) { - if (statistics.start_new_column) { - RETURN_NOT_OK(builder.Append()); - if (statistics.nth_column.has_value()) { - RETURN_NOT_OK(columns_builder->Append(statistics.nth_column.value())); - } else { - RETURN_NOT_OK(columns_builder->AppendNull()); - } - RETURN_NOT_OK(values_builder->Append()); - } - RETURN_NOT_OK(keys_builder->Append(statistics.key, - static_cast(strlen(statistics.key)))); - const auto values_type_index = values_type_indexes[statistics.nth_statistics]; - RETURN_NOT_OK(items_builder->Append(values_type_index)); - struct Visitor { - ArrayBuilder* builder; - - Status operator()(const bool& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const int64_t& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const uint64_t& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const double& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const std::string& value) { - return static_cast(builder)->Append( - value.data(), static_cast(value.size())); - } - } visitor; - visitor.builder = values_builders[values_type_index].get(); - RETURN_NOT_OK(std::visit(visitor, statistics.value)); - return Status::OK(); - })); - - return builder.Finish(); + const StatisticsArrayTOptions& options) const { + ArrayDataVector collected_columns; + for (int i = 0; i < this->num_columns(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto extracted_columns, + internal::ExtractColumnsToArrayData( + column(i)->data(), options.max_recursion_depth)); + collected_columns.insert(collected_columns.end(), + std::make_move_iterator(extracted_columns.begin()), + std::make_move_iterator(extracted_columns.end())); + } + return internal::MakeStatisticsArray(options.memory_pool, std::move(collected_columns), + this->num_rows()); } Status RecordBatch::Validate() const { diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h index 2b9fe0bda890..6b6328208ae1 100644 --- a/cpp/src/arrow/record_batch.h +++ b/cpp/src/arrow/record_batch.h @@ -22,6 +22,7 @@ #include #include +#include "arrow/array/statistics_option.h" #include "arrow/compare.h" #include "arrow/device.h" #include "arrow/result.h" @@ -284,15 +285,16 @@ class ARROW_EXPORT RecordBatch { /// \brief Create a statistics array of this record batch /// - /// The created array follows the C data interface statistics + /// The created array follows the Statistics Schema /// specification. See /// https://arrow.apache.org/docs/format/StatisticsSchema.html /// for details. /// - /// \param[in] pool the memory pool to allocate memory from + /// \param[in] options Options for creating a statistics array + /// /// \return the statistics array of this record batch Result> MakeStatisticsArray( - MemoryPool* pool = default_memory_pool()) const; + const StatisticsArrayTOptions& options = StatisticsArrayTOptions::Default()) const; protected: RecordBatch(const std::shared_ptr& schema, int64_t num_rows); diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index a659f8798e5c..6ad828ad3b48 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -24,15 +24,20 @@ #include #include #include +#include #include #include #include +#include #include #include "arrow/array/array_base.h" #include "arrow/array/array_dict.h" #include "arrow/array/array_nested.h" #include "arrow/array/data.h" +#include "arrow/array/statistics.h" +#include "arrow/array/statistics_option.h" +#include "arrow/array/statistics_test_util.h" #include "arrow/array/util.h" #include "arrow/c/abi.h" #include "arrow/chunked_array.h" @@ -1019,223 +1024,103 @@ TEST_F(TestRecordBatch, ToTensorUnsupportedMixedFloat16) { NotImplemented, "NotImplemented: Casting from or to halffloat is not supported.", batch1->ToTensor()); } - -namespace { -template ::value || - is_number_type::value>> -Result> BuildArray( - const std::vector::CType>& values) { - using BuilderType = typename TypeTraits::BuilderType; - BuilderType builder; - for (const auto& value : values) { - ARROW_RETURN_NOT_OK(builder.Append(value)); - } - return builder.Finish(); -} - -template > -Result> BuildArray(const std::vector& values) { - using BuilderType = typename TypeTraits::BuilderType; - BuilderType builder; - for (const auto& value : values) { - ARROW_RETURN_NOT_OK(builder.Append(value)); - } - return builder.Finish(); +class TestRecordBatchStatisticsArray : public TestStatisticsArray, + public ::testing::Test {}; + +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNesting) { + auto schema = ::arrow::schema({field("f0", int32())}); + ASSERT_OK_AND_ASSIGN(auto batch, RecordBatch::MakeEmpty(schema)); + options_.max_recursion_depth = 1; + CheckDepth(batch, false); + options_.max_recursion_depth = 0; + CheckDepth(batch, true); + + schema = ::arrow::schema({field("f0", int64()), field("f1", int64())}); + batch = RecordBatchFromJSON( + schema, R"([{"f0": 1, "f1": 4},{"f0": 2, "f1": 5},{"f0": 3, "f1": 6}])"); + options_.max_recursion_depth = 1; + CheckDepth(batch, false); + options_.max_recursion_depth = 0; + CheckDepth(batch, true); + + options_.max_recursion_depth = 2; + ASSERT_OK_AND_ASSIGN(auto multi_children_struct, + MakeNestedStruct(options_.max_recursion_depth)); + schema = + ::arrow::schema({field("multi_children_struct", multi_children_struct->type())}); + batch = + RecordBatch::Make(schema, multi_children_struct->length(), {multi_children_struct}); + options_.max_recursion_depth = 1; + CheckDepth(batch, true); + options_.max_recursion_depth = 2; + CheckDepth(batch, false); + + auto array_a = ArrayFromJSON(int64(), R"([1,2,3])"); + schema = ::arrow::schema({field("a", int64()), field("multi_children_struct", + multi_children_struct->type())}); + batch = RecordBatch::Make(schema, 3, {array_a, multi_children_struct}); + options_.max_recursion_depth = 1; + CheckDepth(batch, true); + options_.max_recursion_depth = 2; + CheckDepth(batch, false); } -template -std::vector StatisticsValuesToRawValues( - const std::vector& values) { - std::vector raw_values; - for (const auto& value : values) { - raw_values.push_back(std::get(value)); - } - return raw_values; -} - -template ::value>> -Result> BuildArray(const std::vector& values) { - struct Builder { - const std::vector& values_; - explicit Builder(const std::vector& values) - : values_(values) {} - - Result> operator()(const bool&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const int64_t&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const uint64_t&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const double&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const std::string&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - } builder(values); - return std::visit(builder, values[0]); -} - -Result> MakeStatisticsArray( - const std::string& columns_json, - const std::vector>& nested_statistics_keys, - const std::vector>& - nested_statistics_values) { - auto columns_type = int32(); - auto columns_array = ArrayFromJSON(columns_type, columns_json); - const auto n_columns = columns_array->length(); - - // nested_statistics_keys: - // { - // {"ARROW:row_count:exact", "ARROW:null_count:exact"}, - // {"ARROW:max_value:exact"}, - // {"ARROW:max_value:exact", "ARROW:distinct_count:exact"}, - // } - // nested_statistics_values: - // { - // {int64_t{29}, int64_t{1}}, - // {double{2.9}}, - // {double{-2.9}, int64_t{2}}, - // } - // -> - // keys_dictionary: - // { - // "ARROW:row_count:exact", - // "ARROW:null_count:exact", - // "ARROW:max_value:exact", - // "ARROW:distinct_count:exact", - // } - // keys_indices: {0, 1, 2, 2, 3} - // values_types: {int64(), float64()} - // values_type_codes: {0, 1} - // values_values[0]: {int64_t{29}, int64_t{1}, int64_t{2}} - // values_values[1]: {double{2.9}, double{-2.9}} - // values_value_type_ids: {0, 0, 1, 1, 0} - // values_value_offsets: {0, 1, 0, 1, 2} - // statistics_offsets: {0, 2, 3, 5, 5} - std::vector keys_dictionary; - std::vector keys_indices; - std::vector> values_types; - std::vector values_type_codes; - std::vector> values_values; - std::vector values_value_type_ids; - std::vector values_value_offsets; - std::vector statistics_offsets; - - int32_t offset = 0; - std::vector values_value_offset_counters; - for (size_t i = 0; i < nested_statistics_keys.size(); ++i) { - const auto& statistics_keys = nested_statistics_keys[i]; - const auto& statistics_values = nested_statistics_values[i]; - statistics_offsets.push_back(offset); - for (size_t j = 0; j < statistics_keys.size(); ++j) { - const auto& key = statistics_keys[j]; - const auto& value = statistics_values[j]; - ++offset; - - int32_t key_index = 0; - for (; key_index < static_cast(keys_dictionary.size()); ++key_index) { - if (keys_dictionary[key_index] == key) { - break; - } - } - if (key_index == static_cast(keys_dictionary.size())) { - keys_dictionary.push_back(key); - } - keys_indices.push_back(key_index); - - auto values_type = ArrayStatistics::ValueToArrowType(value, arrow::null()); - int8_t values_type_code = 0; - for (; values_type_code < static_cast(values_types.size()); - ++values_type_code) { - if (values_types[values_type_code] == values_type) { - break; - } - } - if (values_type_code == static_cast(values_types.size())) { - values_types.push_back(values_type); - values_type_codes.push_back(values_type_code); - values_values.emplace_back(); - values_value_offset_counters.push_back(0); - } - values_values[values_type_code].push_back(value); - values_value_type_ids.push_back(values_type_code); - values_value_offsets.push_back(values_value_offset_counters[values_type_code]++); - } - } - statistics_offsets.push_back(offset); +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNestingStressTest) { + ASSERT_OK_AND_ASSIGN(auto deep_children_struct, MakeNestedStruct(100)); + auto schema = ::arrow::schema({field("struct", deep_children_struct->type())}); + auto batch = + RecordBatch::Make(schema, deep_children_struct->length(), {deep_children_struct}); + options_.max_recursion_depth = 100; + CheckDepth(batch, false); + options_.max_recursion_depth = 99; + CheckDepth(batch, true); + +#if !defined(ARROW_VALGRIND) + ASSERT_OK_AND_ASSIGN(deep_children_struct, MakeNestedStruct(5000)); + schema = ::arrow::schema({field("struct", deep_children_struct->type())}); + batch = + RecordBatch::Make(schema, deep_children_struct->length(), {deep_children_struct}); + options_.max_recursion_depth = 5000; + CheckDepth(batch, false); + options_.max_recursion_depth = 4999; + CheckDepth(batch, true); - auto keys_type = dictionary(int32(), utf8(), false); - std::vector> values_fields; - for (const auto& type : values_types) { - values_fields.push_back(field(type->name(), type)); - } - auto values_type = dense_union(values_fields); - auto statistics_type = map(keys_type, values_type, false); - auto struct_type = - struct_({field("column", columns_type), field("statistics", statistics_type)}); - - ARROW_ASSIGN_OR_RAISE(auto keys_indices_array, BuildArray(keys_indices)); - ARROW_ASSIGN_OR_RAISE(auto keys_dictionary_array, - BuildArray(keys_dictionary)); - ARROW_ASSIGN_OR_RAISE( - auto keys_array, - DictionaryArray::FromArrays(keys_type, keys_indices_array, keys_dictionary_array)); - - std::vector> values_arrays; - for (const auto& values : values_values) { - ARROW_ASSIGN_OR_RAISE(auto values_array, - BuildArray(values)); - values_arrays.push_back(values_array); - } - ARROW_ASSIGN_OR_RAISE(auto values_value_type_ids_array, - BuildArray(values_value_type_ids)); - ARROW_ASSIGN_OR_RAISE(auto values_value_offsets_array, - BuildArray(values_value_offsets)); - auto values_array = std::make_shared( - values_type, values_value_offsets_array->length(), values_arrays, - values_value_type_ids_array->data()->buffers[1], - values_value_offsets_array->data()->buffers[1]); - ARROW_ASSIGN_OR_RAISE(auto statistics_offsets_array, - BuildArray(statistics_offsets)); - ARROW_ASSIGN_OR_RAISE(auto statistics_array, - MapArray::FromArrays(statistics_type, statistics_offsets_array, - keys_array, values_array)); - std::vector> struct_arrays = {std::move(columns_array), - std::move(statistics_array)}; - return std::make_shared(struct_type, n_columns, struct_arrays); +#endif } -}; // namespace -TEST_F(TestRecordBatch, MakeStatisticsArrayRowCount) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayRowCount) { auto schema = ::arrow::schema({field("int32", int32())}); auto int32_array = ArrayFromJSON(int32(), "[1, null, -1]"); auto batch = RecordBatch::Make(schema, int32_array->length(), {int32_array}); + CheckStatStatisticsArray(batch, "[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + // I'm not sure whether the following states are valid + batch = RecordBatch::Make(schema, 1, {int32_array}); + CheckStatStatisticsArray(batch, "[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{1}}, + }}); + + batch = RecordBatch::Make(schema, 10, {int32_array}); + CheckStatStatisticsArray(batch, "[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{10}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayNullCount) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNullCount) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("int32", int32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1246,26 +1131,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNullCount) { auto batch = RecordBatch::Make(schema, int32_array->length(), {no_statistics_array, int32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{int64_t{1}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{int64_t{1}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayDistinctCount) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayDistinctCount) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("int32", int32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1277,28 +1158,24 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayDistinctCount) { auto batch = RecordBatch::Make(schema, int32_array->length(), {no_statistics_array, int32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{int64_t{1}}, - ArrayStatistics::ValueType{int64_t{2}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{int64_t{1}}, + ArrayStatistics::ValueType{int64_t{2}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMinExact) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMinExact) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("uint32", uint32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1310,26 +1187,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMinExact) { auto batch = RecordBatch::Make(schema, uint32_array->length(), {no_statistics_array, uint32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MIN_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{uint64_t{1}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MIN_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{uint64_t{1}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMinApproximate) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMinApproximate) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("int32", int32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1340,27 +1213,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMinApproximate) { auto batch = RecordBatch::Make(schema, int32_array->length(), {no_statistics_array, int32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{-1.0}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{-1.0}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMaxExact) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMaxExact) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("boolean", boolean())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1373,26 +1241,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMaxExact) { auto batch = RecordBatch::Make(schema, boolean_array->length(), {no_statistics_array, boolean_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{true}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{true}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMaxApproximate) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMaxApproximate) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("float64", float64())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1403,55 +1267,264 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMaxApproximate) { auto batch = RecordBatch::Make(schema, float64_array->length(), {no_statistics_array, float64_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{1.0}, + }}); +} - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{1.0}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayString) { + // Check each string type + auto CheckString = [&](const std::shared_ptr& type) { + auto schema = + ::arrow::schema({field("no-statistics", boolean()), field("string", type)}); + auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); + auto string_array_data = ArrayFromJSON(type, R"(["a", null, "c"])")->data()->Copy(); + string_array_data->statistics = std::make_shared(); + string_array_data->statistics->is_max_exact = true; + string_array_data->statistics->max = "c"; + auto string_array = MakeArray(std::move(string_array_data)); + auto batch = RecordBatch::Make(schema, string_array->length(), + {no_statistics_array, string_array}); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{"c"}, + }}, + {nullptr, type}); + }; + + CheckString(binary()); + CheckString(large_binary()); + CheckString(utf8()); + CheckString(large_utf8()); + CheckString(binary_view()); + CheckString(utf8_view()); + CheckString(fixed_size_binary(1)); + + // Check the combination of various string types + + auto f0 = ArrayFromJSON(utf8(), R"(["a", "b", "c"])"); + f0->data()->statistics = std::make_shared(); + f0->data()->statistics->max = "c"; + + auto f1 = ArrayFromJSON(int64(), R"([1, 2, 3])"); + + auto f2 = ArrayFromJSON(large_binary(), R"(["a", "b", "c"])"); + f2->data()->statistics = std::make_shared(); + f2->data()->statistics->max = "c"; + + auto schema = ::arrow::schema( + {field("f0", utf8()), field("f1", int64()), field("f2", utf8_view())}); + auto batch = RecordBatch::Make(schema, f0->length(), {f0, f1, f2}); + + CheckStatStatisticsArray(batch, "[null, 0, 2]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{"c"}, + }, + { + ArrayStatistics::ValueType{"c"}, + }}, + {nullptr, utf8(), large_binary()}); +} +// Schema: +// struct_a_b: struct {a: int64, b: int64} +// c: int64 +// d: int64 +// struct_e_f: struct {e: int32, f: int32} +// +// Data: +// [ +// {struct_a_b: {a: 1, b: 6 }, c: 11, d: 16, struct_e_f: {a: 21, b: 26}}, +// {struct_a_b: {a: 2, b: 7 }, c: 12, d: 17, struct_e_f: {a: 22, b: 27}}, +// {struct_a_b: {a: 3, b: 8 }, c: 13, d: 18, struct_e_f: {a: 23, b: 28}}, +// {struct_a_b: {a: 4, b: 9 }, c: 14, d: 19, struct_e_f: {a: 24, b: 29}}, +// {struct_a_b: {a: 5, b: 10}, c: 15, d: 20, struct_e_f: {a: 25, b: 30}} +// ] +// +// Statistics: +// batch: +// row_count_exact: 5 +// struct_a_b: +// null_count_exact: 0 +// max_approximate: {a: 5, b: 10} +// struct_a_b.a: +// min_approximate: 1 +// array_c: +// max_approximate: 15 +// struct_e_f.e: +// max_exact: 30 + +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNestedType) { + const auto struct_a_b_type = struct_({field("a", int64()), field("b", int64())}); + const auto struct_a_b = ArrayFromJSON( + struct_a_b_type, + R"([{"a": 1, "b": 6}, {"a": 2, "b": 7}, {"a": 3,"b": 8}, {"a": 4,"b": 9}, {"a": 5, "b": 10}])"); + const auto struct_a_b_statistics = std::make_shared(); + ASSERT_OK_AND_ASSIGN(struct_a_b_statistics->max, struct_a_b->GetScalar(4)); + struct_a_b_statistics->null_count = 0; + struct_a_b->data()->statistics = struct_a_b_statistics; + const auto struct_a_b_child_a_statistics = std::make_shared(); + struct_a_b_child_a_statistics->min = int64_t{1}; + struct_a_b->data()->child_data[0]->statistics = struct_a_b_child_a_statistics; + const auto array_c = ArrayFromJSON(int64(), R"([11, 12, 13, 14, 15])"); + array_c->data()->statistics = std::make_shared(); + array_c->data()->statistics->max = int64_t{15}; + const auto array_d = ArrayFromJSON(int64(), R"([16, 17, 18, 19, 20])"); + const auto struct_e_f_type = struct_({field("e", int32()), field("f", int32())}); + const auto struct_e_f = ArrayFromJSON( + struct_e_f_type, + R"([{"e": 21, "f": 26}, {"e": 22,"f": 27}, {"e": 23, "f": 28}, {"e": 24, "f": 29}, {"e": 25,"f": 30}])"); + const auto struct_e_f_child_e_statistics = struct_e_f->data()->child_data[0]; + struct_e_f_child_e_statistics->statistics = std::make_shared(); + struct_e_f_child_e_statistics->statistics->max = int64_t{30}; + struct_e_f_child_e_statistics->statistics->is_max_exact = true; + + const auto schema = + ::arrow::schema({field("struct_a_b", struct_a_b_type), field("c", int64()), + field("d", int64()), field("struct_e_f", struct_e_f->type())}); + const auto batch = + RecordBatch::Make(schema, 5, {struct_a_b, array_c, array_d, struct_e_f}); + + const auto expected_scalar = ScalarFromJSON(struct_a_b_type, R"([5 ,10])"); + + CheckStatStatisticsArray(batch, "[null , 0, 1, 3, 6]", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{int64_t{0}}, + ArrayStatistics::ValueType{expected_scalar}}, + {ArrayStatistics::ValueType{int64_t{1}}}, + {ArrayStatistics::ValueType{int64_t{15}}}, + {ArrayStatistics::ValueType{int64_t{30}}}}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayString) { - auto schema = - ::arrow::schema({field("no-statistics", boolean()), field("string", utf8())}); - auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); - auto string_array_data = ArrayFromJSON(utf8(), "[\"a\", null, \"c\"]")->data()->Copy(); - string_array_data->statistics = std::make_shared(); - string_array_data->statistics->is_max_exact = true; - string_array_data->statistics->max = "c"; - auto string_array = MakeArray(std::move(string_array_data)); - auto batch = RecordBatch::Make(schema, string_array->length(), - {no_statistics_array, string_array}); - - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{"c"}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); +// Schema +// struct_parent: {struct_nested_0: {a: int32, b: int32}, +// struct_nested_1: {c: int32, d: int32}, +// struct_nested_2: {a: int32, b: int32}} +// +// Data: +// [ +// struct_parent: {struct_nested_0: {a: 1, b: 6 }, struct_nested_1: {c: 1, d: 6 }, +// struct_nested_2: {a: 1, b: 6 }}, +// struct_parent: {struct_nested_0: {a: 2, b: 7 }, struct_nested_1: {c: 2, d: 7 }, +// struct_nested_2: {a: 2, b: 7 }}, +// struct_parent: {struct_nested_0: {a: 3, b: 8 }, struct_nested_1: {c: 3, d: 8 }, +// struct_nested_2: {a: 3, b: 8 }}, +// struct_parent: {struct_nested_0: {a: 4, b: 9 }, struct_nested_1: {c: 4, d: 9 }, +// struct_nested_2: {a: 4, b: 9 }}, +// struct_parent: {struct_nested_0: {a: 5, b: 10}, struct_nested_1: {c: 5, d: 10}} +// struct_nested_2: {a: 5, b: 10}} +// ] +// +// Statistics: +// batch: +// row_count_exact: 5 +// struct_parent: +// min_approximate: {struct_nested_0: {a: 1, b: 6 }, struct_nested_1: {c: 1, d: 6 }, +// struct_nested_2: {a: 1, b: 6 }}, +// struct_nested_0: +// max_approximate: {a:5, b: 10} +// struct_nested_1: +// max_exact: {c:5, d: 10} +// struct_nested_2: +// min_exact: {a:5, b: 4} +// +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNestedNestedType) { + const auto struct_nested_0_type = struct_({field("a", int32()), field("b", int32())}); + const auto struct_nested_0 = ArrayFromJSON( + struct_nested_0_type, + R"([{"a": 1,"b": 6}, {"a": 2, "b": 7}, {"a": 3, "b": 8}, {"a": 4, "b": 9}, {"a": 5,"b": 10}])"); + struct_nested_0->data()->statistics = std::make_shared(); + ASSERT_OK_AND_ASSIGN(struct_nested_0->data()->statistics->max, + struct_nested_0->GetScalar(4)); + const auto struct_nested_1_type = struct_({field("c", int32()), field("d", int32())}); + const auto struct_nested_1 = ArrayFromJSON( + struct_nested_1_type, + R"([{"c": 1, "d": 6}, {"c": 2, "d": 7}, {"c": 3, "d": 8}, {"c": 4, "d": 9}, {"c": 5,"d": 10}])"); + struct_nested_1->data()->statistics = std::make_shared(); + struct_nested_1->data()->statistics->is_max_exact = true; + ASSERT_OK_AND_ASSIGN(struct_nested_1->data()->statistics->max, + struct_nested_1->GetScalar(4)); + + auto struct_nested_2_type = struct_({field("a", int32()), field("b", int32())}); + const auto struct_nested_2 = ArrayFromJSON( + struct_nested_0_type, + R"([{"a": 1,"b": 6}, {"a": 2, "b": 7}, {"a": 3, "b": 8}, {"a": 4, "b": 9}, {"a": 5,"b": 10}])"); + struct_nested_2->data()->statistics = std::make_shared(); + struct_nested_2->data()->statistics->is_min_exact = true; + ASSERT_OK_AND_ASSIGN(struct_nested_2->data()->statistics->min, + struct_nested_2->GetScalar(0)); + + ASSERT_OK_AND_ASSIGN( + const auto struct_parant, + StructArray::Make({struct_nested_0, struct_nested_1, struct_nested_2}, + {field("struct_nested_0", struct_nested_0->type()), + field("struct_nested_1", struct_nested_1->type()), + field("struct_nested_2", struct_nested_2->type())})); + struct_parant->data()->statistics = std::make_shared(); + ASSERT_OK_AND_ASSIGN(struct_parant->data()->statistics->min, + struct_parant->GetScalar(0)); + + const auto schema = ::arrow::schema({field("struct", struct_parant->type())}); + const auto batch = RecordBatch::Make(schema, 5, {struct_parant}); + + const auto expected_scalar_parent = + ScalarFromJSON(struct_parant->type(), R"([[1 ,6], [1, 6], [1, 6]])"); + + const auto expected_scalar_nested_0 = + ScalarFromJSON(struct_nested_0_type, R"([5, 10])"); + + const auto expected_scalar_nested_1 = + ScalarFromJSON(struct_nested_1_type, R"([5, 10])"); + + const auto expected_scalar_nested_2 = ScalarFromJSON(struct_nested_2_type, R"([1, 6])"); + + CheckStatStatisticsArray(batch, R"([null, 0, 1, 4, 7])", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}, + {ARROW_STATISTICS_KEY_MIN_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{expected_scalar_parent}}, + {ArrayStatistics::ValueType{expected_scalar_nested_0}}, + {ArrayStatistics::ValueType{expected_scalar_nested_1}}, + {expected_scalar_nested_2}}); } template diff --git a/docs/source/cpp/api/array.rst b/docs/source/cpp/api/array.rst index b17d1957a8b6..12e07834705a 100644 --- a/docs/source/cpp/api/array.rst +++ b/docs/source/cpp/api/array.rst @@ -110,3 +110,10 @@ Utilities :project: arrow_cpp :members: :undoc-members: + +Statistics options +================== + +.. doxygenstruct:: arrow::StatisticsArrayTOptions + :project: arrow_cpp + :members: