From f00e630c048d9059b95fdf5aa2597920e948e7cd Mon Sep 17 00:00:00 2001 From: arash andishgar Date: Tue, 4 Mar 2025 11:50:48 +0330 Subject: [PATCH 1/4] add nested types for array statistics --- cpp/src/arrow/array/statistics.cc | 38 ++++++++++++ cpp/src/arrow/array/statistics.h | 32 +--------- cpp/src/arrow/record_batch.cc | 40 ++++++++++--- cpp/src/arrow/record_batch_test.cc | 95 ++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 37 deletions(-) diff --git a/cpp/src/arrow/array/statistics.cc b/cpp/src/arrow/array/statistics.cc index b661c9fbaffe..38e7633c9ccb 100644 --- a/cpp/src/arrow/array/statistics.cc +++ b/cpp/src/arrow/array/statistics.cc @@ -19,3 +19,41 @@ // arrow::ArrayStatistics into libarrow. #include "arrow/array/statistics.h" + +#include "arrow/scalar.h" + +namespace arrow { + +const std::shared_ptr& ArrayStatistics::ValueToArrowType( + const std::optional& value, + const std::shared_ptr& array_type) { + if (!value.has_value()) { + return null(); + } + + struct Visitor { + const std::shared_ptr& array_type; + + const std::shared_ptr& operator()(const bool&) { return boolean(); } + const std::shared_ptr& operator()(const int64_t&) { return int64(); } + const std::shared_ptr& operator()(const uint64_t&) { return uint64(); } + const std::shared_ptr& operator()(const double&) { return float64(); } + const std::shared_ptr& operator()(const std::shared_ptr& value) { + return value->type; + } + const std::shared_ptr& operator()(const std::string&) { + switch (array_type->id()) { + case Type::STRING: + case Type::BINARY: + case Type::FIXED_SIZE_BINARY: + case Type::LARGE_STRING: + case Type::LARGE_BINARY: + return array_type; + default: + return utf8(); + } + } + } visitor{array_type}; + return std::visit(visitor, value.value()); +} +} // namespace arrow diff --git a/cpp/src/arrow/array/statistics.h b/cpp/src/arrow/array/statistics.h index 6ccd2f4766e6..46d6fea26ba3 100644 --- a/cpp/src/arrow/array/statistics.h +++ b/cpp/src/arrow/array/statistics.h @@ -37,37 +37,11 @@ struct ARROW_EXPORT ArrayStatistics { /// \brief The type for maximum and minimum values. If the target /// value exists, one of them is used. `std::nullopt` is used /// otherwise. - using ValueType = std::variant; + using ValueType = + std::variant>; static const std::shared_ptr& ValueToArrowType( - const std::optional& value, - const std::shared_ptr& array_type) { - if (!value.has_value()) { - return null(); - } - - struct Visitor { - const std::shared_ptr& array_type; - - const std::shared_ptr& operator()(const bool&) { return boolean(); } - const std::shared_ptr& operator()(const int64_t&) { return int64(); } - const std::shared_ptr& operator()(const uint64_t&) { return uint64(); } - const std::shared_ptr& operator()(const double&) { return float64(); } - const std::shared_ptr& operator()(const std::string&) { - switch (array_type->id()) { - case Type::STRING: - case Type::BINARY: - case Type::FIXED_SIZE_BINARY: - case Type::LARGE_STRING: - case Type::LARGE_BINARY: - return array_type; - default: - return utf8(); - } - } - } visitor{array_type}; - return std::visit(visitor, value.value()); - } + const std::optional& value, const std::shared_ptr& array_type); /// \brief The number of null values, may not be set std::optional null_count = std::nullopt; diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index 9622fe8322cc..bb88c1aac2dc 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -487,6 +487,27 @@ struct EnumeratedStatistics { }; using OnStatistics = std::function; +using ArrayStatisticsAndTypeVector = + std::vector, std::shared_ptr>>; +ArrayStatisticsAndTypeVector ExtractArrayDataAndType(const RecordBatch& record_batch) { + ArrayDataVector traverse; + ArrayStatisticsAndTypeVector result; + for (int i = 0; i < record_batch.num_columns(); ++i) { + auto column_array_data = record_batch.column(i)->data(); + result.emplace_back(column_array_data->statistics, column_array_data->type); + traverse.insert(traverse.end(), column_array_data->child_data.crbegin(), + column_array_data->child_data.crend()); + while (!traverse.empty()) { + auto child_data = traverse.back(); + traverse.pop_back(); + result.emplace_back(child_data->statistics, child_data->type); + traverse.insert(traverse.end(), child_data->child_data.crbegin(), + child_data->child_data.crend()); + } + } + return result; +} + Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_statistics) { EnumeratedStatistics statistics; statistics.nth_statistics = 0; @@ -499,11 +520,11 @@ Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_stat RETURN_NOT_OK(on_statistics(statistics)); statistics.start_new_column = false; - const auto& schema = record_batch.schema(); - const auto num_fields = schema->num_fields(); - for (int nth_column = 0; nth_column < num_fields; ++nth_column) { - const auto& field = schema->field(nth_column); - auto column_statistics = record_batch.column(nth_column)->statistics(); + const auto& schema = ExtractArrayDataAndType(record_batch); + auto num_fields = static_cast(schema.size()); + for (int64_t nth_column = 0; nth_column < num_fields; ++nth_column) { + const auto& type = schema[nth_column].second; + auto column_statistics = schema[nth_column].first; if (!column_statistics) { continue; } @@ -535,7 +556,7 @@ Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_stat } else { statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE; } - statistics.type = column_statistics->MinArrowType(field->type()); + statistics.type = column_statistics->MinArrowType(type); statistics.value = column_statistics->min.value(); RETURN_NOT_OK(on_statistics(statistics)); statistics.start_new_column = false; @@ -548,7 +569,7 @@ Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_stat } else { statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE; } - statistics.type = column_statistics->MaxArrowType(field->type()); + statistics.type = column_statistics->MaxArrowType(type); statistics.value = column_statistics->max.value(); RETURN_NOT_OK(on_statistics(statistics)); statistics.start_new_column = false; @@ -580,7 +601,7 @@ Result> RecordBatch::MakeStatisticsArray( RETURN_NOT_OK(EnumerateStatistics(*this, [&](const EnumeratedStatistics& statistics) { int8_t i = 0; for (const auto& field : values_types) { - if (field->type()->id() == statistics.type->id()) { + if (field->type()->Equals(statistics.type)) { break; } i++; @@ -683,6 +704,9 @@ Result> RecordBatch::MakeStatisticsArray( return static_cast(builder)->Append( value.data(), static_cast(value.size())); } + Status operator()(const std::shared_ptr& value) { + return builder->AppendScalar(*value); + } } visitor; visitor.builder = values_builders[values_type_index].get(); RETURN_NOT_OK(std::visit(visitor, statistics.value)); diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index a659f8798e5c..5f65ee84169b 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -37,6 +37,7 @@ #include "arrow/c/abi.h" #include "arrow/chunked_array.h" #include "arrow/config.h" +#include "arrow/scalar.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/tensor.h" @@ -1082,6 +1083,13 @@ Result> BuildArray(const std::vector& values) auto values = StatisticsValuesToRawValues(values_); return BuildArray(values); } + Result> operator()(const std::shared_ptr& scalar) { + auto values = StatisticsValuesToRawValues>(values_); + ARROW_ASSIGN_OR_RAISE(auto builder, MakeBuilder(scalar->type)); + ARROW_RETURN_NOT_OK(builder->Reserve(values.size())); + ARROW_RETURN_NOT_OK(builder->AppendScalars(values)); + return builder->Finish(); + } } builder(values); return std::visit(builder, values[0]); } @@ -1454,6 +1462,93 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayString) { AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } +TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { + auto struct_type = struct_({field("a", int64()), field("b", int64())}); + auto struct_array = ArrayFromJSON( + struct_type, + R"([{"a":1,"b":6},{"a":2,"b":7},{"a":3,"b":8},{"a":4,"b":9},{"a":5,"b":10}])"); + ASSERT_OK_AND_ASSIGN(auto struct_nested_stat, + struct_array->CopyTo(default_cpu_memory_manager())); + auto statistics_struct = std::make_shared(); + ASSERT_OK_AND_ASSIGN(statistics_struct->max, struct_array->GetScalar(4)); + statistics_struct->null_count = 0; + auto struct_array_data = struct_array->data(); + auto statistics_struct_child_a = std::make_shared(); + statistics_struct_child_a->min = 1; + struct_array_data->statistics = statistics_struct; + struct_array_data->child_data[0]->statistics = statistics_struct_child_a; + auto array_c = ArrayFromJSON(int64(), R"([11,12,13,14,15])"); + array_c->data()->statistics = std::make_shared(); + array_c->data()->statistics->max = 15; + auto array_d = ArrayFromJSON(int64(), R"([16,17,18,19,20])"); + auto nested_child = struct_nested_stat->data()->child_data[0]; + nested_child->statistics = std::make_shared(); + nested_child->statistics->max = 5; + nested_child->statistics->is_max_exact = true; + + auto rb_schema = + schema({field("struct_a_b", struct_type), field("c", int64()), field("d", int64()), + field("struct_copy", struct_nested_stat->type())}); + auto rb = RecordBatch::Make(rb_schema, 5, + {struct_array, array_c, array_d, struct_nested_stat}); + + auto expected_scalar = std::static_pointer_cast(std::shared_ptr( + new StructScalar({MakeScalar(int64_t{5}), MakeScalar(int64_t{10})}, struct_type))); + auto a = ArrayStatistics::ValueType{std::static_pointer_cast(expected_scalar)}; + + ASSERT_OK_AND_ASSIGN( + auto expected_array, + MakeStatisticsArray("[null,0,1,3,6]", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{int64_t{0}}, + ArrayStatistics::ValueType{ + std::static_pointer_cast(expected_scalar)}}, + {ArrayStatistics::ValueType{int64_t{1}}}, + {ArrayStatistics::ValueType{int64_t{15}}}, + {ArrayStatistics::ValueType{int64_t{5}}}})); + ASSERT_OK_AND_ASSIGN(auto rb_stat, rb->MakeStatisticsArray()); + AssertArraysEqual(*expected_array, *rb_stat, true); +} + +TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { + // TODO add statisics + // TODO TRY with Type 32 + auto struct_type = struct_({field("a", int32()), field("b", int32())}); + auto struct_nested_0 = ArrayFromJSON( + struct_type, + R"([{"a":1,"b":6},{"a":2,"b":7},{"a":3,"b":8},{"a":4,"b":9},{"a":5,"b":10}])"); + + ASSERT_OK_AND_ASSIGN(auto struct_nested_1, + struct_nested_0->CopyTo(default_cpu_memory_manager())) + struct_nested_1->data()->statistics = std::make_shared(); + struct_nested_1->data()->statistics->is_max_exact = true; + ASSERT_OK_AND_ASSIGN(struct_nested_1->data()->statistics->max, + struct_nested_1->GetScalar(4)); + + ASSERT_OK_AND_ASSIGN( + auto struct_parent, + StructArray::Make({struct_nested_0, struct_nested_1}, + {field("struct_nested_0", struct_nested_0->type()), + field("struct_nested_1", struct_nested_1->type())})); + auto expected_scalar = std::static_pointer_cast(std::shared_ptr( + new StructScalar({MakeScalar(int32_t{5}), MakeScalar(int32_t{10})}, struct_type))); + auto rb_schema = schema({field("struct", struct_parent->type())}); + auto rb = RecordBatch::Make(rb_schema, 5, {struct_parent}); + + ASSERT_OK_AND_ASSIGN(auto expected_array, + MakeStatisticsArray(R"([null,4])", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, + {{5}, {expected_scalar}})); + AssertArraysEqual(*expected_array, *rb->MakeStatisticsArray().ValueOrDie(), true); +} + template class TestBatchToTensorColumnMajor : public ::testing::Test {}; From d41c9d817a4a6020e831d081ee9ea543faefd85d Mon Sep 17 00:00:00 2001 From: arash andishgar Date: Thu, 13 Mar 2025 00:40:19 +0330 Subject: [PATCH 2/4] fix casting,include,equal method with relevant tests --- cpp/src/arrow/array/statistics.cc | 42 ++++++++++++++++++++++++-- cpp/src/arrow/array/statistics.h | 11 +++---- cpp/src/arrow/array/statistics_test.cc | 24 +++++++++++++++ cpp/src/arrow/record_batch.cc | 10 +++--- cpp/src/arrow/record_batch_test.cc | 19 ++++++------ 5 files changed, 82 insertions(+), 24 deletions(-) diff --git a/cpp/src/arrow/array/statistics.cc b/cpp/src/arrow/array/statistics.cc index 38e7633c9ccb..0636bc45f593 100644 --- a/cpp/src/arrow/array/statistics.cc +++ b/cpp/src/arrow/array/statistics.cc @@ -15,12 +15,14 @@ // specific language governing permissions and limitations // under the License. -// This empty .cc file is for embedding not inlined symbols in -// arrow::ArrayStatistics into libarrow. - #include "arrow/array/statistics.h" +#include +#include +#include + #include "arrow/scalar.h" +#include "arrow/type.h" namespace arrow { @@ -56,4 +58,38 @@ const std::shared_ptr& ArrayStatistics::ValueToArrowType( } visitor{array_type}; return std::visit(visitor, value.value()); } +namespace { +bool ValueTypeEquality(const std::optional& first, + const std::optional& second) { + if (first == second) { + return true; + } + if (!first || !second) { + return false; + } + + return std::visit( + [](auto& v1, auto& v2) { + if constexpr (std::is_same_v, + std::shared_ptr> && + std::is_same_v, + std::shared_ptr>) { + if (!v1 || !v2) { + // both null case is handled in std::optional and return true + return false; + } + return v1->Equals(*v2); + } + return false; + }, + first.value(), second.value()); +} +} // namespace +bool ArrayStatistics::Equals(const ArrayStatistics& other) const { + return null_count == other.null_count && distinct_count == other.distinct_count && + is_min_exact == other.is_min_exact && is_max_exact == other.is_max_exact && + ValueTypeEquality(this->max, other.max) && + ValueTypeEquality(this->min, other.min); +} + } // namespace arrow diff --git a/cpp/src/arrow/array/statistics.h b/cpp/src/arrow/array/statistics.h index 46d6fea26ba3..42e241e09c9c 100644 --- a/cpp/src/arrow/array/statistics.h +++ b/cpp/src/arrow/array/statistics.h @@ -18,15 +18,16 @@ #pragma once #include +#include #include #include #include -#include "arrow/type.h" #include "arrow/util/visibility.h" namespace arrow { - +class DataType; +struct Scalar; /// \class ArrayStatistics /// \brief Statistics for an Array /// @@ -100,11 +101,7 @@ struct ARROW_EXPORT ArrayStatistics { bool is_max_exact = false; /// \brief Check two statistics for equality - bool Equals(const ArrayStatistics& other) const { - return null_count == other.null_count && distinct_count == other.distinct_count && - min == other.min && is_min_exact == other.is_min_exact && max == other.max && - is_max_exact == other.is_max_exact; - } + bool Equals(const ArrayStatistics& other) const; /// \brief Check two statistics for equality bool operator==(const ArrayStatistics& other) const { return Equals(other); } diff --git a/cpp/src/arrow/array/statistics_test.cc b/cpp/src/arrow/array/statistics_test.cc index cf15a5d38297..000ad66d0d36 100644 --- a/cpp/src/arrow/array/statistics_test.cc +++ b/cpp/src/arrow/array/statistics_test.cc @@ -18,6 +18,8 @@ #include #include "arrow/array/statistics.h" +#include "arrow/scalar.h" +#include "arrow/testing/gtest_util.h" namespace arrow { @@ -91,6 +93,28 @@ TEST(ArrayStatisticsTest, TestEquality) { ASSERT_NE(statistics1, statistics2); statistics2.max = static_cast(-29); ASSERT_EQ(statistics1, statistics2); + statistics2.max = static_cast(2); + ASSERT_NE(statistics1, statistics2); + + statistics1.max = std::nullopt; + statistics2.max = std::nullopt; + // check the state of both of them are std::nullopt + ASSERT_EQ(statistics1.max, statistics2.max); + // the state of one of them is std::nullopt + statistics1.max = std::shared_ptr(); + ASSERT_NE(statistics1, statistics2); + // the state of both of them are nullptr + statistics2.max = std::shared_ptr(); + ASSERT_EQ(statistics1, statistics2); + ASSERT_OK_AND_ASSIGN(statistics1.max, MakeScalar(int64(), 5)); + // the state of one of them is nullptr + ASSERT_NE(statistics1, statistics2); + // the state of one of them has different type + statistics2.max = static_cast(10); + ASSERT_NE(statistics1.max, statistics2.max); + ASSERT_OK_AND_ASSIGN(statistics2.max, MakeScalar(int64(), 5)); + // the state of both of them are equal + ASSERT_EQ(statistics1, statistics2); statistics1.is_max_exact = true; ASSERT_NE(statistics1, statistics2); diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index bb88c1aac2dc..4915abf3b4f7 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -520,17 +520,17 @@ Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_stat RETURN_NOT_OK(on_statistics(statistics)); statistics.start_new_column = false; - const auto& schema = ExtractArrayDataAndType(record_batch); - auto num_fields = static_cast(schema.size()); + const auto& array_statistics_and_type_vector = ExtractArrayDataAndType(record_batch); + auto num_fields = static_cast(array_statistics_and_type_vector.size()); for (int64_t nth_column = 0; nth_column < num_fields; ++nth_column) { - const auto& type = schema[nth_column].second; - auto column_statistics = schema[nth_column].first; + const auto& type = array_statistics_and_type_vector[nth_column].second; + auto column_statistics = array_statistics_and_type_vector[nth_column].first; if (!column_statistics) { continue; } statistics.start_new_column = true; - statistics.nth_column = nth_column; + statistics.nth_column = static_cast(nth_column); if (column_statistics->null_count.has_value()) { statistics.nth_statistics++; statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT; diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index 5f65ee84169b..408683ddc851 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -1474,16 +1474,16 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { statistics_struct->null_count = 0; auto struct_array_data = struct_array->data(); auto statistics_struct_child_a = std::make_shared(); - statistics_struct_child_a->min = 1; + statistics_struct_child_a->min = int64_t{1}; struct_array_data->statistics = statistics_struct; struct_array_data->child_data[0]->statistics = statistics_struct_child_a; auto array_c = ArrayFromJSON(int64(), R"([11,12,13,14,15])"); array_c->data()->statistics = std::make_shared(); - array_c->data()->statistics->max = 15; + array_c->data()->statistics->max = int64_t{15}; auto array_d = ArrayFromJSON(int64(), R"([16,17,18,19,20])"); auto nested_child = struct_nested_stat->data()->child_data[0]; nested_child->statistics = std::make_shared(); - nested_child->statistics->max = 5; + nested_child->statistics->max = int64_t{5}; nested_child->statistics->is_max_exact = true; auto rb_schema = @@ -1492,8 +1492,8 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { auto rb = RecordBatch::Make(rb_schema, 5, {struct_array, array_c, array_d, struct_nested_stat}); - auto expected_scalar = std::static_pointer_cast(std::shared_ptr( - new StructScalar({MakeScalar(int64_t{5}), MakeScalar(int64_t{10})}, struct_type))); + auto expected_scalar = internal::checked_pointer_cast( + ScalarFromJSON(struct_type, R"([5,10])")); auto a = ArrayStatistics::ValueType{std::static_pointer_cast(expected_scalar)}; ASSERT_OK_AND_ASSIGN( @@ -1536,8 +1536,8 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { StructArray::Make({struct_nested_0, struct_nested_1}, {field("struct_nested_0", struct_nested_0->type()), field("struct_nested_1", struct_nested_1->type())})); - auto expected_scalar = std::static_pointer_cast(std::shared_ptr( - new StructScalar({MakeScalar(int32_t{5}), MakeScalar(int32_t{10})}, struct_type))); + auto expected_scalar = internal::checked_pointer_cast( + ScalarFromJSON(struct_type, R"([5,10])")); auto rb_schema = schema({field("struct", struct_parent->type())}); auto rb = RecordBatch::Make(rb_schema, 5, {struct_parent}); @@ -1545,8 +1545,9 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { MakeStatisticsArray(R"([null,4])", {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, - {{5}, {expected_scalar}})); - AssertArraysEqual(*expected_array, *rb->MakeStatisticsArray().ValueOrDie(), true); + {{int64_t{5}}, {expected_scalar}})); + ASSERT_OK_AND_ASSIGN(auto rb_stat, rb->MakeStatisticsArray()); + AssertArraysEqual(*expected_array, *rb_stat, true); } template From ebfe502caccf0cb05dfee14afc3f0da176104ff3 Mon Sep 17 00:00:00 2001 From: arash andishgar Date: Tue, 18 Mar 2025 23:56:43 +0330 Subject: [PATCH 3/4] add options,test utils and new tests fix linting and add more test fix casting and add document fix equal method and relevant tests fix linting the Correction of how the number of rows is calculated fix linting --- cpp/src/arrow/CMakeLists.txt | 2 + cpp/src/arrow/array/statistics.cc | 307 +++++++- cpp/src/arrow/array/statistics.h | 43 +- cpp/src/arrow/array/statistics_option.cc | 18 + cpp/src/arrow/array/statistics_option.h | 43 ++ cpp/src/arrow/array/statistics_test.cc | 22 +- cpp/src/arrow/array/statistics_test_util.cc | 286 ++++++++ cpp/src/arrow/array/statistics_test_util.h | 60 ++ cpp/src/arrow/record_batch.cc | 252 +------ cpp/src/arrow/record_batch.h | 6 +- cpp/src/arrow/record_batch_test.cc | 752 ++++++++++---------- docs/source/cpp/api/array.rst | 7 + 12 files changed, 1166 insertions(+), 632 deletions(-) create mode 100644 cpp/src/arrow/array/statistics_option.cc create mode 100644 cpp/src/arrow/array/statistics_option.h create mode 100644 cpp/src/arrow/array/statistics_test_util.cc create mode 100644 cpp/src/arrow/array/statistics_test_util.h diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 404e2a03501d..3701c7d60228 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -414,6 +414,7 @@ arrow_add_object_library(ARROW_ARRAY array/data.cc array/diff.cc array/statistics.cc + array/statistics_option.cc array/util.cc array/validate.cc) @@ -668,6 +669,7 @@ endif() set(ARROW_TESTING_SRCS io/test_common.cc + array/statistics_test_util.cc ipc/test_common.cc testing/fixed_width_test_util.cc testing/generator.cc diff --git a/cpp/src/arrow/array/statistics.cc b/cpp/src/arrow/array/statistics.cc index 0636bc45f593..599f62d8ff90 100644 --- a/cpp/src/arrow/array/statistics.cc +++ b/cpp/src/arrow/array/statistics.cc @@ -17,11 +17,26 @@ #include "arrow/array/statistics.h" +#include +#include #include +#include #include +#include #include +#include +#include "arrow/array/array_base.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_dict.h" +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_union.h" +#include "arrow/array/data.h" +#include "arrow/c/abi.h" +#include "arrow/memory_pool.h" +#include "arrow/result.h" #include "arrow/scalar.h" +#include "arrow/status.h" #include "arrow/type.h" namespace arrow { @@ -45,6 +60,7 @@ const std::shared_ptr& ArrayStatistics::ValueToArrowType( } const std::shared_ptr& operator()(const std::string&) { switch (array_type->id()) { + // TODO Add StringView and BinaryView to ArrayStatistics (GH-45664) case Type::STRING: case Type::BINARY: case Type::FIXED_SIZE_BINARY: @@ -60,7 +76,8 @@ const std::shared_ptr& ArrayStatistics::ValueToArrowType( } namespace { bool ValueTypeEquality(const std::optional& first, - const std::optional& second) { + const std::optional& second, + const EqualOptions& options) { if (first == second) { return true; } @@ -69,27 +86,303 @@ bool ValueTypeEquality(const std::optional& first, } return std::visit( - [](auto& v1, auto& v2) { + [&options](auto& v1, auto& v2) { if constexpr (std::is_same_v, std::shared_ptr> && std::is_same_v, std::shared_ptr>) { if (!v1 || !v2) { - // both null case is handled in std::optional and return true + // both empty shared_ptr case is handled in std::optional and return true return false; } - return v1->Equals(*v2); + return v1->Equals(*v2, options); } return false; }, first.value(), second.value()); } } // namespace -bool ArrayStatistics::Equals(const ArrayStatistics& other) const { +bool ArrayStatistics::Equals(const ArrayStatistics& other, + const EqualOptions& options) const { return null_count == other.null_count && distinct_count == other.distinct_count && is_min_exact == other.is_min_exact && is_max_exact == other.is_max_exact && - ValueTypeEquality(this->max, other.max) && - ValueTypeEquality(this->min, other.min); + ValueTypeEquality(this->max, other.max, options) && + ValueTypeEquality(this->min, other.min, options); } +namespace internal { +// Perform a depth-first search (DFS) on the given array to extract +// all nested arrays up to the specified maximum nesting depth. +Result ExtractColumnsToArrayData(const std::shared_ptr& array, + const int32_t& max_nesting_depth) { + ArrayDataVector traverse; + ArrayDataVector result; + int32_t remaining_nested_depth = max_nesting_depth; + if (remaining_nested_depth <= 0) { + return Status::Invalid("Max recursion depth reached"); + } + result.push_back(array->data()); + if (!array->data()->child_data.empty()) { + // depth delimiter + traverse.emplace_back(nullptr); + traverse.insert(traverse.end(), array->data()->child_data.crbegin(), + array->data()->child_data.crend()); + remaining_nested_depth--; + } + + while (!traverse.empty()) { + if (remaining_nested_depth <= 0) { + return Status::Invalid("Max recursion depth reached"); + } + + auto child_data = traverse.back(); + traverse.pop_back(); + // Check whether the traversal of a depth is finished or not. + if (!child_data) { + remaining_nested_depth++; + continue; + } + result.push_back(child_data); + if (!child_data->child_data.empty()) { + // depth delimiter + traverse.emplace_back(nullptr); + traverse.insert(traverse.end(), child_data->child_data.crbegin(), + child_data->child_data.crend()); + remaining_nested_depth--; + } + } + return result; +} +namespace { +struct EnumeratedStatistics { + int nth_statistics = 0; + bool start_new_column = false; + std::optional nth_column = std::nullopt; + const char* key = nullptr; + std::shared_ptr type = nullptr; + ArrayStatistics::ValueType value = false; +}; +using OnStatistics = + std::function; + +Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, + std::optional num_rows, OnStatistics on_statistics) { + // TODO implement Statistic Schema attribute in C++(GH-45639) + EnumeratedStatistics statistics; + statistics.nth_statistics = 0; + statistics.start_new_column = true; + statistics.nth_column = std::nullopt; + + statistics.key = ARROW_STATISTICS_KEY_ROW_COUNT_EXACT; + statistics.type = int64(); + statistics.value = ArrayStatistics::ValueType{int64_t{0}}; + if (num_rows.has_value()) { + statistics.value = ArrayStatistics::ValueType{num_rows.value()}; + } else if (!extracted_array_data.empty()) { + statistics.value = + ArrayStatistics::ValueType{int64_t{extracted_array_data[0]->length}}; + } + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + + auto num_fields = static_cast(extracted_array_data.size()); + for (int64_t nth_column = 0; nth_column < num_fields; ++nth_column) { + const auto& type = extracted_array_data[nth_column]->type; + auto column_statistics = extracted_array_data[nth_column]->statistics; + if (!column_statistics) { + continue; + } + + statistics.start_new_column = true; + statistics.nth_column = static_cast(nth_column); + if (column_statistics->null_count.has_value()) { + statistics.nth_statistics++; + statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT; + statistics.type = int64(); + statistics.value = column_statistics->null_count.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + + if (column_statistics->distinct_count.has_value()) { + statistics.nth_statistics++; + statistics.key = ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT; + statistics.type = int64(); + statistics.value = column_statistics->distinct_count.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + + if (column_statistics->min.has_value()) { + statistics.nth_statistics++; + if (column_statistics->is_min_exact) { + statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_EXACT; + } else { + statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE; + } + statistics.type = column_statistics->MinArrowType(type); + statistics.value = column_statistics->min.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + + if (column_statistics->max.has_value()) { + statistics.nth_statistics++; + if (column_statistics->is_max_exact) { + statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_EXACT; + } else { + statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE; + } + statistics.type = column_statistics->MaxArrowType(type); + statistics.value = column_statistics->max.value(); + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; + } + } + return Status::OK(); +} +} // namespace + +Result> MakeStatisticsArray( + MemoryPool* memory_pool, const ArrayDataVector& extracted_array_data_vector, + std::optional num_rows) { + // Statistics schema: + // struct< + // column: int32, + // statistics: map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + // > + + // Statistics schema doesn't define static dense union type for + // values. Each statistics schema have a dense union type that has + // needled value types. The following block collects these types. + + std::vector> values_types; + std::vector values_type_indexes; + RETURN_NOT_OK(EnumerateStatistics( + extracted_array_data_vector, num_rows, [&](const EnumeratedStatistics& statistics) { + int8_t i = 0; + for (const auto& field : values_types) { + if (field->type()->Equals(statistics.type)) { + break; + } + i++; + } + if (i == static_cast(values_types.size())) { + values_types.push_back(field(statistics.type->name(), statistics.type)); + } + values_type_indexes.push_back(i); + return Status::OK(); + })); + + // statistics.key: dictionary + auto keys_type = dictionary(int32(), utf8(), false); + // statistics.items: dense_union<...all needed types...> + auto values_type = dense_union(values_types); + // struct< + // column: int32, + // statistics: map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + // > + auto statistics_type = + struct_({field("column", int32()), + field("statistics", map(keys_type, values_type, false))}); + + std::vector> field_builders; + // columns: int32 + auto columns_builder = std::make_shared(memory_pool); + field_builders.push_back(std::static_pointer_cast(columns_builder)); + // statistics.key: dictionary + auto keys_builder = std::make_shared(); + // statistics.items: dense_union<...all needed types...> + std::vector> values_builders; + for (const auto& values_type : values_types) { + std::unique_ptr values_builder; + RETURN_NOT_OK(MakeBuilder(memory_pool, values_type->type(), &values_builder)); + values_builders.push_back(std::shared_ptr(std::move(values_builder))); + } + auto items_builder = std::make_shared( + memory_pool, std::move(values_builders), values_type); + // statistics: + // map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + auto values_builder = std::make_shared( + memory_pool, std::static_pointer_cast(keys_builder), + std::static_pointer_cast(items_builder)); + field_builders.push_back(std::static_pointer_cast(values_builder)); + // struct< + // column: int32, + // statistics: map< + // key: dictionary< + // indices: int32, + // dictionary: utf8, + // >, + // items: dense_union<...all needed types...>, + // > + // > + StructBuilder builder(statistics_type, memory_pool, std::move(field_builders)); + + // Append statistics. + RETURN_NOT_OK(EnumerateStatistics( + extracted_array_data_vector, num_rows, [&](const EnumeratedStatistics& statistics) { + if (statistics.start_new_column) { + RETURN_NOT_OK(builder.Append()); + if (statistics.nth_column.has_value()) { + RETURN_NOT_OK(columns_builder->Append(statistics.nth_column.value())); + } else { + RETURN_NOT_OK(columns_builder->AppendNull()); + } + RETURN_NOT_OK(values_builder->Append()); + } + RETURN_NOT_OK(keys_builder->Append(statistics.key, + static_cast(strlen(statistics.key)))); + const auto values_type_index = values_type_indexes[statistics.nth_statistics]; + RETURN_NOT_OK(items_builder->Append(values_type_index)); + struct Visitor { + ArrayBuilder* builder; + + Status operator()(const bool& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const int64_t& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const uint64_t& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const double& value) { + return static_cast(builder)->Append(value); + } + Status operator()(const std::string& value) { + return static_cast(builder)->Append( + value.data(), static_cast(value.size())); + } + Status operator()(const std::shared_ptr& value) { + return builder->AppendScalar(*value); + } + } visitor; + visitor.builder = values_builders[values_type_index].get(); + RETURN_NOT_OK(std::visit(visitor, statistics.value)); + return Status::OK(); + })); + + return builder.Finish(); +} + +} // namespace internal } // namespace arrow diff --git a/cpp/src/arrow/array/statistics.h b/cpp/src/arrow/array/statistics.h index 42e241e09c9c..07f8d6c286be 100644 --- a/cpp/src/arrow/array/statistics.h +++ b/cpp/src/arrow/array/statistics.h @@ -23,11 +23,13 @@ #include #include +#include "arrow/compare.h" +#include "arrow/result.h" +#include "arrow/type_fwd.h" #include "arrow/util/visibility.h" namespace arrow { -class DataType; -struct Scalar; + /// \class ArrayStatistics /// \brief Statistics for an Array /// @@ -101,7 +103,13 @@ struct ARROW_EXPORT ArrayStatistics { bool is_max_exact = false; /// \brief Check two statistics for equality - bool Equals(const ArrayStatistics& other) const; + /// + /// \param[in] other the ArrayStatistics to compare with + /// \param[in] options the options for equality comparisons of Scalars + /// + /// \return true if statistics are equal + bool Equals(const ArrayStatistics& other, + const EqualOptions& options = EqualOptions::Defaults()) const; /// \brief Check two statistics for equality bool operator==(const ArrayStatistics& other) const { return Equals(other); } @@ -110,4 +118,33 @@ struct ARROW_EXPORT ArrayStatistics { bool operator!=(const ArrayStatistics& other) const { return !Equals(other); } }; +namespace internal { +/// \brief Extract all nested arrays from an array as \ref ArrayDataVector. +/// +/// Returns all nested arrays within the given array, up to the specified +/// maximum nesting depth, as a vector of ArrayData. +/// +/// \param[in] array The input array from which nested arrays will be extracted. +/// \param[in] max_nesting_depth The maximum depth of nested arrays to extract. +/// +/// \return A vector of \ref ArrayData . +ARROW_EXPORT +Result ExtractColumnsToArrayData(const std::shared_ptr& array, + const int32_t& max_nesting_depth); +/// \brief Collect Statistics From ArrayDataVector and turns into an Array +/// +/// \param[in] memory_pool the memory pool to allocate memory from +/// \param extracted_array_data_vector The input ArrayData from which statistics array is +/// extracted +/// +/// \param num_rows The input specifies the number of rows an object if it is set to +/// std::nullopt, the number of rows is deduced from the length of the first array +/// +/// \return A statistics array. +ARROW_EXPORT +Result> MakeStatisticsArray( + MemoryPool* memory_pool, const ArrayDataVector& extracted_array_data_vector, + std::optional num_rows = std::nullopt); +} // namespace internal + } // namespace arrow diff --git a/cpp/src/arrow/array/statistics_option.cc b/cpp/src/arrow/array/statistics_option.cc new file mode 100644 index 000000000000..f6bd5b5697c9 --- /dev/null +++ b/cpp/src/arrow/array/statistics_option.cc @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/array/statistics_option.h" diff --git a/cpp/src/arrow/array/statistics_option.h b/cpp/src/arrow/array/statistics_option.h new file mode 100644 index 000000000000..f117f9c3fe41 --- /dev/null +++ b/cpp/src/arrow/array/statistics_option.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +// ARROW-109: We set this number arbitrarily to help catch user mistakes. For +// deeply nested schemas, it is expected the user will indicate explicitly the +// maximum allowed recursion depth +constexpr int32_t kMaxNestingDepth = 64; + +/// \brief Options for creating statistics array +struct ARROW_EXPORT StatisticsArrayTOptions { + /// \brief The memory pool to use for allocations made during creating a statistics + /// arrays + MemoryPool* memory_pool = default_memory_pool(); + + /// \brief The maximum permitted schema nesting depth. + int32_t max_recursion_depth = kMaxNestingDepth; + + static StatisticsArrayTOptions Default() { return {}; } +}; +} // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test.cc b/cpp/src/arrow/array/statistics_test.cc index 000ad66d0d36..043d4149b081 100644 --- a/cpp/src/arrow/array/statistics_test.cc +++ b/cpp/src/arrow/array/statistics_test.cc @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#include #include @@ -68,7 +69,6 @@ TEST(ArrayStatisticsTest, TestEquality) { ArrayStatistics statistics2; ASSERT_EQ(statistics1, statistics2); - statistics1.null_count = 29; ASSERT_NE(statistics1, statistics2); statistics2.null_count = 29; @@ -98,16 +98,16 @@ TEST(ArrayStatisticsTest, TestEquality) { statistics1.max = std::nullopt; statistics2.max = std::nullopt; - // check the state of both of them are std::nullopt + // the state of both of them are std::nullopt ASSERT_EQ(statistics1.max, statistics2.max); - // the state of one of them is std::nullopt + // the state of one of them is std::nullopt statistics1.max = std::shared_ptr(); ASSERT_NE(statistics1, statistics2); - // the state of both of them are nullptr + // the state of both of them are empty shared_ptr statistics2.max = std::shared_ptr(); ASSERT_EQ(statistics1, statistics2); ASSERT_OK_AND_ASSIGN(statistics1.max, MakeScalar(int64(), 5)); - // the state of one of them is nullptr + // the state of one of them is empty shared_ptr ASSERT_NE(statistics1, statistics2); // the state of one of them has different type statistics2.max = static_cast(10); @@ -121,5 +121,17 @@ TEST(ArrayStatisticsTest, TestEquality) { statistics2.is_max_exact = true; ASSERT_EQ(statistics1, statistics2); } +TEST(ArrayStatisticsTest, EqualOptions) { + auto list_nan_1 = std::make_shared( + ArrayFromJSON(float32(), R"([0,null,NaN])"), list(float32())); + auto list_nan_2 = std::make_shared( + ArrayFromJSON(float32(), R"([0,null,NaN])"), list(float32())); + ArrayStatistics statistics1; + ArrayStatistics statistics2; + statistics1.max = list_nan_1; + statistics2.max = list_nan_2; + ASSERT_FALSE(statistics1.Equals(statistics2)); + ASSERT_TRUE(statistics1.Equals(statistics2, EqualOptions::Defaults().nans_equal(true))); +} } // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test_util.cc b/cpp/src/arrow/array/statistics_test_util.cc new file mode 100644 index 000000000000..f09dea8da18c --- /dev/null +++ b/cpp/src/arrow/array/statistics_test_util.cc @@ -0,0 +1,286 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/array/statistics_test_util.h" + +#include + +#include +#include +#include +#include + +#include "arrow/array/array_base.h" +#include "arrow/array/array_dict.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_base.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/array/statistics.h" +#include "arrow/array/statistics_option.h" +#include "arrow/result.h" +#include "arrow/scalar.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/type_traits.h" + +namespace arrow { +namespace { +template ::value || + is_number_type::value>> +Result> BuildArray( + const std::vector::CType>& values) { + using BuilderType = typename TypeTraits::BuilderType; + BuilderType builder; + for (const auto& value : values) { + ARROW_RETURN_NOT_OK(builder.Append(value)); + } + return builder.Finish(); +} + +template > +Result> BuildArray(const std::vector& values) { + using BuilderType = typename TypeTraits::BuilderType; + BuilderType builder; + for (const auto& value : values) { + ARROW_RETURN_NOT_OK(builder.Append(value)); + } + return builder.Finish(); +} + +template +std::vector StatisticsValuesToRawValues( + const std::vector& values) { + std::vector raw_values; + for (const auto& value : values) { + raw_values.push_back(std::get(value)); + } + return raw_values; +} + +template ::value>> +Result> BuildArray(const std::vector& values) { + struct Builder { + const std::vector& values_; + explicit Builder(const std::vector& values) + : values_(values) {} + + Result> operator()(const bool&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const int64_t&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const uint64_t&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const double&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const std::string&) { + auto values = StatisticsValuesToRawValues(values_); + return BuildArray(values); + } + Result> operator()(const std::shared_ptr& scalar) { + auto values = StatisticsValuesToRawValues>(values_); + ARROW_ASSIGN_OR_RAISE(auto builder, MakeBuilder(scalar->type)); + ARROW_RETURN_NOT_OK(builder->Reserve(values.size())); + ARROW_RETURN_NOT_OK(builder->AppendScalars(values)); + return builder->Finish(); + } + } builder(values); + return std::visit(builder, values[0]); +} +} // namespace +namespace test { +Result> MakeMockStatisticsArray( + const std::string& columns_json, + const std::vector>& nested_statistics_keys, + const std::vector>& + nested_statistics_values) { + const auto& columns_type = int32(); + auto columns_array = ArrayFromJSON(columns_type, columns_json); + const auto n_columns = columns_array->length(); + + // nested_statistics_keys: + // { + // {"ARROW:row_count:exact", "ARROW:null_count:exact"}, + // {"ARROW:max_value:exact"}, + // {"ARROW:max_value:exact", "ARROW:distinct_count:exact"}, + // } + // nested_statistics_values: + // { + // {int64_t{29}, int64_t{1}}, + // {double{2.9}}, + // {double{-2.9}, int64_t{2}}, + // } + // -> + // keys_dictionary: + // { + // "ARROW:row_count:exact", + // "ARROW:null_count:exact", + // "ARROW:max_value:exact", + // "ARROW:distinct_count:exact", + // } + // keys_indices: {0, 1, 2, 2, 3} + // values_types: {int64(), float64()} + // values_type_codes: {0, 1} + // values_values[0]: {int64_t{29}, int64_t{1}, int64_t{2}} + // values_values[1]: {double{2.9}, double{-2.9}} + // values_value_type_ids: {0, 0, 1, 1, 0} + // values_value_offsets: {0, 1, 0, 1, 2} + // statistics_offsets: {0, 2, 3, 5, 5} + std::vector keys_dictionary; + std::vector keys_indices; + std::vector> values_types; + std::vector values_type_codes; + std::vector> values_values; + std::vector values_value_type_ids; + std::vector values_value_offsets; + std::vector statistics_offsets; + + int32_t offset = 0; + std::vector values_value_offset_counters; + for (size_t i = 0; i < nested_statistics_keys.size(); ++i) { + const auto& statistics_keys = nested_statistics_keys[i]; + const auto& statistics_values = nested_statistics_values[i]; + statistics_offsets.push_back(offset); + for (size_t j = 0; j < statistics_keys.size(); ++j) { + const auto& key = statistics_keys[j]; + const auto& value = statistics_values[j]; + ++offset; + + int32_t key_index = 0; + for (; key_index < static_cast(keys_dictionary.size()); ++key_index) { + if (keys_dictionary[key_index] == key) { + break; + } + } + if (key_index == static_cast(keys_dictionary.size())) { + keys_dictionary.push_back(key); + } + keys_indices.push_back(key_index); + + auto values_type = ArrayStatistics::ValueToArrowType(value, arrow::null()); + int8_t values_type_code = 0; + for (; values_type_code < static_cast(values_types.size()); + ++values_type_code) { + if (values_types[values_type_code]->Equals(values_type)) { + break; + } + } + if (values_type_code == static_cast(values_types.size())) { + values_types.push_back(values_type); + values_type_codes.push_back(values_type_code); + values_values.emplace_back(); + values_value_offset_counters.push_back(0); + } + values_values[values_type_code].push_back(value); + values_value_type_ids.push_back(values_type_code); + values_value_offsets.push_back(values_value_offset_counters[values_type_code]++); + } + } + statistics_offsets.push_back(offset); + + auto keys_type = dictionary(int32(), utf8(), false); + std::vector> values_fields; + for (const auto& type : values_types) { + values_fields.push_back(field(type->name(), type)); + } + auto values_type = dense_union(values_fields); + auto statistics_type = map(keys_type, values_type, false); + auto struct_type = + struct_({field("column", columns_type), field("statistics", statistics_type)}); + + ARROW_ASSIGN_OR_RAISE(auto keys_indices_array, BuildArray(keys_indices)); + ARROW_ASSIGN_OR_RAISE(auto keys_dictionary_array, + BuildArray(keys_dictionary)); + ARROW_ASSIGN_OR_RAISE( + auto keys_array, + DictionaryArray::FromArrays(keys_type, keys_indices_array, keys_dictionary_array)); + + std::vector> values_arrays; + for (const auto& values : values_values) { + ARROW_ASSIGN_OR_RAISE(auto values_array, + BuildArray(values)); + values_arrays.push_back(values_array); + } + ARROW_ASSIGN_OR_RAISE(auto values_value_type_ids_array, + BuildArray(values_value_type_ids)); + ARROW_ASSIGN_OR_RAISE(auto values_value_offsets_array, + BuildArray(values_value_offsets)); + auto values_array = std::make_shared( + values_type, values_value_offsets_array->length(), values_arrays, + values_value_type_ids_array->data()->buffers[1], + values_value_offsets_array->data()->buffers[1]); + ARROW_ASSIGN_OR_RAISE(auto statistics_offsets_array, + BuildArray(statistics_offsets)); + ARROW_ASSIGN_OR_RAISE(auto statistics_array, + MapArray::FromArrays(statistics_type, statistics_offsets_array, + keys_array, values_array)); + std::vector> struct_arrays = {std::move(columns_array), + std::move(statistics_array)}; + return std::make_shared(struct_type, n_columns, struct_arrays); +} + +Result> MakeNestedStruct(const int32_t depth) { + // each struct occupy two depth + if (depth < 2) { + return Status::Invalid("depth must be greater thant 1"); + } + auto parent_struct_type = + struct_({field("a", int64()), field("b", int32()), field("c", int16())}); + auto parent_struct = + ArrayFromJSON(parent_struct_type, + R"([{"a":1,"b":4,"c":7},{"a":2,"b":5,"c":8},{"a":3,"b":6,"c":9}])"); + auto child_struct = + ArrayFromJSON(parent_struct_type, + R"([{"a":1,"b":4,"c":7},{"a":2,"b":5,"c":8},{"a":3,"b":6,"c":9}])"); + + for (int i = 0; i < depth - 2; ++i) { + ARROW_ASSIGN_OR_RAISE( + parent_struct, StructArray::Make({parent_struct, child_struct}, + {field("struct", parent_struct->type()), + field("child_struct", child_struct->type())})); + } + return parent_struct; +} +Status CheckDepth( + std::variant, std::shared_ptr> value, + const StatisticsArrayTOptions& options) { + struct { + StatisticsArrayTOptions options; + Status operator()(const std::shared_ptr& batch) const { + return batch->MakeStatisticsArray(options).status(); + } + Status operator()(const std::shared_ptr& array) const { + // TODO Lack of method for creating statistics array for Array(GH-45804) + return Status::NotImplemented("MakeStatisticsArray not implemented for Array"); + } + } visitor; + visitor.options = options; + return std::visit(visitor, value); +} +}; // namespace test +} // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test_util.h b/cpp/src/arrow/array/statistics_test_util.h new file mode 100644 index 000000000000..7599d0f64af9 --- /dev/null +++ b/cpp/src/arrow/array/statistics_test_util.h @@ -0,0 +1,60 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/array/statistics.h" +#include "arrow/testing/visibility.h" + +#include "arrow/type_fwd.h" + +namespace arrow { +struct StatisticsArrayTOptions; +namespace test { + +ARROW_TESTING_EXPORT Result> MakeMockStatisticsArray( + const std::string& columns_json, + const std::vector>& nested_statistics_keys, + const std::vector>& nested_statistics_values); + +/// \brief Create a nested struct array with a specified depth. +/// +/// \param[in] depth The depth of the nested structs. Must be greater than 1. +/// +/// \return A parent_struct which has depth -1 child struct +ARROW_TESTING_EXPORT +Result> MakeNestedStruct(int32_t depth); + +/// \brief Check whether it's possible to create a statistics array with the given +/// options. +/// +/// \param[in] value object for calling its Make Statistics Array +/// \param[in] options options for creating statistics array +/// +/// \return Whether \ref Status::OK or \ref Status::Invalid("Max recursion depth reached") +ARROW_TESTING_EXPORT +Status CheckDepth( + std::variant, std::shared_ptr> value, + const StatisticsArrayTOptions& options); +} // namespace test +} // namespace arrow diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index 4915abf3b4f7..e2f386e63e20 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -32,8 +33,9 @@ #include "arrow/array/builder_nested.h" #include "arrow/array/builder_union.h" #include "arrow/array/concatenate.h" +#include "arrow/array/statistics.h" +#include "arrow/array/statistics_option.h" #include "arrow/array/validate.h" -#include "arrow/c/abi.h" #include "arrow/pretty_print.h" #include "arrow/status.h" #include "arrow/table.h" @@ -476,244 +478,18 @@ Result> RecordBatch::ViewOrCopyTo( return Make(schema_, num_rows(), std::move(copied_columns)); } -namespace { -struct EnumeratedStatistics { - int nth_statistics = 0; - bool start_new_column = false; - std::optional nth_column = std::nullopt; - const char* key = nullptr; - std::shared_ptr type = nullptr; - ArrayStatistics::ValueType value = false; -}; -using OnStatistics = - std::function; -using ArrayStatisticsAndTypeVector = - std::vector, std::shared_ptr>>; -ArrayStatisticsAndTypeVector ExtractArrayDataAndType(const RecordBatch& record_batch) { - ArrayDataVector traverse; - ArrayStatisticsAndTypeVector result; - for (int i = 0; i < record_batch.num_columns(); ++i) { - auto column_array_data = record_batch.column(i)->data(); - result.emplace_back(column_array_data->statistics, column_array_data->type); - traverse.insert(traverse.end(), column_array_data->child_data.crbegin(), - column_array_data->child_data.crend()); - while (!traverse.empty()) { - auto child_data = traverse.back(); - traverse.pop_back(); - result.emplace_back(child_data->statistics, child_data->type); - traverse.insert(traverse.end(), child_data->child_data.crbegin(), - child_data->child_data.crend()); - } - } - return result; -} - -Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics on_statistics) { - EnumeratedStatistics statistics; - statistics.nth_statistics = 0; - statistics.start_new_column = true; - statistics.nth_column = std::nullopt; - - statistics.key = ARROW_STATISTICS_KEY_ROW_COUNT_EXACT; - statistics.type = int64(); - statistics.value = record_batch.num_rows(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - - const auto& array_statistics_and_type_vector = ExtractArrayDataAndType(record_batch); - auto num_fields = static_cast(array_statistics_and_type_vector.size()); - for (int64_t nth_column = 0; nth_column < num_fields; ++nth_column) { - const auto& type = array_statistics_and_type_vector[nth_column].second; - auto column_statistics = array_statistics_and_type_vector[nth_column].first; - if (!column_statistics) { - continue; - } - - statistics.start_new_column = true; - statistics.nth_column = static_cast(nth_column); - if (column_statistics->null_count.has_value()) { - statistics.nth_statistics++; - statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT; - statistics.type = int64(); - statistics.value = column_statistics->null_count.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - - if (column_statistics->distinct_count.has_value()) { - statistics.nth_statistics++; - statistics.key = ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT; - statistics.type = int64(); - statistics.value = column_statistics->distinct_count.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - - if (column_statistics->min.has_value()) { - statistics.nth_statistics++; - if (column_statistics->is_min_exact) { - statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_EXACT; - } else { - statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE; - } - statistics.type = column_statistics->MinArrowType(type); - statistics.value = column_statistics->min.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - - if (column_statistics->max.has_value()) { - statistics.nth_statistics++; - if (column_statistics->is_max_exact) { - statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_EXACT; - } else { - statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE; - } - statistics.type = column_statistics->MaxArrowType(type); - statistics.value = column_statistics->max.value(); - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; - } - } - return Status::OK(); -} -} // namespace - Result> RecordBatch::MakeStatisticsArray( - MemoryPool* memory_pool) const { - // Statistics schema: - // struct< - // column: int32, - // statistics: map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - // > - - // Statistics schema doesn't define static dense union type for - // values. Each statistics schema have a dense union type that has - // needled value types. The following block collects these types. - std::vector> values_types; - std::vector values_type_indexes; - RETURN_NOT_OK(EnumerateStatistics(*this, [&](const EnumeratedStatistics& statistics) { - int8_t i = 0; - for (const auto& field : values_types) { - if (field->type()->Equals(statistics.type)) { - break; - } - i++; - } - if (i == static_cast(values_types.size())) { - values_types.push_back(field(statistics.type->name(), statistics.type)); - } - values_type_indexes.push_back(i); - return Status::OK(); - })); - - // statistics.key: dictionary - auto keys_type = dictionary(int32(), utf8(), false); - // statistics.items: dense_union<...all needed types...> - auto values_type = dense_union(values_types); - // struct< - // column: int32, - // statistics: map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - // > - auto statistics_type = - struct_({field("column", int32()), - field("statistics", map(keys_type, values_type, false))}); - - std::vector> field_builders; - // columns: int32 - auto columns_builder = std::make_shared(memory_pool); - field_builders.push_back(std::static_pointer_cast(columns_builder)); - // statistics.key: dictionary - auto keys_builder = std::make_shared(); - // statistics.items: dense_union<...all needed types...> - std::vector> values_builders; - for (const auto& values_type : values_types) { - std::unique_ptr values_builder; - RETURN_NOT_OK(MakeBuilder(memory_pool, values_type->type(), &values_builder)); - values_builders.push_back(std::shared_ptr(std::move(values_builder))); - } - auto items_builder = std::make_shared( - memory_pool, std::move(values_builders), values_type); - // statistics: - // map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - auto values_builder = std::make_shared( - memory_pool, std::static_pointer_cast(keys_builder), - std::static_pointer_cast(items_builder)); - field_builders.push_back(std::static_pointer_cast(values_builder)); - // struct< - // column: int32, - // statistics: map< - // key: dictionary< - // indices: int32, - // dictionary: utf8, - // >, - // items: dense_union<...all needed types...>, - // > - // > - StructBuilder builder(statistics_type, memory_pool, std::move(field_builders)); - - // Append statistics. - RETURN_NOT_OK(EnumerateStatistics(*this, [&](const EnumeratedStatistics& statistics) { - if (statistics.start_new_column) { - RETURN_NOT_OK(builder.Append()); - if (statistics.nth_column.has_value()) { - RETURN_NOT_OK(columns_builder->Append(statistics.nth_column.value())); - } else { - RETURN_NOT_OK(columns_builder->AppendNull()); - } - RETURN_NOT_OK(values_builder->Append()); - } - RETURN_NOT_OK(keys_builder->Append(statistics.key, - static_cast(strlen(statistics.key)))); - const auto values_type_index = values_type_indexes[statistics.nth_statistics]; - RETURN_NOT_OK(items_builder->Append(values_type_index)); - struct Visitor { - ArrayBuilder* builder; - - Status operator()(const bool& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const int64_t& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const uint64_t& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const double& value) { - return static_cast(builder)->Append(value); - } - Status operator()(const std::string& value) { - return static_cast(builder)->Append( - value.data(), static_cast(value.size())); - } - Status operator()(const std::shared_ptr& value) { - return builder->AppendScalar(*value); - } - } visitor; - visitor.builder = values_builders[values_type_index].get(); - RETURN_NOT_OK(std::visit(visitor, statistics.value)); - return Status::OK(); - })); - - return builder.Finish(); + const StatisticsArrayTOptions& options) const { + ArrayDataVector vector; + for (int i = 0; i < this->num_columns(); ++i) { + ARROW_ASSIGN_OR_RAISE( + auto extracted_columns, + internal::ExtractColumnsToArrayData(column(i), options.max_recursion_depth)); + vector.insert(vector.end(), std::make_move_iterator(extracted_columns.begin()), + std::make_move_iterator(extracted_columns.end())); + } + return internal::MakeStatisticsArray(options.memory_pool, std::move(vector), + this->num_rows()); } Status RecordBatch::Validate() const { diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h index 2b9fe0bda890..8c4c3ec2c83c 100644 --- a/cpp/src/arrow/record_batch.h +++ b/cpp/src/arrow/record_batch.h @@ -22,6 +22,7 @@ #include #include +#include "arrow/array/statistics_option.h" #include "arrow/compare.h" #include "arrow/device.h" #include "arrow/result.h" @@ -289,10 +290,11 @@ class ARROW_EXPORT RecordBatch { /// https://arrow.apache.org/docs/format/StatisticsSchema.html /// for details. /// - /// \param[in] pool the memory pool to allocate memory from + /// \param[in] options Options for creating a statistics array + /// /// \return the statistics array of this record batch Result> MakeStatisticsArray( - MemoryPool* pool = default_memory_pool()) const; + const StatisticsArrayTOptions& options = StatisticsArrayTOptions::Default()) const; protected: RecordBatch(const std::shared_ptr& schema, int64_t num_rows); diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index 408683ddc851..5767e601db5d 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -33,6 +33,8 @@ #include "arrow/array/array_dict.h" #include "arrow/array/array_nested.h" #include "arrow/array/data.h" +#include "arrow/array/statistics_option.h" +#include "arrow/array/statistics_test_util.h" #include "arrow/array/util.h" #include "arrow/c/abi.h" #include "arrow/chunked_array.h" @@ -1020,210 +1022,80 @@ TEST_F(TestRecordBatch, ToTensorUnsupportedMixedFloat16) { NotImplemented, "NotImplemented: Casting from or to halffloat is not supported.", batch1->ToTensor()); } - -namespace { -template ::value || - is_number_type::value>> -Result> BuildArray( - const std::vector::CType>& values) { - using BuilderType = typename TypeTraits::BuilderType; - BuilderType builder; - for (const auto& value : values) { - ARROW_RETURN_NOT_OK(builder.Append(value)); - } - return builder.Finish(); -} - -template > -Result> BuildArray(const std::vector& values) { - using BuilderType = typename TypeTraits::BuilderType; - BuilderType builder; - for (const auto& value : values) { - ARROW_RETURN_NOT_OK(builder.Append(value)); - } - return builder.Finish(); -} - -template -std::vector StatisticsValuesToRawValues( - const std::vector& values) { - std::vector raw_values; - for (const auto& value : values) { - raw_values.push_back(std::get(value)); - } - return raw_values; -} - -template ::value>> -Result> BuildArray(const std::vector& values) { - struct Builder { - const std::vector& values_; - explicit Builder(const std::vector& values) - : values_(values) {} - - Result> operator()(const bool&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const int64_t&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const uint64_t&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const double&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const std::string&) { - auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); - } - Result> operator()(const std::shared_ptr& scalar) { - auto values = StatisticsValuesToRawValues>(values_); - ARROW_ASSIGN_OR_RAISE(auto builder, MakeBuilder(scalar->type)); - ARROW_RETURN_NOT_OK(builder->Reserve(values.size())); - ARROW_RETURN_NOT_OK(builder->AppendScalars(values)); - return builder->Finish(); - } - } builder(values); - return std::visit(builder, values[0]); +namespace test { + +TEST_F(TestRecordBatch, MakeStatisticsArrayNesting) { + auto schema = ::arrow::schema({field("f0", int32())}); + ASSERT_OK_AND_ASSIGN(auto batch, RecordBatch::MakeEmpty(schema)); + auto options = StatisticsArrayTOptions::Default(); + options.max_recursion_depth = 1; + ASSERT_OK(CheckDepth(batch, options)); + options.max_recursion_depth = 0; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + CheckDepth(batch, options)); + + schema = ::arrow::schema({field("f0", int64()), field("f1", int64())}); + batch = RecordBatchFromJSON( + schema, R"([{"f0": 1, "f1": 4},{"f0": 2, "f1": 5},{"f0": 3, "f1": 6}])"); + options = StatisticsArrayTOptions::Default(); + options.max_recursion_depth = 1; + ASSERT_OK(CheckDepth(batch, options)); + options.max_recursion_depth = 0; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + CheckDepth(batch, options)); + + options.max_recursion_depth = 2; + ASSERT_OK_AND_ASSIGN(auto multi_children_struct, + MakeNestedStruct(options.max_recursion_depth)); + schema = + ::arrow::schema({field("multi_children_struct", multi_children_struct->type())}); + batch = + RecordBatch::Make(schema, multi_children_struct->length(), {multi_children_struct}); + + options.max_recursion_depth = 1; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + CheckDepth(batch, options)); + + options.max_recursion_depth = 2; + ASSERT_OK(CheckDepth(batch, options)); + + auto array_a = ArrayFromJSON(int64(), R"([1,2,3])"); + schema = ::arrow::schema({field("a", int64()), field("multi_children_struct", + multi_children_struct->type())}); + batch = RecordBatch::Make(schema, 3, {array_a, multi_children_struct}); + + options.max_recursion_depth = 1; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + CheckDepth(batch, options)); + + options.max_recursion_depth = 2; + ASSERT_OK(CheckDepth(batch, options)); } -Result> MakeStatisticsArray( - const std::string& columns_json, - const std::vector>& nested_statistics_keys, - const std::vector>& - nested_statistics_values) { - auto columns_type = int32(); - auto columns_array = ArrayFromJSON(columns_type, columns_json); - const auto n_columns = columns_array->length(); - - // nested_statistics_keys: - // { - // {"ARROW:row_count:exact", "ARROW:null_count:exact"}, - // {"ARROW:max_value:exact"}, - // {"ARROW:max_value:exact", "ARROW:distinct_count:exact"}, - // } - // nested_statistics_values: - // { - // {int64_t{29}, int64_t{1}}, - // {double{2.9}}, - // {double{-2.9}, int64_t{2}}, - // } - // -> - // keys_dictionary: - // { - // "ARROW:row_count:exact", - // "ARROW:null_count:exact", - // "ARROW:max_value:exact", - // "ARROW:distinct_count:exact", - // } - // keys_indices: {0, 1, 2, 2, 3} - // values_types: {int64(), float64()} - // values_type_codes: {0, 1} - // values_values[0]: {int64_t{29}, int64_t{1}, int64_t{2}} - // values_values[1]: {double{2.9}, double{-2.9}} - // values_value_type_ids: {0, 0, 1, 1, 0} - // values_value_offsets: {0, 1, 0, 1, 2} - // statistics_offsets: {0, 2, 3, 5, 5} - std::vector keys_dictionary; - std::vector keys_indices; - std::vector> values_types; - std::vector values_type_codes; - std::vector> values_values; - std::vector values_value_type_ids; - std::vector values_value_offsets; - std::vector statistics_offsets; - - int32_t offset = 0; - std::vector values_value_offset_counters; - for (size_t i = 0; i < nested_statistics_keys.size(); ++i) { - const auto& statistics_keys = nested_statistics_keys[i]; - const auto& statistics_values = nested_statistics_values[i]; - statistics_offsets.push_back(offset); - for (size_t j = 0; j < statistics_keys.size(); ++j) { - const auto& key = statistics_keys[j]; - const auto& value = statistics_values[j]; - ++offset; - - int32_t key_index = 0; - for (; key_index < static_cast(keys_dictionary.size()); ++key_index) { - if (keys_dictionary[key_index] == key) { - break; - } - } - if (key_index == static_cast(keys_dictionary.size())) { - keys_dictionary.push_back(key); - } - keys_indices.push_back(key_index); - - auto values_type = ArrayStatistics::ValueToArrowType(value, arrow::null()); - int8_t values_type_code = 0; - for (; values_type_code < static_cast(values_types.size()); - ++values_type_code) { - if (values_types[values_type_code] == values_type) { - break; - } - } - if (values_type_code == static_cast(values_types.size())) { - values_types.push_back(values_type); - values_type_codes.push_back(values_type_code); - values_values.emplace_back(); - values_value_offset_counters.push_back(0); - } - values_values[values_type_code].push_back(value); - values_value_type_ids.push_back(values_type_code); - values_value_offsets.push_back(values_value_offset_counters[values_type_code]++); - } - } - statistics_offsets.push_back(offset); - - auto keys_type = dictionary(int32(), utf8(), false); - std::vector> values_fields; - for (const auto& type : values_types) { - values_fields.push_back(field(type->name(), type)); - } - auto values_type = dense_union(values_fields); - auto statistics_type = map(keys_type, values_type, false); - auto struct_type = - struct_({field("column", columns_type), field("statistics", statistics_type)}); - - ARROW_ASSIGN_OR_RAISE(auto keys_indices_array, BuildArray(keys_indices)); - ARROW_ASSIGN_OR_RAISE(auto keys_dictionary_array, - BuildArray(keys_dictionary)); - ARROW_ASSIGN_OR_RAISE( - auto keys_array, - DictionaryArray::FromArrays(keys_type, keys_indices_array, keys_dictionary_array)); - - std::vector> values_arrays; - for (const auto& values : values_values) { - ARROW_ASSIGN_OR_RAISE(auto values_array, - BuildArray(values)); - values_arrays.push_back(values_array); - } - ARROW_ASSIGN_OR_RAISE(auto values_value_type_ids_array, - BuildArray(values_value_type_ids)); - ARROW_ASSIGN_OR_RAISE(auto values_value_offsets_array, - BuildArray(values_value_offsets)); - auto values_array = std::make_shared( - values_type, values_value_offsets_array->length(), values_arrays, - values_value_type_ids_array->data()->buffers[1], - values_value_offsets_array->data()->buffers[1]); - ARROW_ASSIGN_OR_RAISE(auto statistics_offsets_array, - BuildArray(statistics_offsets)); - ARROW_ASSIGN_OR_RAISE(auto statistics_array, - MapArray::FromArrays(statistics_type, statistics_offsets_array, - keys_array, values_array)); - std::vector> struct_arrays = {std::move(columns_array), - std::move(statistics_array)}; - return std::make_shared(struct_type, n_columns, struct_arrays); +TEST_F(TestRecordBatch, MakeStatisticsArrayNestingStressTest) { + ASSERT_OK_AND_ASSIGN(auto deep_children_struct, MakeNestedStruct(100)); + auto schema = ::arrow::schema({field("struct", deep_children_struct->type())}); + auto batch = + RecordBatch::Make(schema, deep_children_struct->length(), {deep_children_struct}); + StatisticsArrayTOptions options; + options.max_recursion_depth = 100; + ASSERT_OK(CheckDepth(batch, options)); + options.max_recursion_depth = 99; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + batch->MakeStatisticsArray(options)); + +#if !defined(ARROW_VALGRIND) + ASSERT_OK_AND_ASSIGN(deep_children_struct, MakeNestedStruct(500)); + schema = ::arrow::schema({field("struct", deep_children_struct->type())}); + batch = + RecordBatch::Make(schema, deep_children_struct->length(), {deep_children_struct}); + options.max_recursion_depth = 500; + ASSERT_OK(CheckDepth(batch, options)); + options.max_recursion_depth = 499; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + batch->MakeStatisticsArray(options)); +#endif } -}; // namespace TEST_F(TestRecordBatch, MakeStatisticsArrayRowCount) { auto schema = ::arrow::schema({field("int32", int32())}); @@ -1233,13 +1105,40 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayRowCount) { ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }})); + MakeMockStatisticsArray("[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }})); + AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + + // I'm not sure whether the following states are valid + batch = RecordBatch::Make(schema, 1, {int32_array}); + ASSERT_OK_AND_ASSIGN(statistics_array, batch->MakeStatisticsArray()); + + ASSERT_OK_AND_ASSIGN(expected_statistics_array, + MakeMockStatisticsArray("[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{1}}, + }})); + AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + + batch = RecordBatch::Make(schema, 10, {int32_array}); + ASSERT_OK_AND_ASSIGN(statistics_array, batch->MakeStatisticsArray()); + ASSERT_OK_AND_ASSIGN( + expected_statistics_array, + MakeMockStatisticsArray("[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{10}}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1256,20 +1155,21 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNullCount) { ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{int64_t{1}}, - }})); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{int64_t{1}}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1287,22 +1187,23 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayDistinctCount) { ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{int64_t{1}}, - ArrayStatistics::ValueType{int64_t{2}}, - }})); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{int64_t{1}}, + ArrayStatistics::ValueType{int64_t{2}}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1320,20 +1221,21 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMinExact) { ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MIN_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{uint64_t{1}}, - }})); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MIN_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{uint64_t{1}}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1352,19 +1254,19 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMinApproximate) { ASSERT_OK_AND_ASSIGN( auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{-1.0}, - }})); + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{-1.0}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1383,20 +1285,21 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMaxExact) { ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{true}, - }})); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{true}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1415,19 +1318,19 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMaxApproximate) { ASSERT_OK_AND_ASSIGN( auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{1.0}, - }})); + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{1.0}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } @@ -1445,111 +1348,206 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayString) { ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{"c"}, - }})); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + MakeMockStatisticsArray("[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{"c"}, + }})); AssertArraysEqual(*expected_statistics_array, *statistics_array, true); } +// Schema: +// struct_a_b: struct {a: int64, b: int64} +// c: int64 +// d: int64 +// struct_e_f: struct {e: int32, f: int32} +// +// Data: +// [ +// {struct_a_b: {a: 1, b: 6 }, c: 11, d: 16, struct_e_f: {a: 21, b: 26}}, +// {struct_a_b: {a: 2, b: 7 }, c: 12, d: 17, struct_e_f: {a: 22, b: 27}}, +// {struct_a_b: {a: 3, b: 8 }, c: 13, d: 18, struct_e_f: {a: 23, b: 28}}, +// {struct_a_b: {a: 4, b: 9 }, c: 14, d: 19, struct_e_f: {a: 24, b: 29}}, +// {struct_a_b: {a: 5, b: 10}, c: 15, d: 20, struct_e_f: {a: 25, b: 30}} +// ] +// +// Statistics: +// batch: +// row_count_exact: 5 +// struct_a_b: +// null_count_exact: 0 +// max_approximate: {a: 5, b: 10} +// struct_a_b.a: +// min_approximate: 1 +// array_c: +// max_approximate: 15 +// struct_e_f.e: +// max_exact: 30 TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { - auto struct_type = struct_({field("a", int64()), field("b", int64())}); - auto struct_array = ArrayFromJSON( - struct_type, - R"([{"a":1,"b":6},{"a":2,"b":7},{"a":3,"b":8},{"a":4,"b":9},{"a":5,"b":10}])"); - ASSERT_OK_AND_ASSIGN(auto struct_nested_stat, - struct_array->CopyTo(default_cpu_memory_manager())); - auto statistics_struct = std::make_shared(); - ASSERT_OK_AND_ASSIGN(statistics_struct->max, struct_array->GetScalar(4)); - statistics_struct->null_count = 0; - auto struct_array_data = struct_array->data(); - auto statistics_struct_child_a = std::make_shared(); - statistics_struct_child_a->min = int64_t{1}; - struct_array_data->statistics = statistics_struct; - struct_array_data->child_data[0]->statistics = statistics_struct_child_a; - auto array_c = ArrayFromJSON(int64(), R"([11,12,13,14,15])"); + const auto struct_a_b_type = struct_({field("a", int64()), field("b", int64())}); + const auto struct_a_b = ArrayFromJSON( + struct_a_b_type, + R"([{"a": 1, "b": 6}, {"a": 2, "b": 7}, {"a": 3,"b": 8}, {"a": 4,"b": 9}, {"a": 5, "b": 10}])"); + const auto struct_a_b_statistics = std::make_shared(); + ASSERT_OK_AND_ASSIGN(struct_a_b_statistics->max, struct_a_b->GetScalar(4)); + struct_a_b_statistics->null_count = 0; + struct_a_b->data()->statistics = struct_a_b_statistics; + const auto struct_a_b_child_a_statistics = std::make_shared(); + struct_a_b_child_a_statistics->min = int64_t{1}; + struct_a_b->data()->child_data[0]->statistics = struct_a_b_child_a_statistics; + const auto array_c = ArrayFromJSON(int64(), R"([11, 12, 13, 14, 15])"); array_c->data()->statistics = std::make_shared(); array_c->data()->statistics->max = int64_t{15}; - auto array_d = ArrayFromJSON(int64(), R"([16,17,18,19,20])"); - auto nested_child = struct_nested_stat->data()->child_data[0]; - nested_child->statistics = std::make_shared(); - nested_child->statistics->max = int64_t{5}; - nested_child->statistics->is_max_exact = true; - - auto rb_schema = - schema({field("struct_a_b", struct_type), field("c", int64()), field("d", int64()), - field("struct_copy", struct_nested_stat->type())}); - auto rb = RecordBatch::Make(rb_schema, 5, - {struct_array, array_c, array_d, struct_nested_stat}); - - auto expected_scalar = internal::checked_pointer_cast( - ScalarFromJSON(struct_type, R"([5,10])")); - auto a = ArrayStatistics::ValueType{std::static_pointer_cast(expected_scalar)}; + const auto array_d = ArrayFromJSON(int64(), R"([16, 17, 18, 19, 20])"); + const auto struct_e_f_type = struct_({field("e", int32()), field("f", int32())}); + const auto struct_e_f = ArrayFromJSON( + struct_e_f_type, + R"([{"e": 21, "f": 26}, {"e": 22,"f": 27}, {"e": 23, "f": 28}, {"e": 24, "f": 29}, {"e": 25,"f": 30}])"); + const auto struct_e_f_child_e_statistics = struct_e_f->data()->child_data[0]; + struct_e_f_child_e_statistics->statistics = std::make_shared(); + struct_e_f_child_e_statistics->statistics->max = int64_t{30}; + struct_e_f_child_e_statistics->statistics->is_max_exact = true; + + const auto expected_scalar = ScalarFromJSON(struct_a_b_type, R"([5 ,10])"); ASSERT_OK_AND_ASSIGN( - auto expected_array, - MakeStatisticsArray("[null,0,1,3,6]", - {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, - {ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, - {{ArrayStatistics::ValueType{int64_t{5}}}, - {ArrayStatistics::ValueType{int64_t{0}}, - ArrayStatistics::ValueType{ - std::static_pointer_cast(expected_scalar)}}, - {ArrayStatistics::ValueType{int64_t{1}}}, - {ArrayStatistics::ValueType{int64_t{15}}}, - {ArrayStatistics::ValueType{int64_t{5}}}})); - ASSERT_OK_AND_ASSIGN(auto rb_stat, rb->MakeStatisticsArray()); - AssertArraysEqual(*expected_array, *rb_stat, true); + const auto expected_array_statistics, + MakeMockStatisticsArray("[null , 0, 1, 3, 6]", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{int64_t{0}}, + ArrayStatistics::ValueType{expected_scalar}}, + {ArrayStatistics::ValueType{int64_t{1}}}, + {ArrayStatistics::ValueType{int64_t{15}}}, + {ArrayStatistics::ValueType{int64_t{30}}}})); + + const auto schema = + ::arrow::schema({field("struct_a_b", struct_a_b_type), field("c", int64()), + field("d", int64()), field("struct_e_f", struct_e_f->type())}); + const auto batch = + RecordBatch::Make(schema, 5, {struct_a_b, array_c, array_d, struct_e_f}); + ASSERT_OK_AND_ASSIGN(auto batch_statistics, batch->MakeStatisticsArray()); + + AssertArraysEqual(*expected_array_statistics, *batch_statistics, true); } +// Schema +// struct_parent: {struct_nested_0: {a: int32, b: int32}, +// struct_nested_1: {c: int32, d: int32}, +// struct_nested_2: {a: int32, b: int32}} +// +// Data: +// [ +// struct_parent: {struct_nested_0: {a: 1, b: 6 }, struct_nested_1: {c: 1, d: 6 }, +// struct_nested_2: {a: 1, b: 6 }}, +// struct_parent: {struct_nested_0: {a: 2, b: 7 }, struct_nested_1: {c: 2, d: 7 }, +// struct_nested_2: {a: 2, b: 7 }}, +// struct_parent: {struct_nested_0: {a: 3, b: 8 }, struct_nested_1: {c: 3, d: 8 }, +// struct_nested_2: {a: 3, b: 8 }}, +// struct_parent: {struct_nested_0: {a: 4, b: 9 }, struct_nested_1: {c: 4, d: 9 }, +// struct_nested_2: {a: 4, b: 9 }}, +// struct_parent: {struct_nested_0: {a: 5, b: 10}, struct_nested_1: {c: 5, d: 10}} +// struct_nested_2: {a: 5, b: 10}} +// ] +// +// Statistics: +// batch: +// row_count_exact: 5 +// struct_parent: +// min_approximate: {struct_nested_0: {a: 1, b: 6 }, struct_nested_1: {c: 1, d: 6 }, +// struct_nested_2: {a: 1, b: 6 }}, +// struct_nested_0: +// max_approximate: {a:5, b: 10} +// struct_nested_1: +// max_exact: {c:5, d: 10} +// struct_nested_2: +// min_exact: {a:5, b: 4} +// TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { - // TODO add statisics - // TODO TRY with Type 32 - auto struct_type = struct_({field("a", int32()), field("b", int32())}); - auto struct_nested_0 = ArrayFromJSON( - struct_type, - R"([{"a":1,"b":6},{"a":2,"b":7},{"a":3,"b":8},{"a":4,"b":9},{"a":5,"b":10}])"); - - ASSERT_OK_AND_ASSIGN(auto struct_nested_1, - struct_nested_0->CopyTo(default_cpu_memory_manager())) + const auto struct_nested_0_type = struct_({field("a", int32()), field("b", int32())}); + const auto struct_nested_0 = ArrayFromJSON( + struct_nested_0_type, + R"([{"a": 1,"b": 6}, {"a": 2, "b": 7}, {"a": 3, "b": 8}, {"a": 4, "b": 9}, {"a": 5,"b": 10}])"); + struct_nested_0->data()->statistics = std::make_shared(); + ASSERT_OK_AND_ASSIGN(struct_nested_0->data()->statistics->max, + struct_nested_0->GetScalar(4)); + const auto struct_nested_1_type = struct_({field("c", int32()), field("d", int32())}); + const auto struct_nested_1 = ArrayFromJSON( + struct_nested_1_type, + R"([{"c": 1, "d": 6}, {"c": 2, "d": 7}, {"c": 3, "d": 8}, {"c": 4, "d": 9}, {"c": 5,"d": 10}])"); struct_nested_1->data()->statistics = std::make_shared(); struct_nested_1->data()->statistics->is_max_exact = true; ASSERT_OK_AND_ASSIGN(struct_nested_1->data()->statistics->max, struct_nested_1->GetScalar(4)); + auto struct_nested_2_type = struct_({field("a", int32()), field("b", int32())}); + const auto struct_nested_2 = ArrayFromJSON( + struct_nested_0_type, + R"([{"a": 1,"b": 6}, {"a": 2, "b": 7}, {"a": 3, "b": 8}, {"a": 4, "b": 9}, {"a": 5,"b": 10}])"); + struct_nested_2->data()->statistics = std::make_shared(); + struct_nested_2->data()->statistics->is_min_exact = true; + ASSERT_OK_AND_ASSIGN(struct_nested_2->data()->statistics->min, + struct_nested_2->GetScalar(0)); + ASSERT_OK_AND_ASSIGN( - auto struct_parent, - StructArray::Make({struct_nested_0, struct_nested_1}, + const auto struct_parant, + StructArray::Make({struct_nested_0, struct_nested_1, struct_nested_2}, {field("struct_nested_0", struct_nested_0->type()), - field("struct_nested_1", struct_nested_1->type())})); - auto expected_scalar = internal::checked_pointer_cast( - ScalarFromJSON(struct_type, R"([5,10])")); - auto rb_schema = schema({field("struct", struct_parent->type())}); - auto rb = RecordBatch::Make(rb_schema, 5, {struct_parent}); - - ASSERT_OK_AND_ASSIGN(auto expected_array, - MakeStatisticsArray(R"([null,4])", - {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, - {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, - {{int64_t{5}}, {expected_scalar}})); - ASSERT_OK_AND_ASSIGN(auto rb_stat, rb->MakeStatisticsArray()); - AssertArraysEqual(*expected_array, *rb_stat, true); + field("struct_nested_1", struct_nested_1->type()), + field("struct_nested_2", struct_nested_2->type())})); + struct_parant->data()->statistics = std::make_shared(); + ASSERT_OK_AND_ASSIGN(struct_parant->data()->statistics->min, + struct_parant->GetScalar(0)); + + const auto expected_scalar_parent = + ScalarFromJSON(struct_parant->type(), R"([[1 ,6], [1, 6], [1, 6]])"); + + const auto expected_scalar_nested_0 = + ScalarFromJSON(struct_nested_0_type, R"([5, 10])"); + + const auto expected_scalar_nested_1 = + ScalarFromJSON(struct_nested_1_type, R"([5, 10])"); + + const auto expected_scalar_nested_2 = ScalarFromJSON(struct_nested_2_type, R"([1, 6])"); + + ASSERT_OK_AND_ASSIGN( + const auto expected_array_statistics, + MakeMockStatisticsArray(R"([null, 0, 1, 4, 7])", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}, + {ARROW_STATISTICS_KEY_MIN_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{expected_scalar_parent}}, + {ArrayStatistics::ValueType{expected_scalar_nested_0}}, + {ArrayStatistics::ValueType{expected_scalar_nested_1}}, + {expected_scalar_nested_2}})); + + const auto schema = ::arrow::schema({field("struct", struct_parant->type())}); + const auto batch = RecordBatch::Make(schema, 5, {struct_parant}); + ASSERT_OK_AND_ASSIGN(const auto batch_statistics, batch->MakeStatisticsArray()); + + AssertArraysEqual(*expected_array_statistics, *batch_statistics, true); } +} // namespace test + template class TestBatchToTensorColumnMajor : public ::testing::Test {}; diff --git a/docs/source/cpp/api/array.rst b/docs/source/cpp/api/array.rst index b17d1957a8b6..12e07834705a 100644 --- a/docs/source/cpp/api/array.rst +++ b/docs/source/cpp/api/array.rst @@ -110,3 +110,10 @@ Utilities :project: arrow_cpp :members: :undoc-members: + +Statistics options +================== + +.. doxygenstruct:: arrow::StatisticsArrayTOptions + :project: arrow_cpp + :members: From 4d265170b2b3e47695a38b2781f33f26849148db Mon Sep 17 00:00:00 2001 From: Arash Andishgar Date: Mon, 31 Mar 2025 16:36:10 +0330 Subject: [PATCH 4/4] fix string builder issue, add a new class for testing fix export --- cpp/src/arrow/array/statistics.cc | 82 +++- cpp/src/arrow/array/statistics.h | 19 +- cpp/src/arrow/array/statistics_test.cc | 1 + cpp/src/arrow/array/statistics_test_util.cc | 112 +++-- cpp/src/arrow/array/statistics_test_util.h | 52 +- cpp/src/arrow/record_batch.cc | 21 +- cpp/src/arrow/record_batch.h | 2 +- cpp/src/arrow/record_batch_test.cc | 515 ++++++++++---------- 8 files changed, 431 insertions(+), 373 deletions(-) diff --git a/cpp/src/arrow/array/statistics.cc b/cpp/src/arrow/array/statistics.cc index 599f62d8ff90..e952d01afb4a 100644 --- a/cpp/src/arrow/array/statistics.cc +++ b/cpp/src/arrow/array/statistics.cc @@ -38,10 +38,13 @@ #include "arrow/scalar.h" #include "arrow/status.h" #include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/util/macros.h" +#include "arrow/visit_type_inline.h" namespace arrow { -const std::shared_ptr& ArrayStatistics::ValueToArrowType( +Result> ArrayStatistics::ValueToArrowType( const std::optional& value, const std::shared_ptr& array_type) { if (!value.has_value()) { @@ -51,24 +54,25 @@ const std::shared_ptr& ArrayStatistics::ValueToArrowType( struct Visitor { const std::shared_ptr& array_type; - const std::shared_ptr& operator()(const bool&) { return boolean(); } - const std::shared_ptr& operator()(const int64_t&) { return int64(); } - const std::shared_ptr& operator()(const uint64_t&) { return uint64(); } - const std::shared_ptr& operator()(const double&) { return float64(); } - const std::shared_ptr& operator()(const std::shared_ptr& value) { + Result> operator()(const bool&) { return boolean(); } + Result> operator()(const int64_t&) { return int64(); } + Result> operator()(const uint64_t&) { return uint64(); } + Result> operator()(const double&) { return float64(); } + Result> operator()(const std::shared_ptr& value) { return value->type; } - const std::shared_ptr& operator()(const std::string&) { + Result> operator()(const std::string&) { switch (array_type->id()) { - // TODO Add StringView and BinaryView to ArrayStatistics (GH-45664) case Type::STRING: case Type::BINARY: case Type::FIXED_SIZE_BINARY: case Type::LARGE_STRING: case Type::LARGE_BINARY: + case Type::STRING_VIEW: + case Type::BINARY_VIEW: return array_type; default: - return utf8(); + return Status::Invalid("type should be specified for string"); } } } visitor{array_type}; @@ -110,10 +114,10 @@ bool ArrayStatistics::Equals(const ArrayStatistics& other, ValueTypeEquality(this->min, other.min, options); } namespace internal { -// Perform a depth-first search (DFS) on the given array to extract -// all nested arrays up to the specified maximum nesting depth. -Result ExtractColumnsToArrayData(const std::shared_ptr& array, - const int32_t& max_nesting_depth) { +// Perform a depth-first search (DFS) on the given array data to extract +// all nested array data up to the specified maximum nesting depth. +Result ExtractColumnsToArrayData( + const std::shared_ptr& array_data, const int32_t& max_nesting_depth) { ArrayDataVector traverse; ArrayDataVector result; @@ -121,17 +125,17 @@ Result ExtractColumnsToArrayData(const std::shared_ptr& if (remaining_nested_depth <= 0) { return Status::Invalid("Max recursion depth reached"); } - result.push_back(array->data()); - if (!array->data()->child_data.empty()) { + result.push_back(array_data); + if (!array_data->child_data.empty()) { // depth delimiter traverse.emplace_back(nullptr); - traverse.insert(traverse.end(), array->data()->child_data.crbegin(), - array->data()->child_data.crend()); + traverse.insert(traverse.end(), array_data->child_data.crbegin(), + array_data->child_data.crend()); remaining_nested_depth--; } while (!traverse.empty()) { - if (remaining_nested_depth <= 0) { + if (ARROW_PREDICT_FALSE(remaining_nested_depth <= 0)) { return Status::Invalid("Max recursion depth reached"); } @@ -177,13 +181,17 @@ Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, statistics.type = int64(); statistics.value = ArrayStatistics::ValueType{int64_t{0}}; if (num_rows.has_value()) { + // handle Record Batch statistics.value = ArrayStatistics::ValueType{num_rows.value()}; + RETURN_NOT_OK(on_statistics(statistics)); + statistics.start_new_column = false; } else if (!extracted_array_data.empty()) { + // handle parent array statistics.value = ArrayStatistics::ValueType{int64_t{extracted_array_data[0]->length}}; + statistics.nth_column = int32_t{0}; + RETURN_NOT_OK(on_statistics(statistics)); } - RETURN_NOT_OK(on_statistics(statistics)); - statistics.start_new_column = false; auto num_fields = static_cast(extracted_array_data.size()); for (int64_t nth_column = 0; nth_column < num_fields; ++nth_column) { @@ -192,9 +200,13 @@ Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, if (!column_statistics) { continue; } - statistics.start_new_column = true; statistics.nth_column = static_cast(nth_column); + if (nth_column == 0 && !num_rows.has_value()) { + // handle parent array + statistics.start_new_column = false; + } + if (column_statistics->null_count.has_value()) { statistics.nth_statistics++; statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT; @@ -220,7 +232,7 @@ Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, } else { statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE; } - statistics.type = column_statistics->MinArrowType(type); + ARROW_ASSIGN_OR_RAISE(statistics.type, column_statistics->MinArrowType(type)); statistics.value = column_statistics->min.value(); RETURN_NOT_OK(on_statistics(statistics)); statistics.start_new_column = false; @@ -233,7 +245,7 @@ Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, } else { statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE; } - statistics.type = column_statistics->MaxArrowType(type); + ARROW_ASSIGN_OR_RAISE(statistics.type, column_statistics->MaxArrowType(type)); statistics.value = column_statistics->max.value(); RETURN_NOT_OK(on_statistics(statistics)); statistics.start_new_column = false; @@ -241,6 +253,23 @@ Status EnumerateStatistics(const ArrayDataVector& extracted_array_data, } return Status::OK(); } + +struct StringVisitorBuilder { + template ::BuilderType> + enable_if_t::value || + is_binary_view_like_type::value || + std::is_same_v, + Status> + Visit(const DataType&, ArrayBuilder* raw_builder, const std::string& value) { + auto* builder = static_cast(raw_builder); + return builder->Append(value); + } + Status Visit(const DataType& type, ArrayBuilder* raw_builder, + const std::string& value) { + return Status::Invalid("The type is ", type.ToString()); + } +}; } // namespace Result> MakeStatisticsArray( @@ -355,7 +384,7 @@ Result> MakeStatisticsArray( RETURN_NOT_OK(items_builder->Append(values_type_index)); struct Visitor { ArrayBuilder* builder; - + DataType* type; Status operator()(const bool& value) { return static_cast(builder)->Append(value); } @@ -369,14 +398,15 @@ Result> MakeStatisticsArray( return static_cast(builder)->Append(value); } Status operator()(const std::string& value) { - return static_cast(builder)->Append( - value.data(), static_cast(value.size())); + auto string_visitor_builder = StringVisitorBuilder(); + return VisitTypeInline(*type, &string_visitor_builder, builder, value); } Status operator()(const std::shared_ptr& value) { return builder->AppendScalar(*value); } } visitor; visitor.builder = values_builders[values_type_index].get(); + visitor.type = statistics.type.get(); RETURN_NOT_OK(std::visit(visitor, statistics.value)); return Status::OK(); })); diff --git a/cpp/src/arrow/array/statistics.h b/cpp/src/arrow/array/statistics.h index 07f8d6c286be..75c6a18070bc 100644 --- a/cpp/src/arrow/array/statistics.h +++ b/cpp/src/arrow/array/statistics.h @@ -43,7 +43,7 @@ struct ARROW_EXPORT ArrayStatistics { using ValueType = std::variant>; - static const std::shared_ptr& ValueToArrowType( + static Result> ValueToArrowType( const std::optional& value, const std::shared_ptr& array_type); /// \brief The number of null values, may not be set @@ -69,7 +69,7 @@ struct ARROW_EXPORT ArrayStatistics { /// \return \ref arrow::null if the minimum value is `std::nullopt`, /// Arrow type based on \ref ValueType of the \ref min /// otherwise. - const std::shared_ptr& MinArrowType( + Result> MinArrowType( const std::shared_ptr& array_type) { return ValueToArrowType(min, array_type); } @@ -94,7 +94,7 @@ struct ARROW_EXPORT ArrayStatistics { /// \return \ref arrow::null if the maximum value is `std::nullopt`, /// Arrow type based on \ref ValueType of the \ref max /// otherwise. - const std::shared_ptr& MaxArrowType( + Result> MaxArrowType( const std::shared_ptr& array_type) { return ValueToArrowType(max, array_type); } @@ -124,21 +124,24 @@ namespace internal { /// Returns all nested arrays within the given array, up to the specified /// maximum nesting depth, as a vector of ArrayData. /// -/// \param[in] array The input array from which nested arrays will be extracted. +/// \param[in] array_data The input array_data from which nested array_data will be +/// extracted. /// \param[in] max_nesting_depth The maximum depth of nested arrays to extract. /// /// \return A vector of \ref ArrayData . ARROW_EXPORT -Result ExtractColumnsToArrayData(const std::shared_ptr& array, - const int32_t& max_nesting_depth); +Result ExtractColumnsToArrayData( + const std::shared_ptr& array_data, const int32_t& max_nesting_depth); /// \brief Collect Statistics From ArrayDataVector and turns into an Array /// /// \param[in] memory_pool the memory pool to allocate memory from /// \param extracted_array_data_vector The input ArrayData from which statistics array is /// extracted /// -/// \param num_rows The input specifies the number of rows an object if it is set to -/// std::nullopt, the number of rows is deduced from the length of the first array +/// \param num_rows The input specifies the number of rows an object(which uses for +/// record batch), if it is set to std::nullopt(which uses for array), +/// the number of rows is deduced from the +/// length of the first array /// /// \return A statistics array. ARROW_EXPORT diff --git a/cpp/src/arrow/array/statistics_test.cc b/cpp/src/arrow/array/statistics_test.cc index 043d4149b081..ba7e9ee4ae26 100644 --- a/cpp/src/arrow/array/statistics_test.cc +++ b/cpp/src/arrow/array/statistics_test.cc @@ -19,6 +19,7 @@ #include #include "arrow/array/statistics.h" +#include "arrow/compare.h" #include "arrow/scalar.h" #include "arrow/testing/gtest_util.h" diff --git a/cpp/src/arrow/array/statistics_test_util.cc b/cpp/src/arrow/array/statistics_test_util.cc index f09dea8da18c..695b21c076b7 100644 --- a/cpp/src/arrow/array/statistics_test_util.cc +++ b/cpp/src/arrow/array/statistics_test_util.cc @@ -18,11 +18,13 @@ #include "arrow/array/statistics_test_util.h" -#include - +#include +#include #include #include #include +#include +#include #include #include "arrow/array/array_base.h" @@ -32,12 +34,14 @@ #include "arrow/array/builder_binary.h" #include "arrow/array/builder_nested.h" #include "arrow/array/builder_primitive.h" +#include "arrow/array/data.h" #include "arrow/array/statistics.h" -#include "arrow/array/statistics_option.h" #include "arrow/result.h" #include "arrow/scalar.h" #include "arrow/testing/gtest_util.h" +#include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/visit_type_inline.h" namespace arrow { namespace { @@ -54,14 +58,43 @@ Result> BuildArray( return builder.Finish(); } -template > -Result> BuildArray(const std::vector& values) { - using BuilderType = typename TypeTraits::BuilderType; - BuilderType builder; - for (const auto& value : values) { - ARROW_RETURN_NOT_OK(builder.Append(value)); +struct StringVisitorBuilder { + template ::BuilderType> + enable_if_t::value, Status> Visit( + const DataType&, ArrayBuilder* raw_builder, + const std::vector& strings) { + auto* builder = static_cast(raw_builder); + ARROW_RETURN_NOT_OK(builder->AppendValues(strings)); + return Status::OK(); } - return builder.Finish(); + + template ::BuilderType> + enable_if_t::value || + std::is_same_v, + Status> + Visit(const DataType&, ArrayBuilder* raw_builder, + const std::vector& strings) { + auto* builder = static_cast(raw_builder); + for (const auto& string : strings) { + ARROW_RETURN_NOT_OK(builder->Append(string)); + } + return Status::OK(); + } + + Status Visit(const DataType& type, ArrayBuilder*, const std::vector&) { + return Status::Invalid("the type is ", type.ToString()); + } +}; + +Result> BuildArray(const std::vector& values, + const std::shared_ptr& type) { + std::unique_ptr builder; + ARROW_RETURN_NOT_OK(MakeBuilder(default_memory_pool(), type, &builder)); + StringVisitorBuilder visitor; + ARROW_RETURN_NOT_OK(VisitTypeInline(*type, &visitor, builder.get(), values)); + return builder->Finish(); } template @@ -76,11 +109,14 @@ std::vector StatisticsValuesToRawValues( template ::value>> -Result> BuildArray(const std::vector& values) { +Result> BuildArray(const std::vector& values, + const std::shared_ptr& string_type) { struct Builder { const std::vector& values_; - explicit Builder(const std::vector& values) - : values_(values) {} + const std::shared_ptr& type; + explicit Builder(const std::vector& values, + const std::shared_ptr& type) + : values_(values), type(type) {} Result> operator()(const bool&) { auto values = StatisticsValuesToRawValues(values_); @@ -100,7 +136,7 @@ Result> BuildArray(const std::vector& values) } Result> operator()(const std::string&) { auto values = StatisticsValuesToRawValues(values_); - return BuildArray(values); + return BuildArray(values, type); } Result> operator()(const std::shared_ptr& scalar) { auto values = StatisticsValuesToRawValues>(values_); @@ -109,16 +145,16 @@ Result> BuildArray(const std::vector& values) ARROW_RETURN_NOT_OK(builder->AppendScalars(values)); return builder->Finish(); } - } builder(values); + } builder(values, string_type); return std::visit(builder, values[0]); } } // namespace -namespace test { -Result> MakeMockStatisticsArray( + +Result> detail::MakeMockStatisticsArray( const std::string& columns_json, const std::vector>& nested_statistics_keys, - const std::vector>& - nested_statistics_values) { + const std::vector>& nested_statistics_values, + const std::vector>& string_types) { const auto& columns_type = int32(); auto columns_array = ArrayFromJSON(columns_type, columns_json); const auto n_columns = columns_array->length(); @@ -165,6 +201,11 @@ Result> MakeMockStatisticsArray( for (size_t i = 0; i < nested_statistics_keys.size(); ++i) { const auto& statistics_keys = nested_statistics_keys[i]; const auto& statistics_values = nested_statistics_values[i]; + std::shared_ptr string_type = utf8(); + if (!string_types.empty() && string_types[i]) { + string_type = string_types[i]; + } + statistics_offsets.push_back(offset); for (size_t j = 0; j < statistics_keys.size(); ++j) { const auto& key = statistics_keys[j]; @@ -182,7 +223,8 @@ Result> MakeMockStatisticsArray( } keys_indices.push_back(key_index); - auto values_type = ArrayStatistics::ValueToArrowType(value, arrow::null()); + ARROW_ASSIGN_OR_RAISE(auto values_type, + ArrayStatistics::ValueToArrowType(value, string_type)); int8_t values_type_code = 0; for (; values_type_code < static_cast(values_types.size()); ++values_type_code) { @@ -214,16 +256,21 @@ Result> MakeMockStatisticsArray( struct_({field("column", columns_type), field("statistics", statistics_type)}); ARROW_ASSIGN_OR_RAISE(auto keys_indices_array, BuildArray(keys_indices)); - ARROW_ASSIGN_OR_RAISE(auto keys_dictionary_array, - BuildArray(keys_dictionary)); + ARROW_ASSIGN_OR_RAISE(auto keys_dictionary_array, BuildArray(keys_dictionary, utf8())); ARROW_ASSIGN_OR_RAISE( auto keys_array, DictionaryArray::FromArrays(keys_type, keys_indices_array, keys_dictionary_array)); std::vector> values_arrays; - for (const auto& values : values_values) { + for (uint32_t i = 0; i < values_values.size(); ++i) { + const auto& values = values_values[i]; + std::shared_ptr string_type = utf8(); + if (!string_types.empty() && string_types[i]) { + string_type = string_types[i]; + } + ARROW_ASSIGN_OR_RAISE(auto values_array, - BuildArray(values)); + BuildArray(values, string_type)); values_arrays.push_back(values_array); } ARROW_ASSIGN_OR_RAISE(auto values_value_type_ids_array, @@ -266,21 +313,4 @@ Result> MakeNestedStruct(const int32_t depth) { } return parent_struct; } -Status CheckDepth( - std::variant, std::shared_ptr> value, - const StatisticsArrayTOptions& options) { - struct { - StatisticsArrayTOptions options; - Status operator()(const std::shared_ptr& batch) const { - return batch->MakeStatisticsArray(options).status(); - } - Status operator()(const std::shared_ptr& array) const { - // TODO Lack of method for creating statistics array for Array(GH-45804) - return Status::NotImplemented("MakeStatisticsArray not implemented for Array"); - } - } visitor; - visitor.options = options; - return std::visit(visitor, value); -} -}; // namespace test } // namespace arrow diff --git a/cpp/src/arrow/array/statistics_test_util.h b/cpp/src/arrow/array/statistics_test_util.h index 7599d0f64af9..b17eccd5ce77 100644 --- a/cpp/src/arrow/array/statistics_test_util.h +++ b/cpp/src/arrow/array/statistics_test_util.h @@ -18,24 +18,26 @@ #pragma once +#include #include #include -#include #include #include "arrow/array/statistics.h" +#include "arrow/array/statistics_option.h" +#include "arrow/testing/gtest_util.h" #include "arrow/testing/visibility.h" - #include "arrow/type_fwd.h" namespace arrow { -struct StatisticsArrayTOptions; -namespace test { +namespace detail { ARROW_TESTING_EXPORT Result> MakeMockStatisticsArray( const std::string& columns_json, const std::vector>& nested_statistics_keys, - const std::vector>& nested_statistics_values); + const std::vector>& nested_statistics_values, + const std::vector>& string_type = {}); +} // namespace detail /// \brief Create a nested struct array with a specified depth. /// @@ -45,16 +47,32 @@ ARROW_TESTING_EXPORT Result> MakeMockStatisticsArray( ARROW_TESTING_EXPORT Result> MakeNestedStruct(int32_t depth); -/// \brief Check whether it's possible to create a statistics array with the given -/// options. -/// -/// \param[in] value object for calling its Make Statistics Array -/// \param[in] options options for creating statistics array -/// -/// \return Whether \ref Status::OK or \ref Status::Invalid("Max recursion depth reached") -ARROW_TESTING_EXPORT -Status CheckDepth( - std::variant, std::shared_ptr> value, - const StatisticsArrayTOptions& options); -} // namespace test +template +class TestStatisticsArray { + public: + void CheckStatStatisticsArray( + const std::shared_ptr& value, const std::string& columns_json, + const std::vector>& nested_statistics_keys, + const std::vector>& + nested_statistics_values, + const std::vector>& string_type = {}) const { + ASSERT_OK_AND_ASSIGN(auto statistics_array, value->MakeStatisticsArray(options_)); + ASSERT_OK_AND_ASSIGN( + auto expected_statistics_array, + detail::MakeMockStatisticsArray(columns_json, nested_statistics_keys, + nested_statistics_values, string_type)); + AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + } + + void CheckDepth(const std::shared_ptr& value, bool is_error) const { + if (!is_error) { + ASSERT_OK(value->MakeStatisticsArray(options_).status()); + } else { + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", + value->MakeStatisticsArray(options_).status()); + } + } + + StatisticsArrayTOptions options_; +}; } // namespace arrow diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index e2f386e63e20..bd12a0dbcfd5 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -28,10 +28,6 @@ #include #include "arrow/array.h" -#include "arrow/array/builder_binary.h" -#include "arrow/array/builder_dict.h" -#include "arrow/array/builder_nested.h" -#include "arrow/array/builder_union.h" #include "arrow/array/concatenate.h" #include "arrow/array/statistics.h" #include "arrow/array/statistics_option.h" @@ -480,15 +476,16 @@ Result> RecordBatch::ViewOrCopyTo( Result> RecordBatch::MakeStatisticsArray( const StatisticsArrayTOptions& options) const { - ArrayDataVector vector; + ArrayDataVector collected_columns; for (int i = 0; i < this->num_columns(); ++i) { - ARROW_ASSIGN_OR_RAISE( - auto extracted_columns, - internal::ExtractColumnsToArrayData(column(i), options.max_recursion_depth)); - vector.insert(vector.end(), std::make_move_iterator(extracted_columns.begin()), - std::make_move_iterator(extracted_columns.end())); - } - return internal::MakeStatisticsArray(options.memory_pool, std::move(vector), + ARROW_ASSIGN_OR_RAISE(auto extracted_columns, + internal::ExtractColumnsToArrayData( + column(i)->data(), options.max_recursion_depth)); + collected_columns.insert(collected_columns.end(), + std::make_move_iterator(extracted_columns.begin()), + std::make_move_iterator(extracted_columns.end())); + } + return internal::MakeStatisticsArray(options.memory_pool, std::move(collected_columns), this->num_rows()); } diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h index 8c4c3ec2c83c..6b6328208ae1 100644 --- a/cpp/src/arrow/record_batch.h +++ b/cpp/src/arrow/record_batch.h @@ -285,7 +285,7 @@ class ARROW_EXPORT RecordBatch { /// \brief Create a statistics array of this record batch /// - /// The created array follows the C data interface statistics + /// The created array follows the Statistics Schema /// specification. See /// https://arrow.apache.org/docs/format/StatisticsSchema.html /// for details. diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index 5767e601db5d..6ad828ad3b48 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -24,22 +24,24 @@ #include #include #include +#include #include #include #include +#include #include #include "arrow/array/array_base.h" #include "arrow/array/array_dict.h" #include "arrow/array/array_nested.h" #include "arrow/array/data.h" +#include "arrow/array/statistics.h" #include "arrow/array/statistics_option.h" #include "arrow/array/statistics_test_util.h" #include "arrow/array/util.h" #include "arrow/c/abi.h" #include "arrow/chunked_array.h" #include "arrow/config.h" -#include "arrow/scalar.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/tensor.h" @@ -1022,127 +1024,103 @@ TEST_F(TestRecordBatch, ToTensorUnsupportedMixedFloat16) { NotImplemented, "NotImplemented: Casting from or to halffloat is not supported.", batch1->ToTensor()); } -namespace test { +class TestRecordBatchStatisticsArray : public TestStatisticsArray, + public ::testing::Test {}; -TEST_F(TestRecordBatch, MakeStatisticsArrayNesting) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNesting) { auto schema = ::arrow::schema({field("f0", int32())}); ASSERT_OK_AND_ASSIGN(auto batch, RecordBatch::MakeEmpty(schema)); - auto options = StatisticsArrayTOptions::Default(); - options.max_recursion_depth = 1; - ASSERT_OK(CheckDepth(batch, options)); - options.max_recursion_depth = 0; - ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", - CheckDepth(batch, options)); + options_.max_recursion_depth = 1; + CheckDepth(batch, false); + options_.max_recursion_depth = 0; + CheckDepth(batch, true); schema = ::arrow::schema({field("f0", int64()), field("f1", int64())}); batch = RecordBatchFromJSON( schema, R"([{"f0": 1, "f1": 4},{"f0": 2, "f1": 5},{"f0": 3, "f1": 6}])"); - options = StatisticsArrayTOptions::Default(); - options.max_recursion_depth = 1; - ASSERT_OK(CheckDepth(batch, options)); - options.max_recursion_depth = 0; - ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", - CheckDepth(batch, options)); - - options.max_recursion_depth = 2; + options_.max_recursion_depth = 1; + CheckDepth(batch, false); + options_.max_recursion_depth = 0; + CheckDepth(batch, true); + + options_.max_recursion_depth = 2; ASSERT_OK_AND_ASSIGN(auto multi_children_struct, - MakeNestedStruct(options.max_recursion_depth)); + MakeNestedStruct(options_.max_recursion_depth)); schema = ::arrow::schema({field("multi_children_struct", multi_children_struct->type())}); batch = RecordBatch::Make(schema, multi_children_struct->length(), {multi_children_struct}); - - options.max_recursion_depth = 1; - ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", - CheckDepth(batch, options)); - - options.max_recursion_depth = 2; - ASSERT_OK(CheckDepth(batch, options)); + options_.max_recursion_depth = 1; + CheckDepth(batch, true); + options_.max_recursion_depth = 2; + CheckDepth(batch, false); auto array_a = ArrayFromJSON(int64(), R"([1,2,3])"); schema = ::arrow::schema({field("a", int64()), field("multi_children_struct", multi_children_struct->type())}); batch = RecordBatch::Make(schema, 3, {array_a, multi_children_struct}); - - options.max_recursion_depth = 1; - ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", - CheckDepth(batch, options)); - - options.max_recursion_depth = 2; - ASSERT_OK(CheckDepth(batch, options)); + options_.max_recursion_depth = 1; + CheckDepth(batch, true); + options_.max_recursion_depth = 2; + CheckDepth(batch, false); } -TEST_F(TestRecordBatch, MakeStatisticsArrayNestingStressTest) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNestingStressTest) { ASSERT_OK_AND_ASSIGN(auto deep_children_struct, MakeNestedStruct(100)); auto schema = ::arrow::schema({field("struct", deep_children_struct->type())}); auto batch = RecordBatch::Make(schema, deep_children_struct->length(), {deep_children_struct}); - StatisticsArrayTOptions options; - options.max_recursion_depth = 100; - ASSERT_OK(CheckDepth(batch, options)); - options.max_recursion_depth = 99; - ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", - batch->MakeStatisticsArray(options)); + options_.max_recursion_depth = 100; + CheckDepth(batch, false); + options_.max_recursion_depth = 99; + CheckDepth(batch, true); #if !defined(ARROW_VALGRIND) - ASSERT_OK_AND_ASSIGN(deep_children_struct, MakeNestedStruct(500)); + ASSERT_OK_AND_ASSIGN(deep_children_struct, MakeNestedStruct(5000)); schema = ::arrow::schema({field("struct", deep_children_struct->type())}); batch = RecordBatch::Make(schema, deep_children_struct->length(), {deep_children_struct}); - options.max_recursion_depth = 500; - ASSERT_OK(CheckDepth(batch, options)); - options.max_recursion_depth = 499; - ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Max recursion depth reached", - batch->MakeStatisticsArray(options)); + options_.max_recursion_depth = 5000; + CheckDepth(batch, false); + options_.max_recursion_depth = 4999; + CheckDepth(batch, true); + #endif } -TEST_F(TestRecordBatch, MakeStatisticsArrayRowCount) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayRowCount) { auto schema = ::arrow::schema({field("int32", int32())}); auto int32_array = ArrayFromJSON(int32(), "[1, null, -1]"); auto batch = RecordBatch::Make(schema, int32_array->length(), {int32_array}); - - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(auto expected_statistics_array, - MakeMockStatisticsArray("[null]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }}); // I'm not sure whether the following states are valid batch = RecordBatch::Make(schema, 1, {int32_array}); - ASSERT_OK_AND_ASSIGN(statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN(expected_statistics_array, - MakeMockStatisticsArray("[null]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{1}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{1}}, + }}); batch = RecordBatch::Make(schema, 10, {int32_array}); - ASSERT_OK_AND_ASSIGN(statistics_array, batch->MakeStatisticsArray()); - ASSERT_OK_AND_ASSIGN( - expected_statistics_array, - MakeMockStatisticsArray("[null]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{10}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{10}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayNullCount) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNullCount) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("int32", int32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1153,27 +1131,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNullCount) { auto batch = RecordBatch::Make(schema, int32_array->length(), {no_statistics_array, int32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{int64_t{1}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{int64_t{1}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayDistinctCount) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayDistinctCount) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("int32", int32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1185,29 +1158,24 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayDistinctCount) { auto batch = RecordBatch::Make(schema, int32_array->length(), {no_statistics_array, int32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{int64_t{1}}, - ArrayStatistics::ValueType{int64_t{2}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{int64_t{1}}, + ArrayStatistics::ValueType{int64_t{2}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMinExact) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMinExact) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("uint32", uint32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1219,27 +1187,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMinExact) { auto batch = RecordBatch::Make(schema, uint32_array->length(), {no_statistics_array, uint32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MIN_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{uint64_t{1}}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MIN_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{uint64_t{1}}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMinApproximate) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMinApproximate) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("int32", int32())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1250,27 +1213,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMinApproximate) { auto batch = RecordBatch::Make(schema, int32_array->length(), {no_statistics_array, int32_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{-1.0}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{-1.0}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMaxExact) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMaxExact) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("boolean", boolean())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1283,27 +1241,22 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMaxExact) { auto batch = RecordBatch::Make(schema, boolean_array->length(), {no_statistics_array, boolean_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{true}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{true}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayMaxApproximate) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayMaxApproximate) { auto schema = ::arrow::schema({field("no-statistics", boolean()), field("float64", float64())}); auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -1314,56 +1267,94 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayMaxApproximate) { auto batch = RecordBatch::Make(schema, float64_array->length(), {no_statistics_array, float64_array}); - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); - - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{1.0}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{1.0}, + }}); } -TEST_F(TestRecordBatch, MakeStatisticsArrayString) { - auto schema = - ::arrow::schema({field("no-statistics", boolean()), field("string", utf8())}); - auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); - auto string_array_data = ArrayFromJSON(utf8(), "[\"a\", null, \"c\"]")->data()->Copy(); - string_array_data->statistics = std::make_shared(); - string_array_data->statistics->is_max_exact = true; - string_array_data->statistics->max = "c"; - auto string_array = MakeArray(std::move(string_array_data)); - auto batch = RecordBatch::Make(schema, string_array->length(), - {no_statistics_array, string_array}); - - ASSERT_OK_AND_ASSIGN(auto statistics_array, batch->MakeStatisticsArray()); +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayString) { + // Check each string type + auto CheckString = [&](const std::shared_ptr& type) { + auto schema = + ::arrow::schema({field("no-statistics", boolean()), field("string", type)}); + auto no_statistics_array = ArrayFromJSON(boolean(), "[true, false, true]"); + auto string_array_data = ArrayFromJSON(type, R"(["a", null, "c"])")->data()->Copy(); + string_array_data->statistics = std::make_shared(); + string_array_data->statistics->is_max_exact = true; + string_array_data->statistics->max = "c"; + auto string_array = MakeArray(std::move(string_array_data)); + auto batch = RecordBatch::Make(schema, string_array->length(), + {no_statistics_array, string_array}); + CheckStatStatisticsArray(batch, "[null, 1]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{"c"}, + }}, + {nullptr, type}); + }; - ASSERT_OK_AND_ASSIGN( - auto expected_statistics_array, - MakeMockStatisticsArray("[null, 1]", - {{ - ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, - }, - { - ARROW_STATISTICS_KEY_MAX_VALUE_EXACT, - }}, - {{ - ArrayStatistics::ValueType{int64_t{3}}, - }, - { - ArrayStatistics::ValueType{"c"}, - }})); - AssertArraysEqual(*expected_statistics_array, *statistics_array, true); + CheckString(binary()); + CheckString(large_binary()); + CheckString(utf8()); + CheckString(large_utf8()); + CheckString(binary_view()); + CheckString(utf8_view()); + CheckString(fixed_size_binary(1)); + + // Check the combination of various string types + + auto f0 = ArrayFromJSON(utf8(), R"(["a", "b", "c"])"); + f0->data()->statistics = std::make_shared(); + f0->data()->statistics->max = "c"; + + auto f1 = ArrayFromJSON(int64(), R"([1, 2, 3])"); + + auto f2 = ArrayFromJSON(large_binary(), R"(["a", "b", "c"])"); + f2->data()->statistics = std::make_shared(); + f2->data()->statistics->max = "c"; + + auto schema = ::arrow::schema( + {field("f0", utf8()), field("f1", int64()), field("f2", utf8_view())}); + auto batch = RecordBatch::Make(schema, f0->length(), {f0, f1, f2}); + + CheckStatStatisticsArray(batch, "[null, 0, 2]", + {{ + ARROW_STATISTICS_KEY_ROW_COUNT_EXACT, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }, + { + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE, + }}, + {{ + ArrayStatistics::ValueType{int64_t{3}}, + }, + { + ArrayStatistics::ValueType{"c"}, + }, + { + ArrayStatistics::ValueType{"c"}, + }}, + {nullptr, utf8(), large_binary()}); } // Schema: // struct_a_b: struct {a: int64, b: int64} @@ -1393,7 +1384,7 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayString) { // struct_e_f.e: // max_exact: 30 -TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNestedType) { const auto struct_a_b_type = struct_({field("a", int64()), field("b", int64())}); const auto struct_a_b = ArrayFromJSON( struct_a_b_type, @@ -1418,32 +1409,27 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { struct_e_f_child_e_statistics->statistics->max = int64_t{30}; struct_e_f_child_e_statistics->statistics->is_max_exact = true; - const auto expected_scalar = ScalarFromJSON(struct_a_b_type, R"([5 ,10])"); - - ASSERT_OK_AND_ASSIGN( - const auto expected_array_statistics, - MakeMockStatisticsArray("[null , 0, 1, 3, 6]", - {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, - {ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, - ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, - {{ArrayStatistics::ValueType{int64_t{5}}}, - {ArrayStatistics::ValueType{int64_t{0}}, - ArrayStatistics::ValueType{expected_scalar}}, - {ArrayStatistics::ValueType{int64_t{1}}}, - {ArrayStatistics::ValueType{int64_t{15}}}, - {ArrayStatistics::ValueType{int64_t{30}}}})); - const auto schema = ::arrow::schema({field("struct_a_b", struct_a_b_type), field("c", int64()), field("d", int64()), field("struct_e_f", struct_e_f->type())}); const auto batch = RecordBatch::Make(schema, 5, {struct_a_b, array_c, array_d, struct_e_f}); - ASSERT_OK_AND_ASSIGN(auto batch_statistics, batch->MakeStatisticsArray()); - AssertArraysEqual(*expected_array_statistics, *batch_statistics, true); + const auto expected_scalar = ScalarFromJSON(struct_a_b_type, R"([5 ,10])"); + + CheckStatStatisticsArray(batch, "[null , 0, 1, 3, 6]", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_NULL_COUNT_EXACT, + ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{int64_t{0}}, + ArrayStatistics::ValueType{expected_scalar}}, + {ArrayStatistics::ValueType{int64_t{1}}}, + {ArrayStatistics::ValueType{int64_t{15}}}, + {ArrayStatistics::ValueType{int64_t{30}}}}); } // Schema @@ -1478,7 +1464,7 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedType) { // struct_nested_2: // min_exact: {a:5, b: 4} // -TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { +TEST_F(TestRecordBatchStatisticsArray, MakeStatisticsArrayNestedNestedType) { const auto struct_nested_0_type = struct_({field("a", int32()), field("b", int32())}); const auto struct_nested_0 = ArrayFromJSON( struct_nested_0_type, @@ -1514,6 +1500,9 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { ASSERT_OK_AND_ASSIGN(struct_parant->data()->statistics->min, struct_parant->GetScalar(0)); + const auto schema = ::arrow::schema({field("struct", struct_parant->type())}); + const auto batch = RecordBatch::Make(schema, 5, {struct_parant}); + const auto expected_scalar_parent = ScalarFromJSON(struct_parant->type(), R"([[1 ,6], [1, 6], [1, 6]])"); @@ -1525,29 +1514,19 @@ TEST_F(TestRecordBatch, MakeStatisticsArrayNestedNestedType) { const auto expected_scalar_nested_2 = ScalarFromJSON(struct_nested_2_type, R"([1, 6])"); - ASSERT_OK_AND_ASSIGN( - const auto expected_array_statistics, - MakeMockStatisticsArray(R"([null, 0, 1, 4, 7])", - {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, - {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, - {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}, - {ARROW_STATISTICS_KEY_MIN_VALUE_EXACT}}, - {{ArrayStatistics::ValueType{int64_t{5}}}, - {ArrayStatistics::ValueType{expected_scalar_parent}}, - {ArrayStatistics::ValueType{expected_scalar_nested_0}}, - {ArrayStatistics::ValueType{expected_scalar_nested_1}}, - {expected_scalar_nested_2}})); - - const auto schema = ::arrow::schema({field("struct", struct_parant->type())}); - const auto batch = RecordBatch::Make(schema, 5, {struct_parant}); - ASSERT_OK_AND_ASSIGN(const auto batch_statistics, batch->MakeStatisticsArray()); - - AssertArraysEqual(*expected_array_statistics, *batch_statistics, true); + CheckStatStatisticsArray(batch, R"([null, 0, 1, 4, 7])", + {{ARROW_STATISTICS_KEY_ROW_COUNT_EXACT}, + {ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE}, + {ARROW_STATISTICS_KEY_MAX_VALUE_EXACT}, + {ARROW_STATISTICS_KEY_MIN_VALUE_EXACT}}, + {{ArrayStatistics::ValueType{int64_t{5}}}, + {ArrayStatistics::ValueType{expected_scalar_parent}}, + {ArrayStatistics::ValueType{expected_scalar_nested_0}}, + {ArrayStatistics::ValueType{expected_scalar_nested_1}}, + {expected_scalar_nested_2}}); } -} // namespace test - template class TestBatchToTensorColumnMajor : public ::testing::Test {};