From 5d33a61287932cfe1e7141c30da757d6451f3fa8 Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Fri, 17 Feb 2023 17:21:01 -0800 Subject: [PATCH 1/3] Add an order_by node --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/compute/exec/CMakeLists.txt | 6 + cpp/src/arrow/compute/exec/exec_plan.cc | 2 + cpp/src/arrow/compute/exec/options.h | 8 + cpp/src/arrow/compute/exec/order_by_node.cc | 166 ++++++++++++++++++ .../arrow/compute/exec/order_by_node_test.cc | 119 +++++++++++++ cpp/src/arrow/testing/generator.cc | 27 ++- cpp/src/arrow/testing/generator.h | 3 +- 8 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 cpp/src/arrow/compute/exec/order_by_node.cc create mode 100644 cpp/src/arrow/compute/exec/order_by_node_test.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 377423403e63..28f37af9bf14 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -404,6 +404,7 @@ list(APPEND compute/exec/map_node.cc compute/exec/options.cc compute/exec/order_by_impl.cc + compute/exec/order_by_node.cc compute/exec/partition_util.cc compute/exec/pivot_longer_node.cc compute/exec/project_node.cc diff --git a/cpp/src/arrow/compute/exec/CMakeLists.txt b/cpp/src/arrow/compute/exec/CMakeLists.txt index d77333caca5f..c7853b2f67dc 100644 --- a/cpp/src/arrow/compute/exec/CMakeLists.txt +++ b/cpp/src/arrow/compute/exec/CMakeLists.txt @@ -46,6 +46,12 @@ add_arrow_compute_test(hash_join_node_test SOURCES hash_join_node_test.cc bloom_filter_test.cc) +add_arrow_compute_test(order_by_node_test + PREFIX + "arrow-compute" + SOURCES + order_by_node_test.cc + test_nodes.cc) add_arrow_compute_test(pivot_longer_node_test PREFIX "arrow-compute" diff --git a/cpp/src/arrow/compute/exec/exec_plan.cc b/cpp/src/arrow/compute/exec/exec_plan.cc index 0fd5ff231068..5a41fd925d1b 100644 --- a/cpp/src/arrow/compute/exec/exec_plan.cc +++ b/cpp/src/arrow/compute/exec/exec_plan.cc @@ -1053,6 +1053,7 @@ namespace internal { void RegisterSourceNode(ExecFactoryRegistry*); void RegisterFetchNode(ExecFactoryRegistry*); void RegisterFilterNode(ExecFactoryRegistry*); +void RegisterOrderByNode(ExecFactoryRegistry*); void RegisterPivotLongerNode(ExecFactoryRegistry*); void RegisterProjectNode(ExecFactoryRegistry*); void RegisterUnionNode(ExecFactoryRegistry*); @@ -1070,6 +1071,7 @@ ExecFactoryRegistry* default_exec_factory_registry() { internal::RegisterSourceNode(this); internal::RegisterFetchNode(this); internal::RegisterFilterNode(this); + internal::RegisterOrderByNode(this); internal::RegisterPivotLongerNode(this); internal::RegisterProjectNode(this); internal::RegisterUnionNode(this); diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index 2e44015de6c1..3aae70e9fa66 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -420,6 +420,14 @@ class ARROW_EXPORT OrderBySinkNodeOptions : public SinkNodeOptions { SortOptions sort_options; }; +class ARROW_EXPORT OrderByNodeOptions : public ExecNodeOptions { + public: + static constexpr std::string_view kName = "order_by"; + explicit OrderByNodeOptions(Ordering ordering) : ordering(std::move(ordering)) {} + + Ordering ordering; +}; + /// @} enum class JoinType { diff --git a/cpp/src/arrow/compute/exec/order_by_node.cc b/cpp/src/arrow/compute/exec/order_by_node.cc new file mode 100644 index 000000000000..beca15fc49e5 --- /dev/null +++ b/cpp/src/arrow/compute/exec/order_by_node.cc @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#include "arrow/compute/exec/exec_plan.h" +#include "arrow/compute/exec/options.h" +#include "arrow/compute/exec/query_context.h" +#include "arrow/compute/exec/util.h" +#include "arrow/result.h" +#include "arrow/table.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/tracing_internal.h" + +namespace arrow { + +using internal::checked_cast; + +namespace compute { +namespace { + +class OrderByNode : public ExecNode, public TracedNode { + public: + OrderByNode(ExecPlan* plan, std::vector inputs, + std::shared_ptr output_schema, Ordering new_ordering) + : ExecNode(plan, std::move(inputs), {"input"}, std::move(output_schema)), + TracedNode(this), + ordering_(std::move(new_ordering)) {} + + static Result Make(ExecPlan* plan, std::vector inputs, + const ExecNodeOptions& options) { + RETURN_NOT_OK(ValidateExecNodeInputs(plan, inputs, 1, "FetchNode")); + + const auto& order_options = checked_cast(options); + + if (order_options.ordering.is_implicit() || order_options.ordering.is_unordered()) { + return Status::Invalid("`ordering` must be an explicit non-empty ordering"); + } + + std::shared_ptr output_schema = inputs[0]->output_schema(); + return plan->EmplaceNode( + plan, std::move(inputs), std::move(output_schema), order_options.ordering); + } + + const char* kind_name() const override { return "OrderByNode"; } + + const Ordering& ordering() const override { return ordering_; } + + Status InputFinished(ExecNode* input, int total_batches) override { + DCHECK_EQ(input, inputs_[0]); + EVENT_ON_CURRENT_SPAN("InputFinished", {{"batches.length", total_batches}}); + // We can't send InputFinished downstream because we might change the # of batches + // when we sort it. So that happens later in DoFinish + if (counter_.SetTotal(total_batches)) { + return DoFinish(); + } + return Status::OK(); + } + + Status StartProducing() override { + NoteStartProducing(ToStringExtra()); + return Status::OK(); + } + + void PauseProducing(ExecNode* output, int32_t counter) override { + inputs_[0]->PauseProducing(this, counter); + } + + void ResumeProducing(ExecNode* output, int32_t counter) override { + inputs_[0]->ResumeProducing(this, counter); + } + + Status StopProducingImpl() override { return Status::OK(); } + + Status InputReceived(ExecNode* input, ExecBatch batch) override { + auto scope = TraceInputReceived(batch); + DCHECK_EQ(input, inputs_[0]); + + ARROW_ASSIGN_OR_RAISE(std::shared_ptr record_batch, + batch.ToRecordBatch(output_schema_)); + + { + std::lock_guard lk(mutex_); + accumulation_queue_.push_back(std::move(record_batch)); + } + + if (counter_.Increment()) { + return DoFinish(); + } + return Status::OK(); + } + + Status DoFinish() { + ARROW_ASSIGN_OR_RAISE( + auto table, + Table::FromRecordBatches(output_schema_, std::move(accumulation_queue_))); + SortOptions sort_options(ordering_.sort_keys(), ordering_.null_placement()); + ExecContext* ctx = plan_->query_context()->exec_context(); + ARROW_ASSIGN_OR_RAISE(auto indices, SortIndices(table, sort_options, ctx)); + ARROW_ASSIGN_OR_RAISE(Datum sorted, + Take(table, indices, TakeOptions::NoBoundsCheck(), ctx)); + const std::shared_ptr& sorted_table = sorted.table(); + TableBatchReader reader(*sorted_table); + reader.set_chunksize(ExecPlan::kMaxBatchSize); + int batch_index = 0; + while (true) { + ARROW_ASSIGN_OR_RAISE(std::shared_ptr next, reader.Next()); + if (!next) { + return output_->InputFinished(this, batch_index); + } + int index = batch_index++; + plan_->query_context()->ScheduleTask( + [this, batch = std::move(next), index]() mutable { + ExecBatch exec_batch(*batch); + exec_batch.index = index; + return output_->InputReceived(this, std::move(exec_batch)); + }, + "OrderByNode::ProcessBatch"); + } + } + + protected: + std::string ToStringExtra(int indent = 0) const override { + std::stringstream ss; + ss << "ordering=" << ordering_.ToString(); + return ss.str(); + } + + private: + AtomicCounter counter_; + Ordering ordering_; + std::vector> accumulation_queue_; + std::mutex mutex_; +}; + +} // namespace + +namespace internal { + +void RegisterOrderByNode(ExecFactoryRegistry* registry) { + DCHECK_OK( + registry->AddFactory(std::string(OrderByNodeOptions::kName), OrderByNode::Make)); +} + +} // namespace internal +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/exec/order_by_node_test.cc b/cpp/src/arrow/compute/exec/order_by_node_test.cc new file mode 100644 index 000000000000..4dbbf61fd9fd --- /dev/null +++ b/cpp/src/arrow/compute/exec/order_by_node_test.cc @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "arrow/compute/exec/exec_plan.h" +#include "arrow/compute/exec/options.h" +#include "arrow/compute/exec/test_nodes.h" +#include "arrow/table.h" +#include "arrow/testing/generator.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" +#include "arrow/util/checked_cast.h" + +namespace arrow { + +using internal::checked_pointer_cast; + +namespace compute { + +// Sorting is slow, don't use too many rows +static constexpr int kRowsPerBatch = 4; +static constexpr int kNumBatches = 32; + +std::shared_ptr
TestTable() { + return gen::Gen({{"up", gen::Step()}, + {"down", gen::Step(/*start=*/0, /*step=*/-1, /*signed_int=*/true)}}) + ->FailOnError() + ->Table(kRowsPerBatch, kNumBatches); +} + +void CheckOrderBy(OrderByNodeOptions options) { + constexpr random::SeedType kSeed = 42; + constexpr int kJitterMod = 4; + RegisterTestNodes(); + std::shared_ptr
input = TestTable(); + Declaration plan = + Declaration::Sequence({{"table_source", TableSourceNodeOptions(input)}, + {"jitter", JitterNodeOptions(kSeed, kJitterMod)}, + {"order_by", options}}); + for (bool use_threads : {false, true}) { + QueryOptions query_options; + query_options.sequence_output = true; + query_options.use_threads = use_threads; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
actual, + DeclarationToTable(plan, query_options)); + + AssertTablesEqual(*input, *actual, /*same_chunk_layout=*/false); + } +} + +void CheckOrderByInvalid(OrderByNodeOptions options, const std::string& message) { + std::shared_ptr
input = TestTable(); + Declaration plan = Declaration::Sequence( + {{"table_source", TableSourceNodeOptions(input)}, {"order_by", options}}); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr(message), + DeclarationToStatus(std::move(plan))); +} + +TEST(OrderByNode, Basic) { + CheckOrderBy(OrderByNodeOptions({{SortKey("up")}})); + CheckOrderBy(OrderByNodeOptions({{SortKey("down", SortOrder::Descending)}})); + CheckOrderBy( + OrderByNodeOptions({{SortKey("up"), SortKey("down", SortOrder::Descending)}})); +} + +TEST(OrderByNode, Large) { + constexpr random::SeedType kSeed = 42; + constexpr int kJitterMod = 4; + static constexpr int kSmallNumBatches = 8; + + RegisterTestNodes(); + // Confirm that the order by node chunks output for parallel processing + std::shared_ptr
input = gen::Gen({{"up", gen::Step()}}) + ->FailOnError() + ->Table(ExecPlan::kMaxBatchSize, kSmallNumBatches); + Declaration plan = Declaration::Sequence({ + {"table_source", TableSourceNodeOptions(input)}, + {"order_by", OrderByNodeOptions({{SortKey("up", SortOrder::Descending)}})}, + {"jitter", JitterNodeOptions(kSeed, kJitterMod)}, + }); + ASSERT_OK_AND_ASSIGN(BatchesWithCommonSchema batches_and_schema, + DeclarationToExecBatches(std::move(plan))); + ASSERT_EQ(kSmallNumBatches, static_cast(batches_and_schema.batches.size())); + // Jitter simulates parallel processing after the order by node. However, the batch + // index should be correct, regardless of the jitter + for (const auto& batch : batches_and_schema.batches) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr last_row, + batch.values[0].make_array()->GetScalar(batch.length - 1)); + uint32_t last_val = checked_pointer_cast(last_row)->value; + ASSERT_EQ(last_val, ExecPlan::kMaxBatchSize * (kSmallNumBatches - batch.index - 1)); + } +} + +TEST(OrderByNode, Invalid) { + CheckOrderByInvalid(OrderByNodeOptions(Ordering::Imiplicit()), + "`ordering` must be an explicit non-empty ordering"); + CheckOrderByInvalid(OrderByNodeOptions(Ordering::Unordered()), + "`ordering` must be an explicit non-empty ordering"); +} + +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/testing/generator.cc b/cpp/src/arrow/testing/generator.cc index 45316bef8a91..36c88c20efe6 100644 --- a/cpp/src/arrow/testing/generator.cc +++ b/cpp/src/arrow/testing/generator.cc @@ -215,12 +215,14 @@ class ConstantGenerator : public ArrayGenerator { class StepGenerator : public ArrayGenerator { public: - StepGenerator(uint32_t start, uint32_t step) : start_(start), step_(step) {} + StepGenerator(uint32_t start, uint32_t step, bool signed_int) + : start_(start), step_(step), signed_int_(signed_int) {} - Result> Generate(int64_t num_rows) override { - UInt32Builder builder; + template + Result> DoGenerate(int64_t num_rows) { + BuilderType builder; ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); - uint32_t val = start_; + CType val = start_; for (int64_t i = 0; i < num_rows; i++) { builder.UnsafeAppend(val); val += step_; @@ -229,11 +231,22 @@ class StepGenerator : public ArrayGenerator { return builder.Finish(); } - std::shared_ptr type() const override { return uint32(); } + Result> Generate(int64_t num_rows) override { + if (signed_int_) { + return DoGenerate(num_rows); + } else { + return DoGenerate(num_rows); + } + } + + std::shared_ptr type() const override { + return signed_int_ ? int32() : uint32(); + } private: uint32_t start_; uint32_t step_; + bool signed_int_; }; static constexpr random::SeedType kTestSeed = 42; @@ -385,8 +398,8 @@ std::shared_ptr Constant(std::shared_ptr value) { return std::make_shared(std::move(value)); } -std::shared_ptr Step(uint32_t start, uint32_t step) { - return std::make_shared(start, step); +std::shared_ptr Step(uint32_t start, uint32_t step, bool signed_int) { + return std::make_shared(start, step, signed_int); } std::shared_ptr Random(std::shared_ptr type) { diff --git a/cpp/src/arrow/testing/generator.h b/cpp/src/arrow/testing/generator.h index 8b72f8a68aad..4ec8845864b7 100644 --- a/cpp/src/arrow/testing/generator.h +++ b/cpp/src/arrow/testing/generator.h @@ -305,7 +305,8 @@ ARROW_TESTING_EXPORT std::shared_ptr Constant( /// /// Note: overflow is not prevented standard unsigned integer overflow applies ARROW_TESTING_EXPORT std::shared_ptr Step(uint32_t start = 0, - uint32_t step = 1); + uint32_t step = 1, + bool signed_int = false); /// make a generator that returns a random value ARROW_TESTING_EXPORT std::shared_ptr Random( std::shared_ptr type); From 62284a97a5931e384622944c1d97480cb535bc0b Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Thu, 2 Mar 2023 15:40:57 -0800 Subject: [PATCH 2/3] Typo exposed after rebase --- cpp/src/arrow/compute/exec/order_by_node_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/exec/order_by_node_test.cc b/cpp/src/arrow/compute/exec/order_by_node_test.cc index 4dbbf61fd9fd..04d1eaa7b203 100644 --- a/cpp/src/arrow/compute/exec/order_by_node_test.cc +++ b/cpp/src/arrow/compute/exec/order_by_node_test.cc @@ -109,7 +109,7 @@ TEST(OrderByNode, Large) { } TEST(OrderByNode, Invalid) { - CheckOrderByInvalid(OrderByNodeOptions(Ordering::Imiplicit()), + CheckOrderByInvalid(OrderByNodeOptions(Ordering::Implicit()), "`ordering` must be an explicit non-empty ordering"); CheckOrderByInvalid(OrderByNodeOptions(Ordering::Unordered()), "`ordering` must be an explicit non-empty ordering"); From 38bf31304d9cfd0981c241e6fa6c59b77a887b7b Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Mon, 20 Mar 2023 13:37:28 -0700 Subject: [PATCH 3/3] Addressing PR reviews --- cpp/src/arrow/compute/api_vector.cc | 4 ++++ cpp/src/arrow/compute/api_vector.h | 8 ++++++++ cpp/src/arrow/compute/exec/options.h | 7 +++++++ cpp/src/arrow/compute/ordering.h | 4 ++-- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/compute/api_vector.cc b/cpp/src/arrow/compute/api_vector.cc index 1064d5dfb331..9fd8f2aa4834 100644 --- a/cpp/src/arrow/compute/api_vector.cc +++ b/cpp/src/arrow/compute/api_vector.cc @@ -177,6 +177,10 @@ SortOptions::SortOptions(std::vector sort_keys, NullPlacement null_plac : FunctionOptions(internal::kSortOptionsType), sort_keys(std::move(sort_keys)), null_placement(null_placement) {} +SortOptions::SortOptions(const Ordering& ordering) + : FunctionOptions(internal::kSortOptionsType), + sort_keys(ordering.sort_keys()), + null_placement(ordering.null_placement()) {} constexpr char SortOptions::kTypeName[]; PartitionNthOptions::PartitionNthOptions(int64_t pivot, NullPlacement null_placement) diff --git a/cpp/src/arrow/compute/api_vector.h b/cpp/src/arrow/compute/api_vector.h index 91a750dc5c5e..f0d2bacf16c1 100644 --- a/cpp/src/arrow/compute/api_vector.h +++ b/cpp/src/arrow/compute/api_vector.h @@ -106,8 +106,16 @@ class ARROW_EXPORT SortOptions : public FunctionOptions { public: explicit SortOptions(std::vector sort_keys = {}, NullPlacement null_placement = NullPlacement::AtEnd); + explicit SortOptions(const Ordering& ordering); static constexpr char const kTypeName[] = "SortOptions"; static SortOptions Defaults() { return SortOptions(); } + /// Convenience constructor to create an ordering from SortOptions + /// + /// Note: Both classes contain the exact same information. However, + /// sort_options should only be used in a "function options" context while Ordering + /// is used more generally. + Ordering AsOrdering() && { return Ordering(std::move(sort_keys), null_placement); } + Ordering AsOrdering() const& { return Ordering(sort_keys, null_placement); } /// Column key(s) to order by and how to order by these sort keys. std::vector sort_keys; diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index 3aae70e9fa66..5faebd21efd6 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -420,11 +420,18 @@ class ARROW_EXPORT OrderBySinkNodeOptions : public SinkNodeOptions { SortOptions sort_options; }; +/// \brief Apply a new ordering to data +/// +/// Currently this node works by accumulating all data, sorting, and then emitting +/// the new data with an updated batch index. +/// +/// Larger-than-memory sort is not currently supported. class ARROW_EXPORT OrderByNodeOptions : public ExecNodeOptions { public: static constexpr std::string_view kName = "order_by"; explicit OrderByNodeOptions(Ordering ordering) : ordering(std::move(ordering)) {} + /// \brief The new ordering to apply to outgoing data Ordering ordering; }; diff --git a/cpp/src/arrow/compute/ordering.h b/cpp/src/arrow/compute/ordering.h index b4c38e610339..e581269cc20d 100644 --- a/cpp/src/arrow/compute/ordering.h +++ b/cpp/src/arrow/compute/ordering.h @@ -90,8 +90,8 @@ class ARROW_EXPORT Ordering : public util::EqualityComparable { bool is_implicit() const { return is_implicit_; } bool is_unordered() const { return !is_implicit_ && sort_keys_.empty(); } - std::vector sort_keys() { return sort_keys_; } - NullPlacement null_placement() { return null_placement_; } + const std::vector& sort_keys() const { return sort_keys_; } + NullPlacement null_placement() const { return null_placement_; } static const Ordering& Implicit() { static const Ordering kImplicit(true);