Repository navigation
GH-34248: [C++] Add an order_by node #34249
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -420,6 +420,21 @@ class ARROW_EXPORT OrderBySinkNodeOptions : public SinkNodeOptions { | |
| SortOptions sort_options; | ||
| }; | ||
|
|
||
| /// \brief Apply a new ordering to data | ||
| /// | ||
| /// Currently this node works by accumulating all data, sorting, and then emitting | ||
| /// the new data with an updated batch index. | ||
| /// | ||
| /// Larger-than-memory sort is not currently supported. | ||
| class ARROW_EXPORT OrderByNodeOptions : public ExecNodeOptions { | ||
| public: | ||
| static constexpr std::string_view kName = "order_by"; | ||
| explicit OrderByNodeOptions(Ordering ordering) : ordering(std::move(ordering)) {} | ||
|
|
||
| /// \brief The new ordering to apply to outgoing data | ||
| Ordering ordering; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The OrderBySinkNode is using
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes,
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the Ordering certainly makes sense, and it's quite trivial to convert the one into the other, I was just wondering for a consistent API if we want to add a constructor version that accepts the SortOptions (or for OrderBySinkNode to add a version that accepts Ordering) |
||
| }; | ||
|
|
||
| /// @} | ||
|
|
||
| enum class JoinType { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| #include <cstdint> | ||
| #include <memory> | ||
| #include <mutex> | ||
| #include <sstream> | ||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| #include "arrow/compute/exec/exec_plan.h" | ||
| #include "arrow/compute/exec/options.h" | ||
| #include "arrow/compute/exec/query_context.h" | ||
| #include "arrow/compute/exec/util.h" | ||
| #include "arrow/result.h" | ||
| #include "arrow/table.h" | ||
| #include "arrow/util/checked_cast.h" | ||
| #include "arrow/util/tracing_internal.h" | ||
|
|
||
| namespace arrow { | ||
|
|
||
| using internal::checked_cast; | ||
|
|
||
| namespace compute { | ||
| namespace { | ||
|
|
||
| class OrderByNode : public ExecNode, public TracedNode { | ||
| public: | ||
| OrderByNode(ExecPlan* plan, std::vector<ExecNode*> inputs, | ||
| std::shared_ptr<Schema> output_schema, Ordering new_ordering) | ||
| : ExecNode(plan, std::move(inputs), {"input"}, std::move(output_schema)), | ||
| TracedNode(this), | ||
| ordering_(std::move(new_ordering)) {} | ||
|
|
||
| static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs, | ||
| const ExecNodeOptions& options) { | ||
| RETURN_NOT_OK(ValidateExecNodeInputs(plan, inputs, 1, "FetchNode")); | ||
|
|
||
| const auto& order_options = checked_cast<const OrderByNodeOptions&>(options); | ||
|
|
||
| if (order_options.ordering.is_implicit() || order_options.ordering.is_unordered()) { | ||
| return Status::Invalid("`ordering` must be an explicit non-empty ordering"); | ||
| } | ||
|
|
||
| std::shared_ptr<Schema> output_schema = inputs[0]->output_schema(); | ||
| return plan->EmplaceNode<OrderByNode>( | ||
| plan, std::move(inputs), std::move(output_schema), order_options.ordering); | ||
| } | ||
|
|
||
| const char* kind_name() const override { return "OrderByNode"; } | ||
|
|
||
| const Ordering& ordering() const override { return ordering_; } | ||
|
|
||
| Status InputFinished(ExecNode* input, int total_batches) override { | ||
| DCHECK_EQ(input, inputs_[0]); | ||
| EVENT_ON_CURRENT_SPAN("InputFinished", {{"batches.length", total_batches}}); | ||
| // We can't send InputFinished downstream because we might change the # of batches | ||
| // when we sort it. So that happens later in DoFinish | ||
| if (counter_.SetTotal(total_batches)) { | ||
| return DoFinish(); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| Status StartProducing() override { | ||
| NoteStartProducing(ToStringExtra()); | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| void PauseProducing(ExecNode* output, int32_t counter) override { | ||
| inputs_[0]->PauseProducing(this, counter); | ||
| } | ||
|
|
||
| void ResumeProducing(ExecNode* output, int32_t counter) override { | ||
| inputs_[0]->ResumeProducing(this, counter); | ||
| } | ||
|
|
||
| Status StopProducingImpl() override { return Status::OK(); } | ||
|
|
||
| Status InputReceived(ExecNode* input, ExecBatch batch) override { | ||
| auto scope = TraceInputReceived(batch); | ||
| DCHECK_EQ(input, inputs_[0]); | ||
|
|
||
| ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> record_batch, | ||
| batch.ToRecordBatch(output_schema_)); | ||
|
|
||
| { | ||
| std::lock_guard lk(mutex_); | ||
| accumulation_queue_.push_back(std::move(record_batch)); | ||
| } | ||
|
|
||
| if (counter_.Increment()) { | ||
| return DoFinish(); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| Status DoFinish() { | ||
| ARROW_ASSIGN_OR_RAISE( | ||
| auto table, | ||
| Table::FromRecordBatches(output_schema_, std::move(accumulation_queue_))); | ||
| SortOptions sort_options(ordering_.sort_keys(), ordering_.null_placement()); | ||
| ExecContext* ctx = plan_->query_context()->exec_context(); | ||
| ARROW_ASSIGN_OR_RAISE(auto indices, SortIndices(table, sort_options, ctx)); | ||
| ARROW_ASSIGN_OR_RAISE(Datum sorted, | ||
| Take(table, indices, TakeOptions::NoBoundsCheck(), ctx)); | ||
| const std::shared_ptr<Table>& sorted_table = sorted.table(); | ||
| TableBatchReader reader(*sorted_table); | ||
| reader.set_chunksize(ExecPlan::kMaxBatchSize); | ||
| int batch_index = 0; | ||
| while (true) { | ||
| ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> next, reader.Next()); | ||
| if (!next) { | ||
| return output_->InputFinished(this, batch_index); | ||
| } | ||
| int index = batch_index++; | ||
| plan_->query_context()->ScheduleTask( | ||
| [this, batch = std::move(next), index]() mutable { | ||
| ExecBatch exec_batch(*batch); | ||
| exec_batch.index = index; | ||
| return output_->InputReceived(this, std::move(exec_batch)); | ||
| }, | ||
| "OrderByNode::ProcessBatch"); | ||
| } | ||
| } | ||
|
|
||
| protected: | ||
| std::string ToStringExtra(int indent = 0) const override { | ||
| std::stringstream ss; | ||
| ss << "ordering=" << ordering_.ToString(); | ||
| return ss.str(); | ||
| } | ||
|
|
||
| private: | ||
| AtomicCounter counter_; | ||
| Ordering ordering_; | ||
| std::vector<std::shared_ptr<RecordBatch>> accumulation_queue_; | ||
| std::mutex mutex_; | ||
| }; | ||
|
|
||
| } // namespace | ||
|
|
||
| namespace internal { | ||
|
|
||
| void RegisterOrderByNode(ExecFactoryRegistry* registry) { | ||
| DCHECK_OK( | ||
| registry->AddFactory(std::string(OrderByNodeOptions::kName), OrderByNode::Make)); | ||
| } | ||
|
|
||
| } // namespace internal | ||
| } // namespace compute | ||
| } // namespace arrow |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| #include <gtest/gtest.h> | ||
|
|
||
| #include <gmock/gmock-matchers.h> | ||
|
|
||
| #include "arrow/compute/exec/exec_plan.h" | ||
| #include "arrow/compute/exec/options.h" | ||
| #include "arrow/compute/exec/test_nodes.h" | ||
| #include "arrow/table.h" | ||
| #include "arrow/testing/generator.h" | ||
| #include "arrow/testing/gtest_util.h" | ||
| #include "arrow/testing/random.h" | ||
| #include "arrow/util/checked_cast.h" | ||
|
|
||
| namespace arrow { | ||
|
|
||
| using internal::checked_pointer_cast; | ||
|
|
||
| namespace compute { | ||
|
|
||
| // Sorting is slow, don't use too many rows | ||
| static constexpr int kRowsPerBatch = 4; | ||
| static constexpr int kNumBatches = 32; | ||
|
|
||
| std::shared_ptr<Table> TestTable() { | ||
| return gen::Gen({{"up", gen::Step()}, | ||
| {"down", gen::Step(/*start=*/0, /*step=*/-1, /*signed_int=*/true)}}) | ||
| ->FailOnError() | ||
| ->Table(kRowsPerBatch, kNumBatches); | ||
| } | ||
|
|
||
| void CheckOrderBy(OrderByNodeOptions options) { | ||
| constexpr random::SeedType kSeed = 42; | ||
| constexpr int kJitterMod = 4; | ||
| RegisterTestNodes(); | ||
| std::shared_ptr<Table> input = TestTable(); | ||
| Declaration plan = | ||
| Declaration::Sequence({{"table_source", TableSourceNodeOptions(input)}, | ||
| {"jitter", JitterNodeOptions(kSeed, kJitterMod)}, | ||
| {"order_by", options}}); | ||
| for (bool use_threads : {false, true}) { | ||
| QueryOptions query_options; | ||
| query_options.sequence_output = true; | ||
| query_options.use_threads = use_threads; | ||
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<Table> actual, | ||
| DeclarationToTable(plan, query_options)); | ||
|
|
||
| AssertTablesEqual(*input, *actual, /*same_chunk_layout=*/false); | ||
| } | ||
| } | ||
|
|
||
| void CheckOrderByInvalid(OrderByNodeOptions options, const std::string& message) { | ||
| std::shared_ptr<Table> input = TestTable(); | ||
| Declaration plan = Declaration::Sequence( | ||
| {{"table_source", TableSourceNodeOptions(input)}, {"order_by", options}}); | ||
| EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr(message), | ||
| DeclarationToStatus(std::move(plan))); | ||
| } | ||
|
|
||
| TEST(OrderByNode, Basic) { | ||
| CheckOrderBy(OrderByNodeOptions({{SortKey("up")}})); | ||
| CheckOrderBy(OrderByNodeOptions({{SortKey("down", SortOrder::Descending)}})); | ||
| CheckOrderBy( | ||
| OrderByNodeOptions({{SortKey("up"), SortKey("down", SortOrder::Descending)}})); | ||
| } | ||
|
|
||
| TEST(OrderByNode, Large) { | ||
| constexpr random::SeedType kSeed = 42; | ||
| constexpr int kJitterMod = 4; | ||
| static constexpr int kSmallNumBatches = 8; | ||
|
|
||
| RegisterTestNodes(); | ||
| // Confirm that the order by node chunks output for parallel processing | ||
| std::shared_ptr<Table> input = gen::Gen({{"up", gen::Step()}}) | ||
| ->FailOnError() | ||
| ->Table(ExecPlan::kMaxBatchSize, kSmallNumBatches); | ||
| Declaration plan = Declaration::Sequence({ | ||
| {"table_source", TableSourceNodeOptions(input)}, | ||
| {"order_by", OrderByNodeOptions({{SortKey("up", SortOrder::Descending)}})}, | ||
| {"jitter", JitterNodeOptions(kSeed, kJitterMod)}, | ||
| }); | ||
| ASSERT_OK_AND_ASSIGN(BatchesWithCommonSchema batches_and_schema, | ||
| DeclarationToExecBatches(std::move(plan))); | ||
| ASSERT_EQ(kSmallNumBatches, static_cast<int>(batches_and_schema.batches.size())); | ||
| // Jitter simulates parallel processing after the order by node. However, the batch | ||
| // index should be correct, regardless of the jitter | ||
| for (const auto& batch : batches_and_schema.batches) { | ||
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<Scalar> last_row, | ||
| batch.values[0].make_array()->GetScalar(batch.length - 1)); | ||
| uint32_t last_val = checked_pointer_cast<UInt32Scalar>(last_row)->value; | ||
| ASSERT_EQ(last_val, ExecPlan::kMaxBatchSize * (kSmallNumBatches - batch.index - 1)); | ||
| } | ||
| } | ||
|
|
||
| TEST(OrderByNode, Invalid) { | ||
| CheckOrderByInvalid(OrderByNodeOptions(Ordering::Implicit()), | ||
| "`ordering` must be an explicit non-empty ordering"); | ||
| CheckOrderByInvalid(OrderByNodeOptions(Ordering::Unordered()), | ||
| "`ordering` must be an explicit non-empty ordering"); | ||
| } | ||
|
|
||
| } // namespace compute | ||
| } // namespace arrow |
Uh oh!
There was an error while loading. Please reload this page.