Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,7 @@ list(APPEND
compute/exec/map_node.cc
compute/exec/options.cc
compute/exec/order_by_impl.cc
compute/exec/order_by_node.cc
compute/exec/partition_util.cc
compute/exec/pivot_longer_node.cc
compute/exec/project_node.cc
Expand Down
4 changes: 4 additions & 0 deletions cpp/src/arrow/compute/api_vector.cc
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,10 @@ SortOptions::SortOptions(std::vector<SortKey> sort_keys, NullPlacement null_plac
: FunctionOptions(internal::kSortOptionsType),
sort_keys(std::move(sort_keys)),
null_placement(null_placement) {}
SortOptions::SortOptions(const Ordering& ordering)
: FunctionOptions(internal::kSortOptionsType),
sort_keys(ordering.sort_keys()),
null_placement(ordering.null_placement()) {}
constexpr char SortOptions::kTypeName[];

PartitionNthOptions::PartitionNthOptions(int64_t pivot, NullPlacement null_placement)
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/arrow/compute/api_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,16 @@ class ARROW_EXPORT SortOptions : public FunctionOptions {
public:
explicit SortOptions(std::vector<SortKey> sort_keys = {},
NullPlacement null_placement = NullPlacement::AtEnd);
explicit SortOptions(const Ordering& ordering);
static constexpr char const kTypeName[] = "SortOptions";
static SortOptions Defaults() { return SortOptions(); }
/// Convenience constructor to create an ordering from SortOptions
///
/// Note: Both classes contain the exact same information. However,
/// sort_options should only be used in a "function options" context while Ordering
/// is used more generally.
Ordering AsOrdering() && { return Ordering(std::move(sort_keys), null_placement); }
Ordering AsOrdering() const& { return Ordering(sort_keys, null_placement); }

/// Column key(s) to order by and how to order by these sort keys.
std::vector<SortKey> sort_keys;
Expand Down
6 changes: 6 additions & 0 deletions cpp/src/arrow/compute/exec/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ add_arrow_compute_test(hash_join_node_test
SOURCES
hash_join_node_test.cc
bloom_filter_test.cc)
add_arrow_compute_test(order_by_node_test
PREFIX
"arrow-compute"
SOURCES
order_by_node_test.cc
test_nodes.cc)
add_arrow_compute_test(pivot_longer_node_test
PREFIX
"arrow-compute"
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/compute/exec/exec_plan.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1053,6 +1053,7 @@ namespace internal {
void RegisterSourceNode(ExecFactoryRegistry*);
void RegisterFetchNode(ExecFactoryRegistry*);
void RegisterFilterNode(ExecFactoryRegistry*);
void RegisterOrderByNode(ExecFactoryRegistry*);
void RegisterPivotLongerNode(ExecFactoryRegistry*);
void RegisterProjectNode(ExecFactoryRegistry*);
void RegisterUnionNode(ExecFactoryRegistry*);
Expand All @@ -1070,6 +1071,7 @@ ExecFactoryRegistry* default_exec_factory_registry() {
internal::RegisterSourceNode(this);
internal::RegisterFetchNode(this);
internal::RegisterFilterNode(this);
internal::RegisterOrderByNode(this);
internal::RegisterPivotLongerNode(this);
internal::RegisterProjectNode(this);
internal::RegisterUnionNode(this);
Expand Down
15 changes: 15 additions & 0 deletions cpp/src/arrow/compute/exec/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,21 @@ class ARROW_EXPORT OrderBySinkNodeOptions : public SinkNodeOptions {
SortOptions sort_options;
};

/// \brief Apply a new ordering to data
///
/// Currently this node works by accumulating all data, sorting, and then emitting
/// the new data with an updated batch index.
///
/// Larger-than-memory sort is not currently supported.
class ARROW_EXPORT OrderByNodeOptions : public ExecNodeOptions {
Comment thread
lidavidm marked this conversation as resolved.
Resolved
public:
static constexpr std::string_view kName = "order_by";
explicit OrderByNodeOptions(Ordering ordering) : ordering(std::move(ordering)) {}

/// \brief The new ordering to apply to outgoing data
Ordering ordering;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The OrderBySinkNode is using SortOptions as argument instead of Ordering, but looking into those two classes, that should basically be equivalent? (SortOptions also accepts a vector of SortKey and a null_placement arg, just as the Ordering class)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, SortOptions and Ordering are equivalent. However, Ordering doesn't extend from FunctionOptions. I wasn't sure how to proceed here so I ended up creating a new type. It felt a bit weird for a FunctionOptions to be used in exec plans completely unrelated to running functions. Although, to play devil's advocate, FunctionOptions is "mostly" just a marker interface and doesn't really prevent anything from working so I would open to just using that.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the Ordering certainly makes sense, and it's quite trivial to convert the one into the other, I was just wondering for a consistent API if we want to add a constructor version that accepts the SortOptions (or for OrderBySinkNode to add a version that accepts Ordering)

};

/// @}

enum class JoinType {
Expand Down
166 changes: 166 additions & 0 deletions cpp/src/arrow/compute/exec/order_by_node.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include <cstdint>
#include <memory>
#include <mutex>
#include <sstream>
#include <utility>
#include <vector>

#include "arrow/compute/exec/exec_plan.h"
#include "arrow/compute/exec/options.h"
#include "arrow/compute/exec/query_context.h"
#include "arrow/compute/exec/util.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/tracing_internal.h"

namespace arrow {

using internal::checked_cast;

namespace compute {
namespace {

class OrderByNode : public ExecNode, public TracedNode {
public:
OrderByNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
std::shared_ptr<Schema> output_schema, Ordering new_ordering)
: ExecNode(plan, std::move(inputs), {"input"}, std::move(output_schema)),
TracedNode(this),
ordering_(std::move(new_ordering)) {}

static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
const ExecNodeOptions& options) {
RETURN_NOT_OK(ValidateExecNodeInputs(plan, inputs, 1, "FetchNode"));

const auto& order_options = checked_cast<const OrderByNodeOptions&>(options);

if (order_options.ordering.is_implicit() || order_options.ordering.is_unordered()) {
return Status::Invalid("`ordering` must be an explicit non-empty ordering");
}

std::shared_ptr<Schema> output_schema = inputs[0]->output_schema();
return plan->EmplaceNode<OrderByNode>(
plan, std::move(inputs), std::move(output_schema), order_options.ordering);
}

const char* kind_name() const override { return "OrderByNode"; }

const Ordering& ordering() const override { return ordering_; }

Status InputFinished(ExecNode* input, int total_batches) override {
DCHECK_EQ(input, inputs_[0]);
EVENT_ON_CURRENT_SPAN("InputFinished", {{"batches.length", total_batches}});
// We can't send InputFinished downstream because we might change the # of batches
// when we sort it. So that happens later in DoFinish
if (counter_.SetTotal(total_batches)) {
return DoFinish();
}
return Status::OK();
}

Status StartProducing() override {
NoteStartProducing(ToStringExtra());
return Status::OK();
}

void PauseProducing(ExecNode* output, int32_t counter) override {
inputs_[0]->PauseProducing(this, counter);
}

void ResumeProducing(ExecNode* output, int32_t counter) override {
inputs_[0]->ResumeProducing(this, counter);
}

Status StopProducingImpl() override { return Status::OK(); }

Status InputReceived(ExecNode* input, ExecBatch batch) override {
auto scope = TraceInputReceived(batch);
DCHECK_EQ(input, inputs_[0]);

ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> record_batch,
batch.ToRecordBatch(output_schema_));

{
std::lock_guard lk(mutex_);
accumulation_queue_.push_back(std::move(record_batch));
}

if (counter_.Increment()) {
return DoFinish();
}
return Status::OK();
}

Status DoFinish() {
ARROW_ASSIGN_OR_RAISE(
auto table,
Table::FromRecordBatches(output_schema_, std::move(accumulation_queue_)));
SortOptions sort_options(ordering_.sort_keys(), ordering_.null_placement());
ExecContext* ctx = plan_->query_context()->exec_context();
ARROW_ASSIGN_OR_RAISE(auto indices, SortIndices(table, sort_options, ctx));
ARROW_ASSIGN_OR_RAISE(Datum sorted,
Take(table, indices, TakeOptions::NoBoundsCheck(), ctx));
const std::shared_ptr<Table>& sorted_table = sorted.table();
TableBatchReader reader(*sorted_table);
reader.set_chunksize(ExecPlan::kMaxBatchSize);
int batch_index = 0;
while (true) {
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> next, reader.Next());
if (!next) {
return output_->InputFinished(this, batch_index);
}
int index = batch_index++;
plan_->query_context()->ScheduleTask(
[this, batch = std::move(next), index]() mutable {
ExecBatch exec_batch(*batch);
exec_batch.index = index;
return output_->InputReceived(this, std::move(exec_batch));
},
"OrderByNode::ProcessBatch");
}
}

protected:
std::string ToStringExtra(int indent = 0) const override {
std::stringstream ss;
ss << "ordering=" << ordering_.ToString();
return ss.str();
}

private:
AtomicCounter counter_;
Ordering ordering_;
std::vector<std::shared_ptr<RecordBatch>> accumulation_queue_;
std::mutex mutex_;
};

} // namespace

namespace internal {

void RegisterOrderByNode(ExecFactoryRegistry* registry) {
DCHECK_OK(
registry->AddFactory(std::string(OrderByNodeOptions::kName), OrderByNode::Make));
}

} // namespace internal
} // namespace compute
} // namespace arrow
119 changes: 119 additions & 0 deletions cpp/src/arrow/compute/exec/order_by_node_test.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include <gtest/gtest.h>

#include <gmock/gmock-matchers.h>

#include "arrow/compute/exec/exec_plan.h"
#include "arrow/compute/exec/options.h"
#include "arrow/compute/exec/test_nodes.h"
#include "arrow/table.h"
#include "arrow/testing/generator.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/random.h"
#include "arrow/util/checked_cast.h"

namespace arrow {

using internal::checked_pointer_cast;

namespace compute {

// Sorting is slow, don't use too many rows
static constexpr int kRowsPerBatch = 4;
static constexpr int kNumBatches = 32;

std::shared_ptr<Table> TestTable() {
return gen::Gen({{"up", gen::Step()},
{"down", gen::Step(/*start=*/0, /*step=*/-1, /*signed_int=*/true)}})
->FailOnError()
->Table(kRowsPerBatch, kNumBatches);
}

void CheckOrderBy(OrderByNodeOptions options) {
constexpr random::SeedType kSeed = 42;
constexpr int kJitterMod = 4;
RegisterTestNodes();
std::shared_ptr<Table> input = TestTable();
Declaration plan =
Declaration::Sequence({{"table_source", TableSourceNodeOptions(input)},
{"jitter", JitterNodeOptions(kSeed, kJitterMod)},
{"order_by", options}});
for (bool use_threads : {false, true}) {
QueryOptions query_options;
query_options.sequence_output = true;
query_options.use_threads = use_threads;
ASSERT_OK_AND_ASSIGN(std::shared_ptr<Table> actual,
DeclarationToTable(plan, query_options));

AssertTablesEqual(*input, *actual, /*same_chunk_layout=*/false);
}
}

void CheckOrderByInvalid(OrderByNodeOptions options, const std::string& message) {
std::shared_ptr<Table> input = TestTable();
Declaration plan = Declaration::Sequence(
{{"table_source", TableSourceNodeOptions(input)}, {"order_by", options}});
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr(message),
DeclarationToStatus(std::move(plan)));
}

TEST(OrderByNode, Basic) {
CheckOrderBy(OrderByNodeOptions({{SortKey("up")}}));
CheckOrderBy(OrderByNodeOptions({{SortKey("down", SortOrder::Descending)}}));
CheckOrderBy(
OrderByNodeOptions({{SortKey("up"), SortKey("down", SortOrder::Descending)}}));
}

TEST(OrderByNode, Large) {
constexpr random::SeedType kSeed = 42;
constexpr int kJitterMod = 4;
static constexpr int kSmallNumBatches = 8;

RegisterTestNodes();
// Confirm that the order by node chunks output for parallel processing
std::shared_ptr<Table> input = gen::Gen({{"up", gen::Step()}})
->FailOnError()
->Table(ExecPlan::kMaxBatchSize, kSmallNumBatches);
Declaration plan = Declaration::Sequence({
{"table_source", TableSourceNodeOptions(input)},
{"order_by", OrderByNodeOptions({{SortKey("up", SortOrder::Descending)}})},
{"jitter", JitterNodeOptions(kSeed, kJitterMod)},
});
ASSERT_OK_AND_ASSIGN(BatchesWithCommonSchema batches_and_schema,
DeclarationToExecBatches(std::move(plan)));
ASSERT_EQ(kSmallNumBatches, static_cast<int>(batches_and_schema.batches.size()));
// Jitter simulates parallel processing after the order by node. However, the batch
// index should be correct, regardless of the jitter
for (const auto& batch : batches_and_schema.batches) {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<Scalar> last_row,
batch.values[0].make_array()->GetScalar(batch.length - 1));
uint32_t last_val = checked_pointer_cast<UInt32Scalar>(last_row)->value;
ASSERT_EQ(last_val, ExecPlan::kMaxBatchSize * (kSmallNumBatches - batch.index - 1));
}
}

TEST(OrderByNode, Invalid) {
CheckOrderByInvalid(OrderByNodeOptions(Ordering::Implicit()),
"`ordering` must be an explicit non-empty ordering");
CheckOrderByInvalid(OrderByNodeOptions(Ordering::Unordered()),
"`ordering` must be an explicit non-empty ordering");
}

} // namespace compute
} // namespace arrow
4 changes: 2 additions & 2 deletions cpp/src/arrow/compute/ordering.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ class ARROW_EXPORT Ordering : public util::EqualityComparable<Ordering> {
bool is_implicit() const { return is_implicit_; }
bool is_unordered() const { return !is_implicit_ && sort_keys_.empty(); }

std::vector<SortKey> sort_keys() { return sort_keys_; }
NullPlacement null_placement() { return null_placement_; }
const std::vector<SortKey>& sort_keys() const { return sort_keys_; }
NullPlacement null_placement() const { return null_placement_; }

static const Ordering& Implicit() {
static const Ordering kImplicit(true);
Expand Down
Loading