Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -406,9 +406,9 @@ if(ARROW_COMPUTE)
compute/exec/key_hash.cc
compute/exec/key_map.cc
compute/exec/map_node.cc
compute/exec/options.cc
compute/exec/order_by_impl.cc
compute/exec/partition_util.cc
compute/exec/options.cc
compute/exec/project_node.cc
compute/exec/query_context.cc
compute/exec/sink_node.cc
Expand All @@ -422,6 +422,7 @@ if(ARROW_COMPUTE)
compute/function_internal.cc
compute/kernel.cc
compute/light_array.cc
compute/ordering.cc
compute/registry.cc
compute/kernels/aggregate_basic.cc
compute/kernels/aggregate_mode.cc
Expand Down
17 changes: 0 additions & 17 deletions cpp/src/arrow/compute/api_vector.cc
Original file line number Diff line number Diff line change
Expand Up @@ -118,23 +118,6 @@ namespace compute {
// ----------------------------------------------------------------------
// Function options

bool SortKey::Equals(const SortKey& other) const {
return target == other.target && order == other.order;
}
std::string SortKey::ToString() const {
std::stringstream ss;
ss << target.ToString() << ' ';
switch (order) {
case SortOrder::Ascending:
ss << "ASC";
break;
case SortOrder::Descending:
ss << "DESC";
break;
}
return ss.str();
}

namespace internal {
namespace {
using ::arrow::internal::DataMember;
Expand Down
32 changes: 1 addition & 31 deletions cpp/src/arrow/compute/api_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <utility>

#include "arrow/compute/function.h"
#include "arrow/compute/ordering.h"
#include "arrow/datum.h"
#include "arrow/result.h"
#include "arrow/type_fwd.h"
Expand Down Expand Up @@ -79,37 +80,6 @@ class ARROW_EXPORT DictionaryEncodeOptions : public FunctionOptions {
NullEncodingBehavior null_encoding_behavior = MASK;
};

enum class SortOrder {
/// Arrange values in increasing order
Ascending,
/// Arrange values in decreasing order
Descending,
};

enum class NullPlacement {
/// Place nulls and NaNs before any non-null values.
/// NaNs will come after nulls.
AtStart,
/// Place nulls and NaNs after any non-null values.
/// NaNs will come before nulls.
AtEnd,
};

/// \brief One sort key for PartitionNthIndices (TODO) and SortIndices
class ARROW_EXPORT SortKey : public util::EqualityComparable<SortKey> {
public:
explicit SortKey(FieldRef target, SortOrder order = SortOrder::Ascending)
: target(std::move(target)), order(order) {}

bool Equals(const SortKey& other) const;
std::string ToString() const;

/// A FieldRef targetting the sort column.
FieldRef target;
/// How to order by this sort key.
SortOrder order;
};

class ARROW_EXPORT ArraySortOptions : public FunctionOptions {
public:
explicit ArraySortOptions(SortOrder order = SortOrder::Ascending,
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/compute/exec/accumulation_queue.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <queue>
#include <vector>

#include "arrow/compute/exec.h"
#include "arrow/util/logging.h"

namespace arrow {
Expand Down Expand Up @@ -73,6 +74,7 @@ class SequencingQueueImpl : public SequencingQueue {
explicit SequencingQueueImpl(Processor* processor) : processor_(processor) {}

Status InsertBatch(ExecBatch batch) override {
DCHECK_NE(::arrow::compute::kUnsequencedIndex, batch.index);
std::unique_lock lk(mutex_);
if (batch.index == next_index_) {
return DeliverNextUnlocked(std::move(batch), std::move(lk));
Expand Down Expand Up @@ -122,6 +124,7 @@ class SerialSequencingQueueImpl : public SerialSequencingQueue {
explicit SerialSequencingQueueImpl(Processor* processor) : processor_(processor) {}

Status InsertBatch(ExecBatch batch) override {
DCHECK_NE(::arrow::compute::kUnsequencedIndex, batch.index);
std::unique_lock lk(mutex_);
queue_.push(std::move(batch));
if (queue_.top().index == next_index_ && !is_processing_) {
Expand Down
6 changes: 5 additions & 1 deletion cpp/src/arrow/compute/exec/asof_join_node.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1150,8 +1150,8 @@ class AsofJoinNode : public ExecNode {
if (result.ok()) {
auto out_rb = *result;
if (!out_rb) break;
++batches_produced_;
ExecBatch out_b(*out_rb);
out_b.index = batches_produced_++;
Status st = output_->InputReceived(this, std::move(out_b));
if (!st.ok()) {
EndFromProcessThread(std::move(st));
Expand Down Expand Up @@ -1491,6 +1491,7 @@ class AsofJoinNode : public ExecNode {
}

const char* kind_name() const override { return "AsofJoinNode"; }
const Ordering& ordering() const override { return ordering_; }

Status InputReceived(ExecNode* input, ExecBatch batch) override {
// Get the input
Expand Down Expand Up @@ -1540,6 +1541,8 @@ class AsofJoinNode : public ExecNode {
}

private:
// Outputs from this node are always in ascending order according to the on key
const Ordering ordering_;
std::vector<col_index_t> indices_of_on_key_;
std::vector<std::vector<col_index_t>> indices_of_by_key_;
std::vector<std::unique_ptr<KeyHasher>> key_hashers_;
Expand Down Expand Up @@ -1573,6 +1576,7 @@ AsofJoinNode::AsofJoinNode(ExecPlan* plan, NodeVector inputs,
bool must_hash, bool may_rehash)
: ExecNode(plan, inputs, input_labels,
/*output_schema=*/std::move(output_schema)),
ordering_({SortKey(indices_of_on_key[0])}),
indices_of_on_key_(std::move(indices_of_on_key)),
indices_of_by_key_(std::move(indices_of_by_key)),
key_hashers_(std::move(key_hashers)),
Expand Down
37 changes: 37 additions & 0 deletions cpp/src/arrow/compute/exec/asof_join_node_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1314,6 +1314,43 @@ TEST(AsofJoinTest, BackpressureWithBatches) {
/*fast_delay=*/0.01, /*slow_delay=*/0.1, /*noisy=*/false);
}

template <typename BatchesMaker>
void TestSequencing(BatchesMaker maker, int num_batches, int batch_size) {
auto l_schema =
schema({field("time", int32()), field("key", int32()), field("l_value", int32())});
auto r_schema =
schema({field("time", int32()), field("key", int32()), field("r0_value", int32())});

auto make_shift = [&maker, num_batches, batch_size](
const std::shared_ptr<Schema>& schema, int shift) {
return maker({[](int row) -> int64_t { return row; },
[num_batches](int row) -> int64_t { return row / num_batches; },
[shift](int row) -> int64_t { return row * 10 + shift; }},
schema, num_batches, batch_size);
};
ASSERT_OK_AND_ASSIGN(auto l_batches, make_shift(l_schema, 0));
ASSERT_OK_AND_ASSIGN(auto r_batches, make_shift(r_schema, 1));

compute::Declaration l_src = {"source",
SourceNodeOptions(l_schema, l_batches.gen(false, false))};
compute::Declaration r_src = {"source",
SourceNodeOptions(r_schema, r_batches.gen(false, false))};

compute::Declaration asofjoin = {
"asofjoin", {l_src, r_src}, GetRepeatedOptions(2, "time", {"key"}, 1000)};

QueryOptions query_options;
query_options.use_threads = false;
ASSERT_OK_AND_ASSIGN(BatchesWithCommonSchema batches,
DeclarationToExecBatches(asofjoin, query_options));

AssertExecBatchesSequenced(batches.batches);
}

TEST(AsofJoinTest, BatchSequencing) {
return TestSequencing(MakeIntegerBatches, /*num_batches=*/32, /*batch_size=*/1);
}

namespace {

Result<AsyncGenerator<std::optional<ExecBatch>>> MakeIntegerBatchGenForTest(
Expand Down
48 changes: 41 additions & 7 deletions cpp/src/arrow/compute/exec/exec_plan.cc
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,11 @@ ExecNode::ExecNode(ExecPlan* plan, NodeVector inputs,
}
}

const Ordering& ExecNode::ordering() const {
// The safest default is to assume a node destroys ordering
return Ordering::Unordered();
}

Status ExecNode::Init() { return Status::OK(); }

Status ExecNode::Validate() const {
Expand Down Expand Up @@ -644,15 +649,19 @@ Future<BatchesWithCommonSchema> DeclarationToExecBatchesImpl(
ExecContext exec_ctx(options.memory_pool, cpu_executor, options.function_registry);
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan, ExecPlan::Make(exec_ctx));
SinkNodeOptions sink_options(&sink_gen, &out_schema);
sink_options.sequence_delivery = options.sequence_output;
sink_options.sequence_output = options.sequence_output;
Declaration with_sink = Declaration::Sequence({declaration, {"sink", sink_options}});
ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
ARROW_RETURN_NOT_OK(exec_plan->Validate());
exec_plan->StartProducing();
auto collected_fut = CollectAsyncGenerator(sink_gen);
return AllFinished({exec_plan->finished(), Future<>(collected_fut)})
.Then([collected_fut, exec_plan,
schema = std::move(out_schema)]() -> Result<BatchesWithCommonSchema> {
return exec_plan->finished().Then(
[collected_fut, exec_plan,
schema = std::move(out_schema)]() -> Result<BatchesWithCommonSchema> {
if (!collected_fut.is_finished()) {
return Status::Invalid(
"Plan finished but it did not emit the expected number of batches.");
}
ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
std::vector<ExecBatch> exec_batches = ::arrow::internal::MapVector(
[](std::optional<ExecBatch> batch) { return batch.value_or(ExecBatch()); },
Expand All @@ -667,9 +676,9 @@ Future<> DeclarationToStatusImpl(Declaration declaration, QueryOptions options,
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan, ExecPlan::Make(exec_ctx));
ARROW_ASSIGN_OR_RAISE(ExecNode * last_node, declaration.AddToPlan(exec_plan.get()));
if (!last_node->is_sink()) {
Declaration null_sink =
Declaration("consuming_sink", {last_node},
ConsumingSinkNodeOptions(NullSinkNodeConsumer::Make()));
ConsumingSinkNodeOptions sink_options(NullSinkNodeConsumer::Make());
sink_options.sequence_output = options.sequence_output;
Declaration null_sink = Declaration("consuming_sink", {last_node}, sink_options);
ARROW_RETURN_NOT_OK(null_sink.AddToPlan(exec_plan.get()));
}
ARROW_RETURN_NOT_OK(exec_plan->Validate());
Expand Down Expand Up @@ -826,6 +835,20 @@ Result<BatchesWithCommonSchema> DeclarationToExecBatches(
use_threads);
}

Result<BatchesWithCommonSchema> DeclarationToExecBatches(Declaration declaration,
QueryOptions query_options) {
if (query_options.custom_cpu_executor != nullptr) {
return Status::Invalid("Cannot use synchronous methods with a custom CPU executor");
}
return ::arrow::internal::RunSynchronously<Future<BatchesWithCommonSchema>>(
[=, declaration =
std::move(declaration)](::arrow::internal::Executor* executor) mutable {
return DeclarationToExecBatchesImpl(std::move(declaration), query_options,
executor);
},
query_options.use_threads);
}

Future<> DeclarationToStatusAsync(Declaration declaration, ExecContext exec_context) {
return DeclarationToStatusImpl(std::move(declaration),
QueryOptionsFromCustomExecContext(exec_context),
Expand Down Expand Up @@ -858,6 +881,17 @@ Status DeclarationToStatus(Declaration declaration, bool use_threads,
use_threads);
}

Status DeclarationToStatus(Declaration declaration, QueryOptions query_options) {
if (query_options.custom_cpu_executor != nullptr) {
return Status::Invalid("Cannot use synchronous methods with a custom CPU executor");
}
return ::arrow::internal::RunSynchronously<Future<>>(
[=, declaration = std::move(declaration)](::arrow::internal::Executor* executor) {
return DeclarationToStatusImpl(std::move(declaration), query_options, executor);
},
query_options.use_threads);
}

namespace {
struct BatchConverter {
~BatchConverter() {
Expand Down
Loading