From 84ae4c4403867d92b20b91d8ac7e6e1da67289b3 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Thu, 1 Dec 2022 04:27:48 -0500 Subject: [PATCH 1/9] ARROW-18417: [C++] Support emit info in Substrait extension-multi and AsOfJoin --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 23 ++ cpp/proto/substrait/extension_rels.proto | 44 ++ cpp/src/arrow/compute/exec/asof_join_node.cc | 358 +++++++++++++--- cpp/src/arrow/compute/exec/asof_join_node.h | 46 +++ .../arrow/compute/exec/asof_join_node_test.cc | 27 +- cpp/src/arrow/compute/exec/options.h | 41 +- cpp/src/arrow/engine/CMakeLists.txt | 1 + .../arrow/engine/substrait/extension_set.h | 8 +- cpp/src/arrow/engine/substrait/options.cc | 115 ++++++ cpp/src/arrow/engine/substrait/options.h | 17 +- cpp/src/arrow/engine/substrait/relation.h | 50 +++ .../engine/substrait/relation_internal.cc | 148 ++++++- .../engine/substrait/relation_internal.h | 9 +- cpp/src/arrow/engine/substrait/serde_test.cc | 385 ++++++++++++++++++ cpp/src/arrow/engine/substrait/type_fwd.h | 2 + 15 files changed, 1176 insertions(+), 98 deletions(-) create mode 100644 cpp/proto/substrait/extension_rels.proto create mode 100644 cpp/src/arrow/compute/exec/asof_join_node.h create mode 100644 cpp/src/arrow/engine/substrait/options.cc create mode 100644 cpp/src/arrow/engine/substrait/relation.h diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ac239daed011..d0e5d8fb7b42 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1715,6 +1715,8 @@ macro(build_substrait) # Note: not all protos in Substrait actually matter to plan # consumption. No need to build the ones we don't need. set(SUBSTRAIT_PROTOS algebra extensions/extensions plan type) + set(ARROW_SUBSTRAIT_PROTOS extension_rels) + set(ARROW_SUBSTRAIT_PROTOS_DIR "${CMAKE_SOURCE_DIR}/proto") externalproject_add(substrait_ep CONFIGURE_COMMAND "" @@ -1765,6 +1767,27 @@ macro(build_substrait) list(APPEND SUBSTRAIT_SOURCES "${SUBSTRAIT_PROTO_GEN}.cc") endforeach() + foreach(ARROW_SUBSTRAIT_PROTO ${ARROW_SUBSTRAIT_PROTOS}) + set(ARROW_SUBSTRAIT_PROTO_GEN + "${SUBSTRAIT_CPP_DIR}/substrait/${ARROW_SUBSTRAIT_PROTO}.pb") + foreach(EXT h cc) + set_source_files_properties("${ARROW_SUBSTRAIT_PROTO_GEN}.${EXT}" + PROPERTIES COMPILE_OPTIONS + "${SUBSTRAIT_SUPPRESSED_FLAGS}" + GENERATED TRUE + SKIP_UNITY_BUILD_INCLUSION TRUE) + list(APPEND SUBSTRAIT_PROTO_GEN_ALL "${ARROW_SUBSTRAIT_PROTO_GEN}.${EXT}") + endforeach() + add_custom_command(OUTPUT "${ARROW_SUBSTRAIT_PROTO_GEN}.cc" + "${ARROW_SUBSTRAIT_PROTO_GEN}.h" + COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${SUBSTRAIT_LOCAL_DIR}/proto" + "-I${ARROW_SUBSTRAIT_PROTOS_DIR}" + "--cpp_out=${SUBSTRAIT_CPP_DIR}" + "${ARROW_SUBSTRAIT_PROTOS_DIR}/substrait/${ARROW_SUBSTRAIT_PROTO}.proto" + DEPENDS ${PROTO_DEPENDS} substrait_ep) + + list(APPEND SUBSTRAIT_SOURCES "${ARROW_SUBSTRAIT_PROTO_GEN}.cc") + endforeach() add_custom_target(substrait_gen ALL DEPENDS ${SUBSTRAIT_PROTO_GEN_ALL}) diff --git a/cpp/proto/substrait/extension_rels.proto b/cpp/proto/substrait/extension_rels.proto new file mode 100644 index 000000000000..df4d29a0a383 --- /dev/null +++ b/cpp/proto/substrait/extension_rels.proto @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +syntax = "proto3"; + +package arrow.substrait_ext; + +import "substrait/algebra.proto"; + +option csharp_namespace = "Arrow.Substrait"; +option go_package = "github.com/apache/arrow/substrait"; +option java_multiple_files = true; +option java_package = "io.arrow.substrait"; + +// As-Of-Join relation +message AsOfJoinRel { + // One key per input relation, each key describing how to join the corresponding input + repeated AsOfJoinKey keys = 1; + // As-Of tolerance, in units of the on-key + int64 tolerance = 2; + + // As-Of-Join key + message AsOfJoinKey { + // A field reference defining the on-key + // The type and units of the referenced field must be the same across all inputs + .substrait.Expression on = 1; + // A list of field references defining the by-key + // The types corresponding to the referenced fields must be the same across all inputs + repeated .substrait.Expression by = 2; + } +} diff --git a/cpp/src/arrow/compute/exec/asof_join_node.cc b/cpp/src/arrow/compute/exec/asof_join_node.cc index 22f8e5142ddb..c259f6a2455d 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node.cc @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "arrow/compute/exec/asof_join_node.h" + #include #include #include @@ -88,6 +90,10 @@ class ConcurrentQueue { T Pop() { std::unique_lock lock(mutex_); cond_.wait(lock, [&] { return !queue_.empty(); }); + return PopUnlocked(); + } + + T PopUnlocked() { auto item = queue_.front(); queue_.pop(); return item; @@ -95,18 +101,28 @@ class ConcurrentQueue { void Push(const T& item) { std::unique_lock lock(mutex_); + return PushUnlocked(item); + } + + void PushUnlocked(const T& item) { queue_.push(item); cond_.notify_one(); } void Clear() { std::unique_lock lock(mutex_); - queue_ = std::queue(); + ClearUnlocked(); } + void ClearUnlocked() { queue_ = std::queue(); } + std::optional TryPop() { - // Try to pop the oldest value from the queue (or return nullopt if none) std::unique_lock lock(mutex_); + return TryPopUnlocked(); + } + + std::optional TryPopUnlocked() { + // Try to pop the oldest value from the queue (or return nullopt if none) if (queue_.empty()) { return std::nullopt; } else { @@ -127,6 +143,11 @@ class ConcurrentQueue { // 2) pop/try_pop cannot be called concurrently with this const T& UnsyncFront() const { return queue_.front(); } + size_t UnsyncSize() const { return queue_.size(); } + + protected: + std::mutex& GetMutex() { return mutex_; } + private: std::queue queue_; mutable std::mutex mutex_; @@ -236,6 +257,107 @@ class KeyHasher { util::TempVectorStack stack_; }; +class BackpressureController : public BackpressureControl { + public: + BackpressureController(ExecNode* node, ExecNode* output, + std::atomic& backpressure_counter) + : node_(node), output_(output), backpressure_counter_(backpressure_counter) {} + + void Pause() override { node_->PauseProducing(output_, ++backpressure_counter_); } + void Resume() override { node_->ResumeProducing(output_, ++backpressure_counter_); } + + private: + ExecNode* node_; + ExecNode* output_; + std::atomic& backpressure_counter_; +}; + +class BackpressureHandler { + private: + BackpressureHandler(size_t low_threshold, size_t high_threshold, + std::unique_ptr backpressure_control) + : low_threshold_(low_threshold), + high_threshold_(high_threshold), + backpressure_control_(std::move(backpressure_control)) {} + + public: + static Result Make( + size_t low_threshold, size_t high_threshold, + std::unique_ptr backpressure_control) { + if (low_threshold >= high_threshold) { + return Status::Invalid("low threshold (", low_threshold, + ") must be less than high threshold (", high_threshold, ")"); + } + if (backpressure_control == NULLPTR) { + return Status::Invalid("null backpressure control parameter"); + } + BackpressureHandler backpressure_handler(low_threshold, high_threshold, + std::move(backpressure_control)); + return std::move(backpressure_handler); + } + + void Handle(size_t start_level, size_t end_level) { + if (start_level < high_threshold_ && end_level >= high_threshold_) { + backpressure_control_->Pause(); + } else if (start_level > low_threshold_ && end_level <= low_threshold_) { + backpressure_control_->Resume(); + } + } + + private: + size_t low_threshold_; + size_t high_threshold_; + std::unique_ptr backpressure_control_; +}; + +template +class BackpressureConcurrentQueue : public ConcurrentQueue { + private: + struct DoHandle { + explicit DoHandle(BackpressureConcurrentQueue& queue) + : queue_(queue), start_size_(queue_.UnsyncSize()) {} + + ~DoHandle() { + size_t end_size = queue_.UnsyncSize(); + queue_.handler_.Handle(start_size_, end_size); + } + + BackpressureConcurrentQueue& queue_; + size_t start_size_; + }; + + public: + explicit BackpressureConcurrentQueue(BackpressureHandler handler) + : handler_(std::move(handler)) {} + + T Pop() { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + return ConcurrentQueue::PopUnlocked(); + } + + void Push(const T& item) { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + ConcurrentQueue::PushUnlocked(item); + } + + void Clear() { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + ConcurrentQueue::ClearUnlocked(); + } + + std::optional TryPop() { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + return ConcurrentQueue::TryPopUnlocked(); + } + + private: + BackpressureHandler handler_; +}; + class InputState { // InputState correponds to an input // Input record batches are queued up in InputState until processed and @@ -243,10 +365,10 @@ class InputState { public: InputState(bool must_hash, bool may_rehash, KeyHasher* key_hasher, - const std::shared_ptr& schema, + BackpressureHandler handler, const std::shared_ptr& schema, const col_index_t time_col_index, const std::vector& key_col_index) - : queue_(), + : queue_(std::move(handler)), schema_(schema), time_col_index_(time_col_index), key_col_index_(key_col_index), @@ -260,6 +382,22 @@ class InputState { } } + static Result> Make( + bool must_hash, bool may_rehash, KeyHasher* key_hasher, ExecNode* node, + ExecNode* output, std::atomic& backpressure_counter, + const std::shared_ptr& schema, const col_index_t time_col_index, + const std::vector& key_col_index) { + constexpr size_t low_threshold = 4, high_threshold = 8; + std::unique_ptr backpressure_control = + std::make_unique(node, output, backpressure_counter); + ARROW_ASSIGN_OR_RAISE(auto handler, + BackpressureHandler::Make(low_threshold, high_threshold, + std::move(backpressure_control))); + return std::make_unique(must_hash, may_rehash, key_hasher, + std::move(handler), schema, time_col_index, + key_col_index); + } + col_index_t InitSrcToDstMapping(col_index_t dst_offset, bool skip_time_and_key_fields) { src_to_dst_.resize(schema_->num_fields()); for (int i = 0; i < schema_->num_fields(); ++i) @@ -468,7 +606,7 @@ class InputState { private: // Pending record batches. The latest is the front. Batches cannot be empty. - ConcurrentQueue> queue_; + BackpressureConcurrentQueue> queue_; // Schema associated with the input std::shared_ptr schema_; // Total number of batches (only int because InputFinished uses int) @@ -668,20 +806,29 @@ class CompositeReferenceTable { void AddRecordBatchRef(const std::shared_ptr& ref) { if (!_ptr2ref.count((uintptr_t)ref.get())) _ptr2ref[(uintptr_t)ref.get()] = ref; } + template ::BuilderType> - enable_if_fixed_width_type static BuilderAppend( + enable_if_boolean static BuilderAppend( Builder& builder, const std::shared_ptr& source, row_index_t row) { if (source->IsNull(row)) { builder.UnsafeAppendNull(); return Status::OK(); } + builder.UnsafeAppend(bit_util::GetBit(source->template GetValues(1), row)); + return Status::OK(); + } - if constexpr (is_boolean_type::value) { - builder.UnsafeAppend(bit_util::GetBit(source->template GetValues(1), row)); - } else { - using CType = typename TypeTraits::CType; - builder.UnsafeAppend(source->template GetValues(1)[row]); + template ::BuilderType> + enable_if_t::value && !is_boolean_type::value, + Status> static BuilderAppend(Builder& builder, + const std::shared_ptr& source, + row_index_t row) { + if (source->IsNull(row)) { + builder.UnsafeAppendNull(); + return Status::OK(); } + using CType = typename TypeTraits::CType; + builder.UnsafeAppend(source->template GetValues(1)[row]); return Status::OK(); } @@ -806,10 +953,29 @@ class AsofJoinNode : public ExecNode { } } - void Process() { + template + struct Defer { + Callable callable; + explicit Defer(Callable callable) : callable(std::move(callable)) {} + ~Defer() noexcept { callable(); } + }; + + bool CheckEnded() { + if (state_.at(0)->Finished()) { + ErrorIfNotOk(plan_->ScheduleTask([this] { + Defer cleanup([this]() { finished_.MarkFinished(); }); + outputs_[0]->InputFinished(this, batches_produced_); + return Status::OK(); + })); + return false; + } + return true; + } + + bool Process() { std::lock_guard guard(gate_); - if (finished_.is_finished()) { - return; + if (!CheckEnded()) { + return false; } // Process batches while we have data @@ -821,10 +987,13 @@ class AsofJoinNode : public ExecNode { if (!out_rb) break; ++batches_produced_; ExecBatch out_b(*out_rb); - outputs_[0]->InputReceived(this, std::move(out_b)); + ErrorIfNotOk(plan_->ScheduleTask([this, out_b = std::move(out_b)]() mutable { + outputs_[0]->InputReceived(this, std::move(out_b)); + return Status::OK(); + })); } else { ErrorIfNotOk(result.status()); - return; + return false; } } @@ -833,10 +1002,13 @@ class AsofJoinNode : public ExecNode { // // It may happen here in cases where InputFinished was called before we were finished // producing results (so we didn't know the output size at that time) - if (state_.at(0)->Finished()) { - outputs_[0]->InputFinished(this, batches_produced_); - finished_.MarkFinished(); + if (!CheckEnded()) { + return false; } + + // There is no more we can do now but there is still work remaining for later when + // more data arrives. + return true; } void ProcessThread() { @@ -844,7 +1016,9 @@ class AsofJoinNode : public ExecNode { if (!process_.Pop()) { return; } - Process(); + if (!Process()) { + return; + } } } @@ -862,9 +1036,12 @@ class AsofJoinNode : public ExecNode { auto inputs = this->inputs(); for (size_t i = 0; i < inputs.size(); i++) { RETURN_NOT_OK(key_hashers_[i]->Init(plan()->exec_context(), output_schema())); - state_.push_back(std::make_unique( - must_hash_, may_rehash_, key_hashers_[i].get(), inputs[i]->output_schema(), - indices_of_on_key_[i], indices_of_by_key_[i])); + ARROW_ASSIGN_OR_RAISE( + auto input_state, + InputState::Make(must_hash_, may_rehash_, key_hashers_[i].get(), inputs[i], + this, backpressure_counter_, inputs[i]->output_schema(), + indices_of_on_key_[i], indices_of_by_key_[i])); + state_.push_back(std::move(input_state)); } col_index_t dst_offset = 0; @@ -875,8 +1052,12 @@ class AsofJoinNode : public ExecNode { } virtual ~AsofJoinNode() { - process_.Push(false); // poison pill - process_thread_.join(); + process_.Push(false); // poison pill + if (process_thread_.get_id() != std::this_thread::get_id()) { // avoid deadlock + process_thread_.join(); + } else { + process_thread_.detach(); + } } const std::vector& indices_of_on_key() { return indices_of_on_key_; } @@ -962,17 +1143,18 @@ class AsofJoinNode : public ExecNode { } static arrow::Result> MakeOutputSchema( - const std::vector& inputs, + const std::vector> input_schema, const std::vector& indices_of_on_key, - const std::vector>& indices_of_by_key) { + const std::vector>& indices_of_by_key, + std::vector* field_output_indices = nullptr) { std::vector> fields; - size_t n_by = indices_of_by_key[0].size(); + size_t n_by = indices_of_by_key.size() == 0 ? 0 : indices_of_by_key[0].size(); const DataType* on_key_type = NULLPTR; std::vector by_key_type(n_by, NULLPTR); // Take all non-key, non-time RHS fields - for (size_t j = 0; j < inputs.size(); ++j) { - const auto& input_schema = inputs[j]->output_schema(); + int output_field_idx = 0; + for (size_t j = 0; j < input_schema.size(); ++j) { const auto& on_field_ix = indices_of_on_key[j]; const auto& by_field_ix = indices_of_by_key[j]; @@ -980,10 +1162,10 @@ class AsofJoinNode : public ExecNode { return Status::Invalid("Missing join key on table ", j); } - const auto& on_field = input_schema->fields()[on_field_ix]; + const auto& on_field = input_schema[j]->fields()[on_field_ix]; std::vector by_field(n_by); for (size_t k = 0; k < n_by; k++) { - by_field[k] = input_schema->fields()[by_field_ix[k]].get(); + by_field[k] = input_schema[j]->fields()[by_field_ix[k]].get(); } if (on_key_type == NULLPTR) { @@ -1003,24 +1185,27 @@ class AsofJoinNode : public ExecNode { } } - for (int i = 0; i < input_schema->num_fields(); ++i) { - const auto field = input_schema->field(i); + for (int i = 0; i < input_schema[j]->num_fields(); ++i) { + const auto field = input_schema[j]->field(i); + bool as_output; if (i == on_field_ix) { ARROW_RETURN_NOT_OK(is_valid_on_field(field)); // Only add on field from the left table - if (j == 0) { - fields.push_back(field); - } + as_output = (j == 0); } else if (std_has(by_field_ix, i)) { ARROW_RETURN_NOT_OK(is_valid_by_field(field)); // Only add by field from the left table - if (j == 0) { - fields.push_back(field); - } + as_output = (j == 0); } else { ARROW_RETURN_NOT_OK(is_valid_data_field(field)); + as_output = true; + } + if (as_output) { fields.push_back(field); } + if (field_output_indices) { + field_output_indices->push_back(as_output ? output_field_idx++ : i); + } } } return std::make_shared(fields); @@ -1041,6 +1226,56 @@ class AsofJoinNode : public ExecNode { return match.indices()[0]; } + static Result GetByKeySize( + const std::vector& input_keys) { + size_t n_by = 0; + for (size_t i = 0; i < input_keys.size(); ++i) { + const auto& by_key = input_keys[i].by_key; + if (i == 0) { + n_by = by_key.size(); + } else if (n_by != by_key.size()) { + return Status::Invalid("inconsistent size of by-key across inputs"); + } + } + return n_by; + } + + static Result> GetIndicesOfOnKey( + const std::vector>& input_schema, + const std::vector& input_keys) { + if (input_schema.size() != input_keys.size()) { + return Status::Invalid("mismatching number of input schema and keys"); + } + size_t n_input = input_schema.size(); + std::vector indices_of_on_key(n_input); + for (size_t i = 0; i < n_input; ++i) { + const auto& on_key = input_keys[i].on_key; + ARROW_ASSIGN_OR_RAISE(indices_of_on_key[i], + FindColIndex(*input_schema[i], on_key, "on")); + } + return indices_of_on_key; + } + + static Result>> GetIndicesOfByKey( + const std::vector>& input_schema, + const std::vector& input_keys) { + if (input_schema.size() != input_keys.size()) { + return Status::Invalid("mismatching number of input schema and keys"); + } + ARROW_ASSIGN_OR_RAISE(size_t n_by, GetByKeySize(input_keys)); + size_t n_input = input_schema.size(); + std::vector> indices_of_by_key( + n_input, std::vector(n_by)); + for (size_t i = 0; i < n_input; ++i) { + for (size_t k = 0; k < n_by; k++) { + const auto& by_key = input_keys[i].by_key; + ARROW_ASSIGN_OR_RAISE(indices_of_by_key[i][k], + FindColIndex(*input_schema[i], by_key[k], "by")); + } + } + return indices_of_by_key; + } + static arrow::Result Make(ExecPlan* plan, std::vector inputs, const ExecNodeOptions& options) { DCHECK_GE(inputs.size(), 2) << "Must have at least two inputs"; @@ -1051,24 +1286,21 @@ class AsofJoinNode : public ExecNode { join_options.tolerance); } - size_t n_input = inputs.size(), n_by = join_options.by_key.size(); + ARROW_ASSIGN_OR_RAISE(size_t n_by, GetByKeySize(join_options.input_keys)); + size_t n_input = inputs.size(); std::vector input_labels(n_input); - std::vector indices_of_on_key(n_input); - std::vector> indices_of_by_key( - n_input, std::vector(n_by)); + std::vector> input_schema(n_input); for (size_t i = 0; i < n_input; ++i) { input_labels[i] = i == 0 ? "left" : "right_" + ToChars(i); - const Schema& input_schema = *inputs[i]->output_schema(); - ARROW_ASSIGN_OR_RAISE(indices_of_on_key[i], - FindColIndex(input_schema, join_options.on_key, "on")); - for (size_t k = 0; k < n_by; k++) { - ARROW_ASSIGN_OR_RAISE(indices_of_by_key[i][k], - FindColIndex(input_schema, join_options.by_key[k], "by")); - } + input_schema[i] = inputs[i]->output_schema(); } - - ARROW_ASSIGN_OR_RAISE(std::shared_ptr output_schema, - MakeOutputSchema(inputs, indices_of_on_key, indices_of_by_key)); + ARROW_ASSIGN_OR_RAISE(std::vector indices_of_on_key, + GetIndicesOfOnKey(input_schema, join_options.input_keys)); + ARROW_ASSIGN_OR_RAISE(std::vector> indices_of_by_key, + GetIndicesOfByKey(input_schema, join_options.input_keys)); + ARROW_ASSIGN_OR_RAISE( + std::shared_ptr output_schema, + MakeOutputSchema(input_schema, indices_of_on_key, indices_of_by_key)); std::vector> key_hashers; for (size_t i = 0; i < n_input; i++) { @@ -1147,6 +1379,8 @@ class AsofJoinNode : public ExecNode { std::mutex gate_; OnType tolerance_; + // Backpressure counter common to all inputs + std::atomic backpressure_counter_; // Queue for triggering processing of a given input // (a false value is a poison pill) ConcurrentQueue process_; @@ -1173,6 +1407,7 @@ AsofJoinNode::AsofJoinNode(ExecPlan* plan, NodeVector inputs, must_hash_(must_hash), may_rehash_(may_rehash), tolerance_(tolerance), + backpressure_counter_(0), process_(), process_thread_(&AsofJoinNode::ProcessThreadWrapper, this) { finished_ = arrow::Future<>::MakeFinished(); @@ -1184,5 +1419,20 @@ void RegisterAsofJoinNode(ExecFactoryRegistry* registry) { } } // namespace internal +namespace asofjoin { + +Result> MakeOutputSchema( + const std::vector>& input_schema, + const std::vector& input_keys, std::vector* field_output_indices) { + ARROW_ASSIGN_OR_RAISE(std::vector indices_of_on_key, + AsofJoinNode::GetIndicesOfOnKey(input_schema, input_keys)); + ARROW_ASSIGN_OR_RAISE(std::vector> indices_of_by_key, + AsofJoinNode::GetIndicesOfByKey(input_schema, input_keys)); + return AsofJoinNode::MakeOutputSchema(input_schema, indices_of_on_key, + indices_of_by_key, field_output_indices); +} + +} // namespace asofjoin + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/exec/asof_join_node.h b/cpp/src/arrow/compute/exec/asof_join_node.h new file mode 100644 index 000000000000..e8a4f62900f2 --- /dev/null +++ b/cpp/src/arrow/compute/exec/asof_join_node.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "arrow/compute/exec.h" +#include "arrow/compute/exec/options.h" +#include "arrow/type.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace compute { +namespace asofjoin { + +using AsofJoinKeys = AsofJoinNodeOptions::Keys; + +/// \brief Make the output schema of an as-of-join node +/// +/// Optionally, also provides the field output indices for this node. +/// \see arrow::engine::RelationInfo +/// +/// \param[in] input_schema the schema of each input to the node +/// \param[in] input_keys the key of each input to the node +/// \param[out] field_output_indices the output index of each field +ARROW_EXPORT Result> MakeOutputSchema( + const std::vector>& input_schema, + const std::vector& input_keys, + std::vector* field_output_indices = NULLPTR); + +} // namespace asofjoin +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/exec/asof_join_node_test.cc b/cpp/src/arrow/compute/exec/asof_join_node_test.cc index 31bc094c52eb..7da23a45d7f3 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node_test.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node_test.cc @@ -142,6 +142,15 @@ void BuildZeroBaseBinaryArray(std::shared_ptr& empty, int64_t length) { ASSERT_OK(builder.Finish(&empty)); } +AsofJoinNodeOptions GetRepeatedOptions(size_t repeat, FieldRef on_key, + std::vector by_key, int64_t tolerance) { + std::vector input_keys(repeat); + for (size_t i = 0; i < repeat; i++) { + input_keys[i] = {on_key, by_key}; + } + return AsofJoinNodeOptions(input_keys, tolerance); +} + // mutates by copying from_key into to_key and changing from_key to zero Result MutateByKey(BatchesWithSchema& batches, std::string from_key, std::string to_key, bool replace_key = false, @@ -260,7 +269,7 @@ void CheckRunOutput(const BatchesWithSchema& l_batches, const BatchesWithSchema& r1_batches, const BatchesWithSchema& exp_batches, \ const FieldRef time, by_key_type key, const int64_t tolerance) { \ CheckRunOutput(l_batches, r0_batches, r1_batches, exp_batches, \ - AsofJoinNodeOptions(time, {key}, tolerance)); \ + GetRepeatedOptions(3, time, {key}, tolerance)); \ } EXPAND_BY_KEY_TYPE(CHECK_RUN_OUTPUT) @@ -313,7 +322,7 @@ void DoRunInvalidPlanTest(const std::shared_ptr& l_schema, const std::shared_ptr& r_schema, int64_t tolerance, const std::string& expected_error_str) { DoRunInvalidPlanTest(l_schema, r_schema, - AsofJoinNodeOptions("time", {"key"}, tolerance), + GetRepeatedOptions(2, "time", {"key"}, tolerance), expected_error_str); } @@ -336,27 +345,27 @@ void DoRunMissingKeysTest(const std::shared_ptr& l_schema, void DoRunMissingOnKeyTest(const std::shared_ptr& l_schema, const std::shared_ptr& r_schema) { DoRunInvalidPlanTest(l_schema, r_schema, - AsofJoinNodeOptions("invalid_time", {"key"}, 0), + GetRepeatedOptions(2, "invalid_time", {"key"}, 0), "Bad join key on table : No match"); } void DoRunMissingByKeyTest(const std::shared_ptr& l_schema, const std::shared_ptr& r_schema) { DoRunInvalidPlanTest(l_schema, r_schema, - AsofJoinNodeOptions("time", {"invalid_key"}, 0), + GetRepeatedOptions(2, "time", {"invalid_key"}, 0), "Bad join key on table : No match"); } void DoRunNestedOnKeyTest(const std::shared_ptr& l_schema, const std::shared_ptr& r_schema) { - DoRunInvalidPlanTest(l_schema, r_schema, AsofJoinNodeOptions({0, "time"}, {"key"}, 0), + DoRunInvalidPlanTest(l_schema, r_schema, GetRepeatedOptions(2, {0, "time"}, {"key"}, 0), "Bad join key on table : No match"); } void DoRunNestedByKeyTest(const std::shared_ptr& l_schema, const std::shared_ptr& r_schema) { DoRunInvalidPlanTest(l_schema, r_schema, - AsofJoinNodeOptions("time", {FieldRef{0, 1}}, 0), + GetRepeatedOptions(2, "time", {FieldRef{0, 1}}, 0), "Bad join key on table : No match"); } @@ -417,7 +426,7 @@ void DoRunUnorderedPlanTest(bool l_unordered, bool r_unordered, const std::shared_ptr& l_schema, const std::shared_ptr& r_schema) { DoRunUnorderedPlanTest(l_unordered, r_unordered, l_schema, r_schema, - AsofJoinNodeOptions("time", {"key"}, 1000), + GetRepeatedOptions(2, "time", {"key"}, 1000), "out-of-order on-key values"); } @@ -514,7 +523,7 @@ struct BasicTest { ASSERT_OK_AND_ASSIGN(exp_nokey_batches, MutateByKey(exp_nokey_batches, "key", "key2", true, true)); CheckRunOutput(l_batches, r0_batches, r1_batches, exp_nokey_batches, - AsofJoinNodeOptions("time", {"key2"}, tolerance)); + GetRepeatedOptions(3, "time", {"key2"}, tolerance)); }); } static void DoMutateNullKey(BasicTest& basic_tests) { basic_tests.RunMutateNullKey(); } @@ -527,7 +536,7 @@ struct BasicTest { ASSERT_OK_AND_ASSIGN(r1_batches, MutateByKey(r1_batches, "key", "key", false, false, true)); CheckRunOutput(l_batches, r0_batches, r1_batches, exp_emptykey_batches, - AsofJoinNodeOptions("time", {}, tolerance)); + GetRepeatedOptions(3, "time", {}, tolerance)); }); } static void DoMutateEmptyKey(BasicTest& basic_tests) { diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index 8600b1134899..58ae80324722 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -445,22 +445,35 @@ class ARROW_EXPORT HashJoinNodeOptions : public ExecNodeOptions { /// This node will output one row for each row in the left table. class ARROW_EXPORT AsofJoinNodeOptions : public ExecNodeOptions { public: - AsofJoinNodeOptions(FieldRef on_key, std::vector by_key, int64_t tolerance) - : on_key(std::move(on_key)), by_key(by_key), tolerance(tolerance) {} - - /// \brief "on" key for the join. + /// \brief Keys for one input table of the AsofJoin operation /// - /// All inputs tables must be sorted by the "on" key. Must be a single field of a common - /// type. Inexact match is used on the "on" key. i.e., a row is considered match iff - /// left_on - tolerance <= right_on <= left_on. - /// Currently, the "on" key must be of an integer, date, or timestamp type. - FieldRef on_key; - /// \brief "by" key for the join. + /// The keys must be consistent across the input tables: + /// Each "on" key must refer to a field of the same type and units across the tables. + /// Each "by" key must refer to a list of fields of the same types across the tables. + struct Keys { + /// \brief "on" key for the join. + /// + /// The input table must be sorted by the "on" key. Must be a single field of a common + /// type. Inexact match is used on the "on" key. i.e., a row is considered match iff + /// left_on - tolerance <= right_on <= left_on. + /// Currently, the "on" key must be of an integer, date, or timestamp type. + FieldRef on_key; + /// \brief "by" key for the join. + /// + /// The input table must have each field of the "by" key. Exact equality is used for + /// each field of the "by" key. + /// Currently, each field of the "by" key must be of an integer, date, timestamp, or + /// base-binary type. + std::vector by_key; + }; + + AsofJoinNodeOptions(std::vector input_keys, int64_t tolerance) + : input_keys(std::move(input_keys)), tolerance(tolerance) {} + + /// \brief AsofJoin keys per input table. /// - /// All input tables must have the "by" key. Exact equality - /// is used for the "by" key. - /// Currently, the "by" key must be of an integer, date, timestamp, or base-binary type - std::vector by_key; + /// See `Keys` for details. + std::vector input_keys; /// \brief Tolerance for inexact "on" key matching. Must be non-negative. /// /// The tolerance is interpreted in the same units as the "on" key. diff --git a/cpp/src/arrow/engine/CMakeLists.txt b/cpp/src/arrow/engine/CMakeLists.txt index 09dff51f5fe5..57db71027131 100644 --- a/cpp/src/arrow/engine/CMakeLists.txt +++ b/cpp/src/arrow/engine/CMakeLists.txt @@ -23,6 +23,7 @@ set(ARROW_SUBSTRAIT_SRCS substrait/expression_internal.cc substrait/extension_set.cc substrait/extension_types.cc + substrait/options.cc substrait/plan_internal.cc substrait/relation_internal.cc substrait/serde.cc diff --git a/cpp/src/arrow/engine/substrait/extension_set.h b/cpp/src/arrow/engine/substrait/extension_set.h index 12aa40115b15..cb6c0ad5d420 100644 --- a/cpp/src/arrow/engine/substrait/extension_set.h +++ b/cpp/src/arrow/engine/substrait/extension_set.h @@ -66,12 +66,12 @@ constexpr const char* kSubstraitAggregateGenericFunctionsUri = "https://github.com/substrait-io/substrait/blob/main/extensions/" "functions_aggregate_generic.yaml"; -struct Id { +struct ARROW_ENGINE_EXPORT Id { std::string_view uri, name; bool empty() const { return uri.empty() && name.empty(); } std::string ToString() const; }; -struct IdHashEq { +struct ARROW_ENGINE_EXPORT IdHashEq { size_t operator()(Id id) const; bool operator()(Id l, Id r) const; }; @@ -83,7 +83,7 @@ struct IdHashEq { /// convert a plan from Substrait to Arrow we need to copy these strings out of /// the Substrait buffer and into owned storage. This class serves as that owned /// storage. -class IdStorage { +class ARROW_ENGINE_EXPORT IdStorage { public: virtual ~IdStorage() = default; /// \brief Get an equivalent id pointing into this storage @@ -111,7 +111,7 @@ class IdStorage { /// Substrait call expressions contain a list of arguments which can either /// be enum arguments (which are serialized as strings), value arguments (which) /// are Arrow expressions, or type arguments (not yet implemented) -class SubstraitCall { +class ARROW_ENGINE_EXPORT SubstraitCall { public: SubstraitCall(Id id, std::shared_ptr output_type, bool output_nullable, bool is_hash = false) diff --git a/cpp/src/arrow/engine/substrait/options.cc b/cpp/src/arrow/engine/substrait/options.cc new file mode 100644 index 000000000000..48fb6c53b614 --- /dev/null +++ b/cpp/src/arrow/engine/substrait/options.cc @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#include + +#include "arrow/engine/substrait/options.h" + +#include +#include "arrow/compute/exec/asof_join_node.h" +#include "arrow/compute/exec/options.h" +#include "arrow/engine/substrait/expression_internal.h" +#include "arrow/engine/substrait/relation_internal.h" +#include "substrait/extension_rels.pb.h" + +namespace arrow { +namespace engine { + +class DefaultExtensionProvider : public ExtensionProvider { + public: + Result MakeRel(std::vector inputs, + const google::protobuf::Any& rel, + const ExtensionSet& ext_set) override { + if (rel.Is()) { + substrait_ext::AsOfJoinRel as_of_join_rel; + rel.UnpackTo(&as_of_join_rel); + return MakeAsOfJoinRel(std::move(inputs), as_of_join_rel, ext_set); + } + return Status::NotImplemented("Unrecognized extension in Susbstrait plan: ", + rel.DebugString()); + } + + private: + Result MakeAsOfJoinRel(std::vector inputs, + const substrait_ext::AsOfJoinRel& as_of_join_rel, + const ExtensionSet& ext_set) { + if (inputs.size() < 2) { + return Status::Invalid("substrait::AsOfJoinNode too few input tables: ", + inputs.size()); + } + if (static_cast(as_of_join_rel.keys_size()) != inputs.size()) { + return Status::Invalid("substrait::AsOfJoinNode mismatched number of inputs"); + } + + size_t n_input = inputs.size(), i = 0; + std::vector input_keys(n_input); + for (const auto& keys : as_of_join_rel.keys()) { + // on-key + if (!keys.has_on()) { + return Status::Invalid("substrait::AsOfJoinNode missing on-key for input ", i); + } + ARROW_ASSIGN_OR_RAISE(auto on_key_expr, FromProto(keys.on(), ext_set, {})); + if (on_key_expr.field_ref() == NULLPTR) { + return Status::NotImplemented( + "substrait::AsOfJoinNode non-field-ref on-key for input ", i); + } + const FieldRef& on_key = *on_key_expr.field_ref(); + + // by-key + std::vector by_key; + for (const auto& by_item : keys.by()) { + ARROW_ASSIGN_OR_RAISE(auto by_key_expr, FromProto(by_item, ext_set, {})); + if (by_key_expr.field_ref() == NULLPTR) { + return Status::NotImplemented( + "substrait::AsOfJoinNode non-field-ref by-key for input ", i); + } + by_key.push_back(*by_key_expr.field_ref()); + } + + input_keys[i] = {std::move(on_key), std::move(by_key)}; + ++i; + } + + // schema + int64_t tolerance = as_of_join_rel.tolerance(); + std::vector> input_schema(inputs.size()); + for (size_t i = 0; i < inputs.size(); i++) { + input_schema[i] = inputs[i].output_schema; + } + std::vector field_output_indices; + ARROW_ASSIGN_OR_RAISE(auto schema, + compute::asofjoin::MakeOutputSchema(input_schema, input_keys, + &field_output_indices)); + compute::AsofJoinNodeOptions asofjoin_node_opts{std::move(input_keys), tolerance}; + + // declaration + std::vector input_decls(inputs.size()); + for (size_t i = 0; i < inputs.size(); i++) { + input_decls[i] = inputs[i].declaration; + } + return RelationInfo{ + {compute::Declaration("asofjoin", input_decls, std::move(asofjoin_node_opts)), + std::move(schema)}, + std::move(inputs), + std::move(field_output_indices)}; + } +}; + +std::shared_ptr ExtensionProvider::kDefaultExtensionProvider = + std::make_shared(); + +} // namespace engine +} // namespace arrow diff --git a/cpp/src/arrow/engine/substrait/options.h b/cpp/src/arrow/engine/substrait/options.h index 014842f4d8f1..8d37639ca3b6 100644 --- a/cpp/src/arrow/engine/substrait/options.h +++ b/cpp/src/arrow/engine/substrait/options.h @@ -23,7 +23,11 @@ #include #include +#include + #include "arrow/compute/type_fwd.h" +#include "arrow/engine/substrait/type_fwd.h" +#include "arrow/engine/substrait/visibility.h" #include "arrow/type_fwd.h" namespace arrow { @@ -65,9 +69,18 @@ using NamedTableProvider = std::function(const std::vector&)>; static NamedTableProvider kDefaultNamedTableProvider; +class ARROW_ENGINE_EXPORT ExtensionProvider { + public: + static std::shared_ptr kDefaultExtensionProvider; + virtual ~ExtensionProvider() = default; + virtual Result MakeRel(std::vector inputs, + const google::protobuf::Any& rel, + const ExtensionSet& ext_set) = 0; +}; + /// Options that control the conversion between Substrait and Acero representations of a /// plan. -struct ConversionOptions { +struct ARROW_ENGINE_EXPORT ConversionOptions { /// \brief How strictly the converter should adhere to the structure of the input. ConversionStrictness strictness = ConversionStrictness::BEST_EFFORT; /// \brief A custom strategy to be used for providing named tables @@ -75,6 +88,8 @@ struct ConversionOptions { /// The default behavior will return an invalid status if the plan has any /// named table relations. NamedTableProvider named_table_provider = kDefaultNamedTableProvider; + std::shared_ptr extension_provider = + ExtensionProvider::kDefaultExtensionProvider; }; } // namespace engine diff --git a/cpp/src/arrow/engine/substrait/relation.h b/cpp/src/arrow/engine/substrait/relation.h new file mode 100644 index 000000000000..e240d23ae318 --- /dev/null +++ b/cpp/src/arrow/engine/substrait/relation.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/compute/exec/exec_plan.h" +#include "arrow/type_fwd.h" + +namespace arrow { +namespace engine { + +/// Execution information resulting from converting a Substrait relation. +struct ARROW_ENGINE_EXPORT DeclarationInfo { + /// The compute declaration produced thus far. + compute::Declaration declaration; + + std::shared_ptr output_schema; +}; + +/// Information resulting from converting a Substrait relation. +struct ARROW_ENGINE_EXPORT RelationInfo { + /// The execution information produced thus far. + DeclarationInfo decl_info; + /// The execution information of the inputs to the relation. + std::vector inputs; + /// A vector of indices, one per input field per input in order, each index referring + /// to the corresponding field within the output schema, if it is in the output, or -1 + /// otherwise. Each location in this vector is a field input index. This vector is + /// useful for translating selected field input indices (often from an output mapping in + /// a Substrait plan) of a join-type relation to their locations in the output schema of + /// the relation. This vector is undefined if the translation is unsupported. + std::optional> field_output_indices; +}; + +} // namespace engine +} // namespace arrow diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index 019e402a44c6..84dee8733251 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -49,10 +49,9 @@ struct EmitInfo { std::shared_ptr schema; }; -template -Result GetEmitInfo(const RelMessage& rel, +Result GetEmitInfo(const substrait::RelCommon& rel_common, const std::shared_ptr& input_schema) { - const auto& emit = rel.common().emit(); + const auto& emit = rel_common.emit(); int emit_size = emit.output_mapping_size(); std::vector proj_field_refs(emit_size); EmitInfo emit_info; @@ -67,16 +66,15 @@ Result GetEmitInfo(const RelMessage& rel, return std::move(emit_info); } -template -Result ProcessEmit(const RelMessage& rel, +Result ProcessEmit(std::optional rel_common_opt, const DeclarationInfo& no_emit_declr, const std::shared_ptr& schema) { - if (rel.has_common()) { - switch (rel.common().emit_kind_case()) { + if (rel_common_opt) { + switch (rel_common_opt->emit_kind_case()) { case substrait::RelCommon::EmitKindCase::kDirect: return no_emit_declr; case substrait::RelCommon::EmitKindCase::kEmit: { - ARROW_ASSIGN_OR_RAISE(auto emit_info, GetEmitInfo(rel, schema)); + ARROW_ASSIGN_OR_RAISE(auto emit_info, GetEmitInfo(*rel_common_opt, schema)); return DeclarationInfo{ compute::Declaration::Sequence( {no_emit_declr.declaration, @@ -92,6 +90,102 @@ Result ProcessEmit(const RelMessage& rel, } } +template +Result ProcessEmit(const RelMessage& rel, + const DeclarationInfo& no_emit_declr, + const std::shared_ptr& schema) { + return ProcessEmit(rel.has_common() ? std::optional(rel.common()) : std::nullopt, + no_emit_declr, schema); +} + +Result ProcessExtensionEmit( + const DeclarationInfo& no_emit_declr, const std::vector& emit_info, + const std::vector& field_output_indices) { + const std::shared_ptr& input_schema = no_emit_declr.output_schema; + std::vector proj_field_refs; + proj_field_refs.reserve(emit_info.size()); + FieldVector emit_fields; + emit_fields.reserve(emit_info.size()); + + for (int emit_idx : emit_info) { + if (emit_idx < 0 || static_cast(emit_idx) >= field_output_indices.size()) { + return Status::Invalid("Out of bounds emit index ", emit_idx); + } + int field_idx = field_output_indices[emit_idx]; + if (field_idx < 0) { + return Status::Invalid("Non-output emit index ", emit_idx); + } + proj_field_refs.push_back(compute::field_ref(FieldRef(field_idx))); + emit_fields.push_back(input_schema->field(field_idx)); + } + + std::shared_ptr emit_schema = schema(std::move(emit_fields)); + + return DeclarationInfo{ + compute::Declaration::Sequence( + {no_emit_declr.declaration, + {"project", compute::ProjectNodeOptions{std::move(proj_field_refs)}}}), + std::move(emit_schema)}; +} + +Result GetExtensionRelationInfo( + const substrait::Rel& rel, const ExtensionSet& ext_set, + const ConversionOptions& conversion_options) { + switch (rel.rel_type_case()) { + case substrait::Rel::RelTypeCase::kExtensionLeaf: { + const auto& ext = rel.extension_leaf(); + return conversion_options.extension_provider->MakeRel({}, ext.detail(), ext_set); + } + + case substrait::Rel::RelTypeCase::kExtensionSingle: { + const auto& ext = rel.extension_single(); + ARROW_ASSIGN_OR_RAISE(DeclarationInfo input, + FromProto(ext.input(), ext_set, conversion_options)); + return conversion_options.extension_provider->MakeRel({input}, ext.detail(), + ext_set); + } + + case substrait::Rel::RelTypeCase::kExtensionMulti: { + const auto& ext = rel.extension_multi(); + std::vector inputs; + for (const auto& input : ext.inputs()) { + ARROW_ASSIGN_OR_RAISE(auto input_info, + FromProto(input, ext_set, conversion_options)); + inputs.push_back(std::move(input_info)); + } + return conversion_options.extension_provider->MakeRel(std::move(inputs), + ext.detail(), ext_set); + } + + default: { + return Status::Invalid("Invalid extension relation case ", rel.rel_type_case()); + } + } +} + +std::optional GetExtensionRelCommon(const substrait::Rel& rel) { + switch (rel.rel_type_case()) { + case substrait::Rel::RelTypeCase::kExtensionLeaf: { + const auto& ext = rel.extension_leaf(); + return ext.has_common() ? std::optional(ext.common()) : std::nullopt; + } + + case substrait::Rel::RelTypeCase::kExtensionSingle: { + const auto& ext = rel.extension_single(); + return ext.has_common() ? std::optional(ext.common()) : std::nullopt; + } + + case substrait::Rel::RelTypeCase::kExtensionMulti: { + const auto& ext = rel.extension_multi(); + return ext.has_common() ? std::optional(ext.common()) : std::nullopt; + } + + default: { + return std::nullopt; + } + } +} + template Status CheckRelCommon(const RelMessage& rel, const ConversionOptions& conversion_options) { @@ -619,6 +713,44 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& std::move(aggregate_schema)); } + case substrait::Rel::RelTypeCase::kExtensionLeaf: + case substrait::Rel::RelTypeCase::kExtensionSingle: + case substrait::Rel::RelTypeCase::kExtensionMulti: { + ARROW_ASSIGN_OR_RAISE(auto ext_rel_info, + GetExtensionRelationInfo(rel, ext_set, conversion_options)); + const auto& ext_decl_info = ext_rel_info.decl_info; + auto ext_common_opt = GetExtensionRelCommon(rel); + bool has_emit = ext_common_opt && ext_common_opt->emit_kind_case() == + substrait::RelCommon::EmitKindCase::kEmit; + if (!ext_rel_info.field_output_indices) { + if (!has_emit) { + return ProcessEmit(ext_common_opt, ext_decl_info, ext_decl_info.output_schema); + } + return Status::NotImplemented("Emit not supported by ", + ext_decl_info.declaration.factory_name); + } + std::vector emit_order; + if (has_emit) { + const auto& emit_info = ext_common_opt->emit(); + emit_order.reserve(emit_info.output_mapping_size()); + for (const auto& emit_idx : emit_info.output_mapping()) { + emit_order.push_back(emit_idx); + } + } else { + // default output mapping + int emit_size = 0; + for (const auto& input : ext_rel_info.inputs) { + emit_size += input.output_schema->num_fields(); + } + emit_order.reserve(emit_size); + for (int emit_idx = 0; emit_idx < emit_size; emit_idx++) { + emit_order.push_back(emit_idx); + } + } + return ProcessExtensionEmit(std::move(ext_decl_info), emit_order, + *ext_rel_info.field_output_indices); + } + default: break; } diff --git a/cpp/src/arrow/engine/substrait/relation_internal.h b/cpp/src/arrow/engine/substrait/relation_internal.h index 514f3f97fc05..df7804b67961 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.h +++ b/cpp/src/arrow/engine/substrait/relation_internal.h @@ -22,6 +22,7 @@ #include "arrow/compute/exec/exec_plan.h" #include "arrow/engine/substrait/extension_types.h" #include "arrow/engine/substrait/options.h" +#include "arrow/engine/substrait/relation.h" #include "arrow/engine/substrait/serde.h" #include "arrow/engine/substrait/visibility.h" #include "arrow/type_fwd.h" @@ -31,14 +32,6 @@ namespace arrow { namespace engine { -/// Information resulting from converting a Substrait relation. -struct DeclarationInfo { - /// The compute declaration produced thus far. - compute::Declaration declaration; - - std::shared_ptr output_schema; -}; - /// \brief Convert a Substrait Rel object to an Acero declaration ARROW_ENGINE_EXPORT Result FromProto(const substrait::Rel&, const ExtensionSet&, diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index 210edcf75e71..eb7306aeb51a 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -20,8 +20,10 @@ #include #include +#include "arrow/compute/exec/asof_join_node.h" #include "arrow/compute/exec/exec_plan.h" #include "arrow/compute/exec/expression_internal.h" +#include "arrow/compute/exec/test_util.h" #include "arrow/dataset/file_base.h" #include "arrow/dataset/file_ipc.h" #include "arrow/dataset/file_parquet.h" @@ -2260,6 +2262,16 @@ TEST(SubstraitRoundTrip, BasicPlanEndToEnd) { EXPECT_TRUE(expected_table->Equals(*rnd_trp_table)); } +NamedTableProvider ProvideMadeTable( + std::function>(const std::vector&)> make) { + return [make](const std::vector& names) -> Result { + ARROW_ASSIGN_OR_RAISE(auto table, make(names)); + std::shared_ptr options = + std::make_shared(table); + return compute::Declaration("table_source", {}, options, "mock_source"); + }; +} + TEST(SubstraitRoundTrip, ProjectRel) { compute::ExecContext exec_context; auto dummy_schema = @@ -3726,5 +3738,378 @@ TEST(Substrait, ReadRelWithGlobFiles) { buf, {}, {}, &options); } +TEST(Substrait, PlanWithExtension) { + // This demos an extension relation + std::string substrait_json = R"({ + "extensionUris": [], + "extensions": [], + "relations": [{ + "root": { + "input": { + "extension_multi": { + "common": { + "emit": { + "outputMapping": [0, 1, 2, 5] + } + }, + "inputs": [ + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value1"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T1"] + } + } + }, + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value2"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T2"] + } + } + } + ], + "detail": { + "@type": "/arrow.substrait_ext.AsOfJoinRel", + "keys" : [ + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + }, + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + } + ], + "tolerance": 1000 + } + } + }, + "names": ["time", "key", "value1", "value2"] + } + }], + "expectedTypeUrls": [] + })"; + + std::vector> input_schema = { + schema({field("time", int32()), field("key", int32()), field("value1", float64())}), + schema( + {field("time", int32()), field("key", int32()), field("value2", float64())})}; + NamedTableProvider table_provider = ProvideMadeTable( + [&input_schema]( + const std::vector& names) -> Result> { + if (names.size() != 1) { + return Status::Invalid("Multiple test table names"); + } + if (names[0] == "T1") { + return TableFromJSON(input_schema[0], + {"[[2, 1, 1.1], [4, 1, 2.1], [6, 2, 3.1]]"}); + } + if (names[0] == "T2") { + return TableFromJSON(input_schema[1], + {"[[1, 1, 1.2], [3, 2, 2.2], [5, 2, 3.2]]"}); + } + return Status::Invalid("Unknown test table name ", names[0]); + }); + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + ASSERT_OK_AND_ASSIGN(auto buf, internal::SubstraitFromJSON("Plan", substrait_json)); + + ASSERT_OK_AND_ASSIGN( + auto out_schema, + compute::asofjoin::MakeOutputSchema( + input_schema, {{FieldRef(0), {FieldRef(1)}}, {FieldRef(0), {FieldRef(1)}}})); + auto expected_table = TableFromJSON( + out_schema, {"[[2, 1, 1.1, 1.2], [4, 1, 2.1, 1.2], [6, 2, 3.1, 3.2]]"}); + CheckRoundTripResult(std::move(out_schema), std::move(expected_table), + *compute::default_exec_context(), buf, {}, conversion_options); +} + +TEST(Substrait, AsOfJoinDefaultEmit) { + std::string substrait_json = R"({ + "extensionUris": [], + "extensions": [], + "relations": [{ + "root": { + "input": { + "extension_multi": { + "inputs": [ + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value1"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T1"] + } + } + }, + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value2"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T2"] + } + } + } + ], + "detail": { + "@type": "/arrow.substrait_ext.AsOfJoinRel", + "keys" : [ + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + }, + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + } + ], + "tolerance": 1000 + } + } + }, + "names": ["time", "key", "value1", "time2", "key2", "value2"] + } + }], + "expectedTypeUrls": [] + })"; + + std::vector> input_schema = { + schema({field("time", int32()), field("key", int32()), field("value1", float64())}), + schema( + {field("time", int32()), field("key", int32()), field("value2", float64())})}; + NamedTableProvider table_provider = ProvideMadeTable( + [&input_schema]( + const std::vector& names) -> Result> { + if (names.size() != 1) { + return Status::Invalid("Multiple test table names"); + } + if (names[0] == "T1") { + return TableFromJSON(input_schema[0], + {"[[2, 1, 1.1], [4, 1, 2.1], [6, 2, 3.1]]"}); + } + if (names[0] == "T2") { + return TableFromJSON(input_schema[1], + {"[[1, 1, 1.2], [3, 2, 2.2], [5, 2, 3.2]]"}); + } + return Status::Invalid("Unknown test table name ", names[0]); + }); + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + ASSERT_OK_AND_ASSIGN(auto buf, internal::SubstraitFromJSON("Plan", substrait_json)); + + auto out_schema = schema({field("time", int32()), field("key", int32()), + field("value1", float64()), field("time2", int32()), + field("key2", int32()), field("value2", float64())}); + + auto expected_table = TableFromJSON( + out_schema, + {"[[2, 1, 1.1, 2, 1, 1.2], [4, 1, 2.1, 4, 1, 1.2], [6, 2, 3.1, 6, 2, 3.2]]"}); + CheckRoundTripResult(std::move(out_schema), std::move(expected_table), + *compute::default_exec_context(), buf, {}, conversion_options); +} + } // namespace engine } // namespace arrow diff --git a/cpp/src/arrow/engine/substrait/type_fwd.h b/cpp/src/arrow/engine/substrait/type_fwd.h index 235d9e82d1b8..bbeb4ef6897a 100644 --- a/cpp/src/arrow/engine/substrait/type_fwd.h +++ b/cpp/src/arrow/engine/substrait/type_fwd.h @@ -26,6 +26,8 @@ class ExtensionIdRegistry; class ExtensionSet; struct ConversionOptions; +struct DeclarationInfo; +struct RelationInfo; } // namespace engine } // namespace arrow From 27f0e1e4676090a7b651cef3f29414577c0a93e2 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Thu, 1 Dec 2022 05:01:59 -0500 Subject: [PATCH 2/9] fix asof_join_benchmark.cc --- cpp/src/arrow/compute/exec/asof_join_benchmark.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/exec/asof_join_benchmark.cc b/cpp/src/arrow/compute/exec/asof_join_benchmark.cc index a0362eb1ba8a..55b76c6e8ca3 100644 --- a/cpp/src/arrow/compute/exec/asof_join_benchmark.cc +++ b/cpp/src/arrow/compute/exec/asof_join_benchmark.cc @@ -107,9 +107,19 @@ static void TableJoinOverhead(benchmark::State& state, benchmark::Counter(static_cast(ctx.memory_pool()->max_memory())); } +AsofJoinNodeOptions GetRepeatedOptions(size_t repeat, FieldRef on_key, + std::vector by_key, int64_t tolerance) { + std::vector input_keys(repeat); + for (size_t i = 0; i < repeat; i++) { + input_keys[i] = {on_key, by_key}; + } + return AsofJoinNodeOptions(input_keys, tolerance); +} + static void AsOfJoinOverhead(benchmark::State& state) { int64_t tolerance = 0; - AsofJoinNodeOptions options = AsofJoinNodeOptions(kTimeCol, {kKeyCol}, tolerance); + AsofJoinNodeOptions options = + GetRepeatedOptions(int(state.range(4)), kTimeCol, {kKeyCol}, tolerance); TableJoinOverhead( state, TableGenerationProperties{int(state.range(0)), int(state.range(1)), From 5e9a61a36217b0c6963a2934e3de0ae81a071aa9 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Tue, 6 Dec 2022 07:13:03 -0500 Subject: [PATCH 3/9] hide protobuf dependency --- cpp/src/arrow/engine/substrait/options.cc | 18 ++++++++-- cpp/src/arrow/engine/substrait/options.h | 7 +++- .../arrow/engine/substrait/options_internal.h | 35 +++++++++++++++++++ .../engine/substrait/relation_internal.cc | 13 ++++--- 4 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 cpp/src/arrow/engine/substrait/options_internal.h diff --git a/cpp/src/arrow/engine/substrait/options.cc b/cpp/src/arrow/engine/substrait/options.cc index 48fb6c53b614..93c8e11a7502 100644 --- a/cpp/src/arrow/engine/substrait/options.cc +++ b/cpp/src/arrow/engine/substrait/options.cc @@ -14,7 +14,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include #include "arrow/engine/substrait/options.h" @@ -22,13 +21,28 @@ #include "arrow/compute/exec/asof_join_node.h" #include "arrow/compute/exec/options.h" #include "arrow/engine/substrait/expression_internal.h" +#include "arrow/engine/substrait/options_internal.h" #include "arrow/engine/substrait/relation_internal.h" #include "substrait/extension_rels.pb.h" namespace arrow { namespace engine { -class DefaultExtensionProvider : public ExtensionProvider { +class BaseExtensionProvider : public ExtensionProvider { + public: + Result MakeRel(std::vector inputs, + const ExtensionDetails& ext_details, + const ExtensionSet& ext_set) override { + auto details = dynamic_cast(ext_details); + return MakeRel(inputs, details.rel, ext_set); + } + + virtual Result MakeRel(std::vector inputs, + const google::protobuf::Any& rel, + const ExtensionSet& ext_set) = 0; +}; + +class DefaultExtensionProvider : public BaseExtensionProvider { public: Result MakeRel(std::vector inputs, const google::protobuf::Any& rel, diff --git a/cpp/src/arrow/engine/substrait/options.h b/cpp/src/arrow/engine/substrait/options.h index 8d37639ca3b6..988a8302e4d9 100644 --- a/cpp/src/arrow/engine/substrait/options.h +++ b/cpp/src/arrow/engine/substrait/options.h @@ -69,12 +69,17 @@ using NamedTableProvider = std::function(const std::vector&)>; static NamedTableProvider kDefaultNamedTableProvider; +class ARROW_ENGINE_EXPORT ExtensionDetails { + public: + virtual ~ExtensionDetails() = default; +}; + class ARROW_ENGINE_EXPORT ExtensionProvider { public: static std::shared_ptr kDefaultExtensionProvider; virtual ~ExtensionProvider() = default; virtual Result MakeRel(std::vector inputs, - const google::protobuf::Any& rel, + const ExtensionDetails& ext_details, const ExtensionSet& ext_set) = 0; }; diff --git a/cpp/src/arrow/engine/substrait/options_internal.h b/cpp/src/arrow/engine/substrait/options_internal.h new file mode 100644 index 000000000000..9983efbc91d4 --- /dev/null +++ b/cpp/src/arrow/engine/substrait/options_internal.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This API is EXPERIMENTAL. + +#pragma once + +#include "arrow/engine/substrait/options.h" + +namespace arrow { +namespace engine { + +class DefaultExtensionDetails : public ExtensionDetails { + public: + explicit DefaultExtensionDetails(google::protobuf::Any rel) : rel(rel) {} + + google::protobuf::Any rel; +}; + +} // namespace engine +} // namespace arrow diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index 84dee8733251..7058b729f9e1 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -25,6 +25,7 @@ #include "arrow/dataset/plan.h" #include "arrow/dataset/scanner.h" #include "arrow/engine/substrait/expression_internal.h" +#include "arrow/engine/substrait/options_internal.h" #include "arrow/engine/substrait/type_internal.h" #include "arrow/filesystem/localfs.h" #include "arrow/filesystem/path_util.h" @@ -134,15 +135,16 @@ Result GetExtensionRelationInfo( switch (rel.rel_type_case()) { case substrait::Rel::RelTypeCase::kExtensionLeaf: { const auto& ext = rel.extension_leaf(); - return conversion_options.extension_provider->MakeRel({}, ext.detail(), ext_set); + DefaultExtensionDetails detail{ext.detail()}; + return conversion_options.extension_provider->MakeRel({}, detail, ext_set); } case substrait::Rel::RelTypeCase::kExtensionSingle: { const auto& ext = rel.extension_single(); ARROW_ASSIGN_OR_RAISE(DeclarationInfo input, FromProto(ext.input(), ext_set, conversion_options)); - return conversion_options.extension_provider->MakeRel({input}, ext.detail(), - ext_set); + DefaultExtensionDetails detail{ext.detail()}; + return conversion_options.extension_provider->MakeRel({input}, detail, ext_set); } case substrait::Rel::RelTypeCase::kExtensionMulti: { @@ -153,8 +155,9 @@ Result GetExtensionRelationInfo( FromProto(input, ext_set, conversion_options)); inputs.push_back(std::move(input_info)); } - return conversion_options.extension_provider->MakeRel(std::move(inputs), - ext.detail(), ext_set); + DefaultExtensionDetails detail{ext.detail()}; + return conversion_options.extension_provider->MakeRel(std::move(inputs), detail, + ext_set); } default: { From fbe34360ebde144a1ec907f7a08da0e7b2e41277 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Sat, 17 Dec 2022 09:36:11 -0500 Subject: [PATCH 4/9] requested changes --- cpp/src/arrow/compute/exec.cc | 2 +- cpp/src/arrow/compute/exec/asof_join_node.cc | 24 ++++++++- cpp/src/arrow/engine/substrait/options.cc | 11 ++-- cpp/src/arrow/engine/substrait/options.h | 2 +- cpp/src/arrow/engine/substrait/relation.h | 5 +- .../engine/substrait/relation_internal.cc | 52 ++++++++++++------- cpp/src/arrow/engine/substrait/serde_test.cc | 10 ++-- 7 files changed, 69 insertions(+), 37 deletions(-) diff --git a/cpp/src/arrow/compute/exec.cc b/cpp/src/arrow/compute/exec.cc index 8d3dcf0f2cd7..884cadc3461b 100644 --- a/cpp/src/arrow/compute/exec.cc +++ b/cpp/src/arrow/compute/exec.cc @@ -172,7 +172,7 @@ Result ExecBatch::Make(std::vector values) { Result> ExecBatch::ToRecordBatch( std::shared_ptr schema, MemoryPool* pool) const { if (static_cast(schema->num_fields()) > values.size()) { - return Status::Invalid("ExecBatch::ToTRecordBatch mismatching schema size"); + return Status::Invalid("ExecBatch::ToRecordBatch mismatching schema size"); } ArrayVector columns(schema->num_fields()); diff --git a/cpp/src/arrow/compute/exec/asof_join_node.cc b/cpp/src/arrow/compute/exec/asof_join_node.cc index c259f6a2455d..2155e63611c9 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node.cc @@ -59,6 +59,12 @@ inline bool std_has(const T& container, const V& val) { return container.end() != std_find(container, val); } +template +inline D std_index(const T& container, const V& val) { + return std_find(container, val) - container.begin(); +} + typedef uint64_t ByType; typedef uint64_t OnType; typedef uint64_t HashType; @@ -1142,6 +1148,15 @@ class AsofJoinNode : public ExecNode { } } + /// \brief Make the output schema of an as-of-join node + /// + /// Optionally, also provides the field output indices for this node. + /// \see arrow::engine::RelationInfo + /// + /// \param[in] input_schema the schema of each input to the node + /// \param[in] indices_of_on_key the on-key index of each input to the node + /// \param[in] indices_of_by_key the by-key indices of each input to the node + /// \param[out] field_output_indices the output index of each field static arrow::Result> MakeOutputSchema( const std::vector> input_schema, const std::vector& indices_of_on_key, @@ -1187,24 +1202,29 @@ class AsofJoinNode : public ExecNode { for (int i = 0; i < input_schema[j]->num_fields(); ++i) { const auto field = input_schema[j]->field(i); - bool as_output; + bool as_output; // true if the field appears as an output + int final_output_idx; // the final output index for the field if (i == on_field_ix) { ARROW_RETURN_NOT_OK(is_valid_on_field(field)); // Only add on field from the left table as_output = (j == 0); + final_output_idx = as_output ? output_field_idx++ : indices_of_on_key[0]; } else if (std_has(by_field_ix, i)) { ARROW_RETURN_NOT_OK(is_valid_by_field(field)); // Only add by field from the left table as_output = (j == 0); + final_output_idx = as_output ? output_field_idx++ + : indices_of_by_key[0][std_index(by_field_ix, i)]; } else { ARROW_RETURN_NOT_OK(is_valid_data_field(field)); as_output = true; + final_output_idx = output_field_idx++; } if (as_output) { fields.push_back(field); } if (field_output_indices) { - field_output_indices->push_back(as_output ? output_field_idx++ : i); + field_output_indices->push_back(final_output_idx); } } } diff --git a/cpp/src/arrow/engine/substrait/options.cc b/cpp/src/arrow/engine/substrait/options.cc index 93c8e11a7502..0254c2a39536 100644 --- a/cpp/src/arrow/engine/substrait/options.cc +++ b/cpp/src/arrow/engine/substrait/options.cc @@ -30,34 +30,34 @@ namespace engine { class BaseExtensionProvider : public ExtensionProvider { public: - Result MakeRel(std::vector inputs, + Result MakeRel(const std::vector& inputs, const ExtensionDetails& ext_details, const ExtensionSet& ext_set) override { auto details = dynamic_cast(ext_details); return MakeRel(inputs, details.rel, ext_set); } - virtual Result MakeRel(std::vector inputs, + virtual Result MakeRel(const std::vector& inputs, const google::protobuf::Any& rel, const ExtensionSet& ext_set) = 0; }; class DefaultExtensionProvider : public BaseExtensionProvider { public: - Result MakeRel(std::vector inputs, + Result MakeRel(const std::vector& inputs, const google::protobuf::Any& rel, const ExtensionSet& ext_set) override { if (rel.Is()) { substrait_ext::AsOfJoinRel as_of_join_rel; rel.UnpackTo(&as_of_join_rel); - return MakeAsOfJoinRel(std::move(inputs), as_of_join_rel, ext_set); + return MakeAsOfJoinRel(inputs, as_of_join_rel, ext_set); } return Status::NotImplemented("Unrecognized extension in Susbstrait plan: ", rel.DebugString()); } private: - Result MakeAsOfJoinRel(std::vector inputs, + Result MakeAsOfJoinRel(const std::vector& inputs, const substrait_ext::AsOfJoinRel& as_of_join_rel, const ExtensionSet& ext_set) { if (inputs.size() < 2) { @@ -117,7 +117,6 @@ class DefaultExtensionProvider : public BaseExtensionProvider { return RelationInfo{ {compute::Declaration("asofjoin", input_decls, std::move(asofjoin_node_opts)), std::move(schema)}, - std::move(inputs), std::move(field_output_indices)}; } }; diff --git a/cpp/src/arrow/engine/substrait/options.h b/cpp/src/arrow/engine/substrait/options.h index 988a8302e4d9..0024f7e48a8f 100644 --- a/cpp/src/arrow/engine/substrait/options.h +++ b/cpp/src/arrow/engine/substrait/options.h @@ -78,7 +78,7 @@ class ARROW_ENGINE_EXPORT ExtensionProvider { public: static std::shared_ptr kDefaultExtensionProvider; virtual ~ExtensionProvider() = default; - virtual Result MakeRel(std::vector inputs, + virtual Result MakeRel(const std::vector& inputs, const ExtensionDetails& ext_details, const ExtensionSet& ext_set) = 0; }; diff --git a/cpp/src/arrow/engine/substrait/relation.h b/cpp/src/arrow/engine/substrait/relation.h index f59a889d7e03..39f7b650697b 100644 --- a/cpp/src/arrow/engine/substrait/relation.h +++ b/cpp/src/arrow/engine/substrait/relation.h @@ -35,11 +35,12 @@ struct ARROW_ENGINE_EXPORT DeclarationInfo { }; /// Information resulting from converting a Substrait relation. +/// +/// RelationInfo adds the "output indices" field for the extension to define how the +/// fields should be mapped to get the standard indices expected by Substrait. struct ARROW_ENGINE_EXPORT RelationInfo { /// The execution information produced thus far. DeclarationInfo decl_info; - /// The execution information of the inputs to the relation. - std::vector inputs; /// A vector of indices, one per input field per input in order, each index referring /// to the corresponding field within the output schema, if it is in the output, or -1 /// otherwise. Each location in this vector is a field input index. This vector is diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index 2b6db90b8a53..c422222b174e 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -124,15 +124,15 @@ Result ProcessEmit(const RelMessage& rel, } Result ProcessExtensionEmit( - const DeclarationInfo& no_emit_declr, const std::vector& emit_info, + const DeclarationInfo& no_emit_declr, const std::vector& emit_order, const std::vector& field_output_indices) { const std::shared_ptr& input_schema = no_emit_declr.output_schema; std::vector proj_field_refs; - proj_field_refs.reserve(emit_info.size()); + proj_field_refs.reserve(emit_order.size()); FieldVector emit_fields; - emit_fields.reserve(emit_info.size()); + emit_fields.reserve(emit_order.size()); - for (int emit_idx : emit_info) { + for (int emit_idx : emit_order) { if (emit_idx < 0 || static_cast(emit_idx) >= field_output_indices.size()) { return Status::Invalid("Out of bounds emit index ", emit_idx); } @@ -153,35 +153,40 @@ Result ProcessExtensionEmit( std::move(emit_schema)}; } -Result GetExtensionRelationInfo( - const substrait::Rel& rel, const ExtensionSet& ext_set, - const ConversionOptions& conversion_options) { +Result GetExtensionRelationInfo(const substrait::Rel& rel, + const ExtensionSet& ext_set, + const ConversionOptions& conv_opts, + std::vector* inputs_arg) { + if (inputs_arg == nullptr) { + std::vector inputs_tmp; + return GetExtensionRelationInfo(rel, ext_set, conv_opts, &inputs_tmp); + } + std::vector& inputs = *inputs_arg; + inputs.clear(); switch (rel.rel_type_case()) { case substrait::Rel::RelTypeCase::kExtensionLeaf: { const auto& ext = rel.extension_leaf(); DefaultExtensionDetails detail{ext.detail()}; - return conversion_options.extension_provider->MakeRel({}, detail, ext_set); + return conv_opts.extension_provider->MakeRel(inputs, detail, ext_set); } case substrait::Rel::RelTypeCase::kExtensionSingle: { const auto& ext = rel.extension_single(); - ARROW_ASSIGN_OR_RAISE(DeclarationInfo input, - FromProto(ext.input(), ext_set, conversion_options)); + ARROW_ASSIGN_OR_RAISE(DeclarationInfo input_info, + FromProto(ext.input(), ext_set, conv_opts)); + inputs.push_back(std::move(input_info)); DefaultExtensionDetails detail{ext.detail()}; - return conversion_options.extension_provider->MakeRel({input}, detail, ext_set); + return conv_opts.extension_provider->MakeRel(inputs, detail, ext_set); } case substrait::Rel::RelTypeCase::kExtensionMulti: { const auto& ext = rel.extension_multi(); - std::vector inputs; for (const auto& input : ext.inputs()) { - ARROW_ASSIGN_OR_RAISE(auto input_info, - FromProto(input, ext_set, conversion_options)); + ARROW_ASSIGN_OR_RAISE(auto input_info, FromProto(input, ext_set, conv_opts)); inputs.push_back(std::move(input_info)); } DefaultExtensionDetails detail{ext.detail()}; - return conversion_options.extension_provider->MakeRel(std::move(inputs), detail, - ext_set); + return conv_opts.extension_provider->MakeRel(inputs, detail, ext_set); } default: { @@ -743,8 +748,10 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& case substrait::Rel::RelTypeCase::kExtensionLeaf: case substrait::Rel::RelTypeCase::kExtensionSingle: case substrait::Rel::RelTypeCase::kExtensionMulti: { - ARROW_ASSIGN_OR_RAISE(auto ext_rel_info, - GetExtensionRelationInfo(rel, ext_set, conversion_options)); + std::vector ext_rel_inputs; + ARROW_ASSIGN_OR_RAISE( + auto ext_rel_info, + GetExtensionRelationInfo(rel, ext_set, conversion_options, &ext_rel_inputs)); const auto& ext_decl_info = ext_rel_info.decl_info; auto ext_common_opt = GetExtensionRelCommon(rel); bool has_emit = ext_common_opt && ext_common_opt->emit_kind_case() == @@ -756,17 +763,22 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& return Status::NotImplemented("Emit not supported by ", ext_decl_info.declaration.factory_name); } + // Set up the emit order - an ordered list of indices that specifies an output + // mapping as expected by Substrait. This is a sublist of [0..N), where N is the + // total number of input fields across all inputs of the relation, that selects + // from these input fields. std::vector emit_order; if (has_emit) { + // the emit order in defined in the Substrait plan - pick it up const auto& emit_info = ext_common_opt->emit(); emit_order.reserve(emit_info.output_mapping_size()); for (const auto& emit_idx : emit_info.output_mapping()) { emit_order.push_back(emit_idx); } } else { - // default output mapping + // the emit order is the default output mapping [0..N) int emit_size = 0; - for (const auto& input : ext_rel_info.inputs) { + for (const auto& input : ext_rel_inputs) { emit_size += input.output_schema->num_fields(); } emit_order.reserve(emit_size); diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index fe19625e95c2..a33543aa3ae3 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -4049,7 +4049,7 @@ TEST(Substrait, SetRelationBasic) { compute::SortOptions sort_options( {compute::SortKey("A", compute::SortOrder::Ascending)}); - CheckRoundTripResult(dummy_schema, std::move(expected_table), exec_context, buf, {}, + CheckRoundTripResult(std::move(expected_table), exec_context, buf, {}, conversion_options, &sort_options); } @@ -4238,8 +4238,8 @@ TEST(Substrait, PlanWithExtension) { input_schema, {{FieldRef(0), {FieldRef(1)}}, {FieldRef(0), {FieldRef(1)}}})); auto expected_table = TableFromJSON( out_schema, {"[[2, 1, 1.1, 1.2], [4, 1, 2.1, 1.2], [6, 2, 3.1, 3.2]]"}); - CheckRoundTripResult(std::move(out_schema), std::move(expected_table), - *compute::default_exec_context(), buf, {}, conversion_options); + CheckRoundTripResult(std::move(expected_table), *compute::default_exec_context(), buf, + {}, conversion_options); } TEST(Substrait, AsOfJoinDefaultEmit) { @@ -4422,8 +4422,8 @@ TEST(Substrait, AsOfJoinDefaultEmit) { auto expected_table = TableFromJSON( out_schema, {"[[2, 1, 1.1, 2, 1, 1.2], [4, 1, 2.1, 4, 1, 1.2], [6, 2, 3.1, 6, 2, 3.2]]"}); - CheckRoundTripResult(std::move(out_schema), std::move(expected_table), - *compute::default_exec_context(), buf, {}, conversion_options); + CheckRoundTripResult(std::move(expected_table), *compute::default_exec_context(), buf, + {}, conversion_options); } } // namespace engine From b715529213effcffd1ec27f21b9270a0f5f185f8 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Wed, 21 Dec 2022 08:40:02 -0500 Subject: [PATCH 5/9] remove header dependency on protobuf --- cpp/src/arrow/engine/substrait/options.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/arrow/engine/substrait/options.h b/cpp/src/arrow/engine/substrait/options.h index 0024f7e48a8f..5d4533b1786a 100644 --- a/cpp/src/arrow/engine/substrait/options.h +++ b/cpp/src/arrow/engine/substrait/options.h @@ -23,8 +23,6 @@ #include #include -#include - #include "arrow/compute/type_fwd.h" #include "arrow/engine/substrait/type_fwd.h" #include "arrow/engine/substrait/visibility.h" From 39df9e1c0ca2f27521021e5292a6055082eda4fe Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Wed, 21 Dec 2022 08:40:36 -0500 Subject: [PATCH 6/9] avoid dynamic cast failure in debug mode --- cpp/src/arrow/util/future.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/util/future.cc b/cpp/src/arrow/util/future.cc index c430ad1fc738..05cf8f655e97 100644 --- a/cpp/src/arrow/util/future.cc +++ b/cpp/src/arrow/util/future.cc @@ -169,7 +169,7 @@ class ConcreteFutureImpl : public FutureImpl { namespace { ConcreteFutureImpl* GetConcreteFuture(FutureImpl* future) { - return checked_cast(future); + return static_cast(future); } } // namespace From c19a2bddb1b8fcaec4a443df56021c3d5348dd91 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Thu, 22 Dec 2022 04:49:29 -0500 Subject: [PATCH 7/9] require non-null executor in as-of-join --- cpp/src/arrow/compute/exec/asof_join_node.cc | 3 ++ .../arrow/compute/exec/asof_join_node_test.cc | 8 ++-- cpp/src/arrow/compute/exec/exec_plan.cc | 3 ++ cpp/src/arrow/engine/substrait/serde_test.cc | 41 ++++++++++++++++--- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/compute/exec/asof_join_node.cc b/cpp/src/arrow/compute/exec/asof_join_node.cc index 2155e63611c9..4df99a6eebdf 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node.cc @@ -1039,6 +1039,9 @@ class AsofJoinNode : public ExecNode { bool may_rehash); Status Init() override { + if (plan()->exec_context()->executor() == nullptr) { + return Status::Invalid("AsOfJoinNode requires a non-null executor"); + } auto inputs = this->inputs(); for (size_t i = 0; i < inputs.size(); i++) { RETURN_NOT_OK(key_hashers_[i]->Init(plan()->exec_context(), output_schema())); diff --git a/cpp/src/arrow/compute/exec/asof_join_node_test.cc b/cpp/src/arrow/compute/exec/asof_join_node_test.cc index 7da23a45d7f3..adb1efd65531 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node_test.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node_test.cc @@ -232,7 +232,8 @@ void CheckRunOutput(const BatchesWithSchema& l_batches, const BatchesWithSchema& r1_batches, const BatchesWithSchema& exp_batches, const AsofJoinNodeOptions join_options) { - auto exec_ctx = std::make_unique(default_memory_pool(), nullptr); + ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); + auto exec_ctx = std::make_unique(default_memory_pool(), io_executor.get()); ASSERT_OK_AND_ASSIGN(auto plan, ExecPlan::Make(exec_ctx.get())); Declaration join{"asofjoin", join_options}; @@ -279,8 +280,9 @@ void DoInvalidPlanTest(const BatchesWithSchema& l_batches, const AsofJoinNodeOptions& join_options, const std::string& expected_error_str, bool fail_on_plan_creation = false) { - ExecContext exec_ctx; - ASSERT_OK_AND_ASSIGN(auto plan, ExecPlan::Make(&exec_ctx)); + ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); + auto exec_ctx = std::make_unique(default_memory_pool(), io_executor.get()); + ASSERT_OK_AND_ASSIGN(auto plan, ExecPlan::Make(exec_ctx.get())); Declaration join{"asofjoin", join_options}; join.inputs.emplace_back(Declaration{ diff --git a/cpp/src/arrow/compute/exec/exec_plan.cc b/cpp/src/arrow/compute/exec/exec_plan.cc index 22250d2f993f..c3968ca786af 100644 --- a/cpp/src/arrow/compute/exec/exec_plan.cc +++ b/cpp/src/arrow/compute/exec/exec_plan.cc @@ -159,6 +159,9 @@ struct ExecPlanImpl : public ExecPlan { // away soon (or at least be replaced by a sub-scheduler to facilitate OT) for (auto& n : nodes_) { RETURN_NOT_OK(n->Init()); + } + // schedule only after all nodes are initialized, to avoid hanging tasks + for (auto& n : nodes_) { async_scheduler->AddSimpleTask([&] { return n->finished(); }); } diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index a33543aa3ae3..417af38bba9b 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -209,6 +209,19 @@ inline compute::Expression UseBoringRefs(const compute::Expression& expr) { return compute::Expression{std::move(modified_call)}; } +Status CheckTable(compute::ExecContext& exec_context, std::shared_ptr& buf, + const ConversionOptions& conversion_options = {}) { + std::shared_ptr sp_ext_id_reg = MakeExtensionIdRegistry(); + ExtensionIdRegistry* ext_id_reg = sp_ext_id_reg.get(); + ExtensionSet ext_set(ext_id_reg); + ARROW_ASSIGN_OR_RAISE(auto sink_decls, DeserializePlans( + *buf, [] { return kNullConsumer; }, + ext_id_reg, &ext_set, conversion_options)); + auto& other_declrs = std::get(sink_decls[0].inputs[0]); + ARROW_RETURN_NOT_OK(compute::DeclarationToTable(other_declrs, &exec_context)); + return Status::OK(); +} + void CheckRoundTripResult(const std::shared_ptr expected_table, compute::ExecContext& exec_context, std::shared_ptr& buf, @@ -4053,7 +4066,7 @@ TEST(Substrait, SetRelationBasic) { conversion_options, &sort_options); } -TEST(Substrait, PlanWithExtension) { +void TestSubstraitPlanWithExtension(compute::ExecContext* exec_context) { // This demos an extension relation std::string substrait_json = R"({ "extensionUris": [], @@ -4238,8 +4251,24 @@ TEST(Substrait, PlanWithExtension) { input_schema, {{FieldRef(0), {FieldRef(1)}}, {FieldRef(0), {FieldRef(1)}}})); auto expected_table = TableFromJSON( out_schema, {"[[2, 1, 1.1, 1.2], [4, 1, 2.1, 1.2], [6, 2, 3.1, 3.2]]"}); - CheckRoundTripResult(std::move(expected_table), *compute::default_exec_context(), buf, - {}, conversion_options); + if (exec_context == nullptr) { + ASSERT_THAT(CheckTable(*compute::default_exec_context(), buf, conversion_options), + Raises(StatusCode::Invalid, + HasSubstr("AsOfJoinNode requires a non-null executor"))); + } else { + CheckRoundTripResult(std::move(expected_table), *exec_context, buf, {}, + conversion_options); + } +} + +TEST(Substrait, PlanWithExtension) { + ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); + compute::ExecContext exec_context(default_memory_pool(), io_executor.get()); + TestSubstraitPlanWithExtension(&exec_context); +} + +TEST(Substrait, PlanWithExtensionNoContext) { + TestSubstraitPlanWithExtension(/*exec_context=*/nullptr); } TEST(Substrait, AsOfJoinDefaultEmit) { @@ -4422,8 +4451,10 @@ TEST(Substrait, AsOfJoinDefaultEmit) { auto expected_table = TableFromJSON( out_schema, {"[[2, 1, 1.1, 2, 1, 1.2], [4, 1, 2.1, 4, 1, 1.2], [6, 2, 3.1, 6, 2, 3.2]]"}); - CheckRoundTripResult(std::move(expected_table), *compute::default_exec_context(), buf, - {}, conversion_options); + ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); + compute::ExecContext exec_context(default_memory_pool(), io_executor.get()); + CheckRoundTripResult(std::move(expected_table), exec_context, buf, {}, + conversion_options); } } // namespace engine From cd4bf47e7c845294430ed8c9a2da091e6b7e4b08 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Sat, 14 Jan 2023 08:02:38 -0500 Subject: [PATCH 8/9] fix merge --- .../arrow/compute/exec/asof_join_benchmark.cc | 9 -- cpp/src/arrow/compute/exec/asof_join_node.cc | 14 +-- .../engine/substrait/relation_internal.cc | 101 +++++++----------- cpp/src/arrow/engine/substrait/serde_test.cc | 40 ++----- cpp/src/arrow/util/future.cc | 2 +- 5 files changed, 52 insertions(+), 114 deletions(-) diff --git a/cpp/src/arrow/compute/exec/asof_join_benchmark.cc b/cpp/src/arrow/compute/exec/asof_join_benchmark.cc index a5e9f2a33bfd..7ab4ee7a7ff3 100644 --- a/cpp/src/arrow/compute/exec/asof_join_benchmark.cc +++ b/cpp/src/arrow/compute/exec/asof_join_benchmark.cc @@ -113,15 +113,6 @@ AsofJoinNodeOptions GetRepeatedOptions(size_t repeat, FieldRef on_key, return AsofJoinNodeOptions(input_keys, tolerance); } -AsofJoinNodeOptions GetRepeatedOptions(size_t repeat, FieldRef on_key, - std::vector by_key, int64_t tolerance) { - std::vector input_keys(repeat); - for (size_t i = 0; i < repeat; i++) { - input_keys[i] = {on_key, by_key}; - } - return AsofJoinNodeOptions(input_keys, tolerance); -} - static void AsOfJoinOverhead(benchmark::State& state) { int64_t tolerance = 0; auto options = std::make_shared( diff --git a/cpp/src/arrow/compute/exec/asof_join_node.cc b/cpp/src/arrow/compute/exec/asof_join_node.cc index a18610297a7d..a14384ae362a 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node.cc @@ -1149,10 +1149,11 @@ class AsofJoinNode : public ExecNode { if (!out_rb) break; ++batches_produced_; ExecBatch out_b(*out_rb); - ErrorIfNotOk(plan_->ScheduleTask([this, out_b = std::move(out_b)]() mutable { - outputs_[0]->InputReceived(this, std::move(out_b)); - return Status::OK(); - })); + ErrorIfNotOk(plan_->query_context()->ScheduleTask( + [this, out_b = std::move(out_b)]() mutable { + outputs_[0]->InputReceived(this, std::move(out_b)); + return Status::OK(); + })); } else { ErrorIfNotOk(result.status()); EndFromProcessThread(); @@ -1197,12 +1198,13 @@ class AsofJoinNode : public ExecNode { bool may_rehash); Status Init() override { - if (plan()->exec_context()->executor() == nullptr) { + if (plan()->query_context()->exec_context()->executor() == nullptr) { return Status::Invalid("AsOfJoinNode requires a non-null executor"); } auto inputs = this->inputs(); for (size_t i = 0; i < inputs.size(); i++) { - RETURN_NOT_OK(key_hashers_[i]->Init(plan()->exec_context(), output_schema())); + RETURN_NOT_OK(key_hashers_[i]->Init(plan()->query_context()->exec_context(), + output_schema())); ARROW_ASSIGN_OR_RAISE( auto input_state, InputState::Make(i, tolerance_, must_hash_, may_rehash_, key_hashers_[i].get(), diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index d5ed83985418..c056227b4447 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -74,9 +74,10 @@ struct EmitInfo { std::shared_ptr schema; }; -Result GetEmitInfo(const substrait::RelCommon& rel_common, +template +Result GetEmitInfo(const RelMessage& rel, const std::shared_ptr& input_schema) { - const auto& emit = rel_common.emit(); + const auto& emit = rel.common().emit(); int emit_size = emit.output_mapping_size(); std::vector proj_field_refs(emit_size); EmitInfo emit_info; @@ -91,41 +92,15 @@ Result GetEmitInfo(const substrait::RelCommon& rel_common, return std::move(emit_info); } -Result ProcessEmit(std::optional rel_common_opt, - const DeclarationInfo& no_emit_declr, - const std::shared_ptr& schema) { +Result ProcessEmitProject( + std::optional rel_common_opt, + const DeclarationInfo& project_declr, const std::shared_ptr& input_schema) { if (rel_common_opt) { switch (rel_common_opt->emit_kind_case()) { - case substrait::RelCommon::EmitKindCase::kDirect: - return no_emit_declr; - case substrait::RelCommon::EmitKindCase::kEmit: { - ARROW_ASSIGN_OR_RAISE(auto emit_info, GetEmitInfo(*rel_common_opt, schema)); - return DeclarationInfo{ - compute::Declaration::Sequence( - {no_emit_declr.declaration, - {"project", - compute::ProjectNodeOptions{std::move(emit_info.expressions)}}}), - std::move(emit_info.schema)}; - } - default: - return Status::Invalid("Invalid emit case"); - } - } else { - return no_emit_declr; - } -} -/// In the specialization, a single ProjectNode is being used to -/// get the Acero relation with or without emit. -template <> -Result ProcessEmit(const substrait::ProjectRel& rel, - const DeclarationInfo& project_declr, - const std::shared_ptr& input_schema) { - if (rel.has_common()) { - switch (rel.common().emit_kind_case()) { case substrait::RelCommon::EmitKindCase::kDirect: return project_declr; case substrait::RelCommon::EmitKindCase::kEmit: { - const auto& emit = rel.common().emit(); + const auto& emit = rel_common_opt->emit(); int emit_size = emit.output_mapping_size(); const auto& proj_options = checked_cast( *project_declr.declaration.options); @@ -157,8 +132,34 @@ template Result ProcessEmit(const RelMessage& rel, const DeclarationInfo& no_emit_declr, const std::shared_ptr& schema) { - return ProcessEmit(rel.has_common() ? std::optional(rel.common()) : std::nullopt, - no_emit_declr, schema); + if (rel.has_common()) { + switch (rel.common().emit_kind_case()) { + case substrait::RelCommon::EmitKindCase::kDirect: + return no_emit_declr; + case substrait::RelCommon::EmitKindCase::kEmit: { + ARROW_ASSIGN_OR_RAISE(auto emit_info, GetEmitInfo(rel, schema)); + return DeclarationInfo{ + compute::Declaration::Sequence( + {no_emit_declr.declaration, + {"project", + compute::ProjectNodeOptions{std::move(emit_info.expressions)}}}), + std::move(emit_info.schema)}; + } + default: + return Status::Invalid("Invalid emit case"); + } + } else { + return no_emit_declr; + } +} +/// In the specialization, a single ProjectNode is being used to +/// get the Acero relation with or without emit. +template <> +Result ProcessEmit(const substrait::ProjectRel& rel, + const DeclarationInfo& no_emit_declr, + const std::shared_ptr& schema) { + return ProcessEmitProject(rel.has_common() ? std::optional(rel.common()) : std::nullopt, + no_emit_declr, schema); } Result ProcessExtensionEmit( @@ -797,7 +798,8 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& substrait::RelCommon::EmitKindCase::kEmit; if (!ext_rel_info.field_output_indices) { if (!has_emit) { - return ProcessEmit(ext_common_opt, ext_decl_info, ext_decl_info.output_schema); + return ProcessEmitProject(ext_common_opt, ext_decl_info, + ext_decl_info.output_schema); } return Status::NotImplemented("Emit not supported by ", ext_decl_info.declaration.factory_name); @@ -871,35 +873,6 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& return ProcessEmit(std::move(set), std::move(set_declaration), std::move(union_schema)); } - case substrait::Rel::RelTypeCase::kExtensionLeaf: { - const auto& ext = rel.extension_leaf(); - ARROW_ASSIGN_OR_RAISE( - auto ext_leaf_decl, - conversion_options.extension_provider->MakeRel({}, ext.detail(), ext_set)); - return ProcessEmit(ext, std::move(ext_leaf_decl), ext_leaf_decl.output_schema); - } - case substrait::Rel::RelTypeCase::kExtensionSingle: { - const auto& ext = rel.extension_single(); - ARROW_ASSIGN_OR_RAISE(DeclarationInfo input, - FromProto(ext.input(), ext_set, conversion_options)); - ARROW_ASSIGN_OR_RAISE( - auto ext_single_decl, - conversion_options.extension_provider->MakeRel({input}, ext.detail(), ext_set)); - return ProcessEmit(ext, std::move(ext_single_decl), ext_single_decl.output_schema); - } - case substrait::Rel::RelTypeCase::kExtensionMulti: { - const auto& ext = rel.extension_multi(); - std::vector inputs; - for (const auto& input : ext.inputs()) { - ARROW_ASSIGN_OR_RAISE(auto input_info, - FromProto(input, ext_set, conversion_options)); - inputs.push_back(std::move(input_info)); - } - ARROW_ASSIGN_OR_RAISE( - auto ext_multi_decl, - conversion_options.extension_provider->MakeRel(inputs, ext.detail(), ext_set)); - return ProcessEmit(ext, std::move(ext_multi_decl), ext_multi_decl.output_schema); - } default: break; diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index 79d8254f3e38..971d529284db 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -169,7 +169,7 @@ inline compute::Expression UseBoringRefs(const compute::Expression& expr) { return compute::Expression{std::move(modified_call)}; } -Status CheckTable(compute::ExecContext& exec_context, std::shared_ptr& buf, +Status CheckTable(std::shared_ptr& buf, const ConversionOptions& conversion_options = {}) { std::shared_ptr sp_ext_id_reg = MakeExtensionIdRegistry(); ExtensionIdRegistry* ext_id_reg = sp_ext_id_reg.get(); @@ -178,7 +178,7 @@ Status CheckTable(compute::ExecContext& exec_context, std::shared_ptr& b *buf, [] { return kNullConsumer; }, ext_id_reg, &ext_set, conversion_options)); auto& other_declrs = std::get(sink_decls[0].inputs[0]); - ARROW_RETURN_NOT_OK(compute::DeclarationToTable(other_declrs, &exec_context)); + ARROW_RETURN_NOT_OK(compute::DeclarationToTable(other_declrs)); return Status::OK(); } @@ -2293,16 +2293,6 @@ TEST(SubstraitRoundTrip, BasicPlanEndToEnd) { compute::AssertTablesEqualIgnoringOrder(expected_table, rnd_trp_table); } -NamedTableProvider ProvideMadeTable( - std::function>(const std::vector&)> make) { - return [make](const std::vector& names) -> Result { - ARROW_ASSIGN_OR_RAISE(auto table, make(names)); - std::shared_ptr options = - std::make_shared(table); - return compute::Declaration("table_source", {}, options, "mock_source"); - }; -} - TEST(SubstraitRoundTrip, FilterNamedTable) { arrow::dataset::internal::Initialize(); @@ -4258,7 +4248,7 @@ TEST(Substrait, PlanWithAsOfJoinExtension) { "extension_multi": { "common": { "emit": { - "outputMapping": [0, 1, 2, 3] + "outputMapping": [0, 1, 2, 5] } }, "inputs": [ @@ -4435,7 +4425,7 @@ TEST(Substrait, PlanWithAsOfJoinExtension) { CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); } -void TestSubstraitPlanWithExtension(compute::ExecContext* exec_context) { +TEST(Substrait, PlanWithExtension) { // This demos an extension relation std::string substrait_json = R"({ "extensionUris": [], @@ -4620,24 +4610,7 @@ void TestSubstraitPlanWithExtension(compute::ExecContext* exec_context) { input_schema, {{FieldRef(0), {FieldRef(1)}}, {FieldRef(0), {FieldRef(1)}}})); auto expected_table = TableFromJSON( out_schema, {"[[2, 1, 1.1, 1.2], [4, 1, 2.1, 1.2], [6, 2, 3.1, 3.2]]"}); - if (exec_context == nullptr) { - ASSERT_THAT(CheckTable(*compute::default_exec_context(), buf, conversion_options), - Raises(StatusCode::Invalid, - HasSubstr("AsOfJoinNode requires a non-null executor"))); - } else { - CheckRoundTripResult(std::move(expected_table), *exec_context, buf, {}, - conversion_options); - } -} - -TEST(Substrait, PlanWithExtension) { - ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); - compute::ExecContext exec_context(default_memory_pool(), io_executor.get()); - TestSubstraitPlanWithExtension(&exec_context); -} - -TEST(Substrait, PlanWithExtensionNoContext) { - TestSubstraitPlanWithExtension(/*exec_context=*/nullptr); + CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); } TEST(Substrait, AsOfJoinDefaultEmit) { @@ -4822,8 +4795,7 @@ TEST(Substrait, AsOfJoinDefaultEmit) { {"[[2, 1, 1.1, 2, 1, 1.2], [4, 1, 2.1, 4, 1, 1.2], [6, 2, 3.1, 6, 2, 3.2]]"}); ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); compute::ExecContext exec_context(default_memory_pool(), io_executor.get()); - CheckRoundTripResult(std::move(expected_table), exec_context, buf, {}, - conversion_options); + CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); } } // namespace engine diff --git a/cpp/src/arrow/util/future.cc b/cpp/src/arrow/util/future.cc index 05cf8f655e97..c430ad1fc738 100644 --- a/cpp/src/arrow/util/future.cc +++ b/cpp/src/arrow/util/future.cc @@ -169,7 +169,7 @@ class ConcreteFutureImpl : public FutureImpl { namespace { ConcreteFutureImpl* GetConcreteFuture(FutureImpl* future) { - return static_cast(future); + return checked_cast(future); } } // namespace From 396036e597aeab8989e7279986f4c79b3bb67058 Mon Sep 17 00:00:00 2001 From: Yaron Gvili Date: Sat, 21 Jan 2023 13:04:37 -0500 Subject: [PATCH 9/9] requested fixes --- cpp/src/arrow/compute/exec/asof_join_node.cc | 11 ++--------- .../arrow/engine/substrait/relation_internal.cc | 5 ++--- cpp/src/arrow/engine/substrait/serde_test.cc | 15 --------------- 3 files changed, 4 insertions(+), 27 deletions(-) diff --git a/cpp/src/arrow/compute/exec/asof_join_node.cc b/cpp/src/arrow/compute/exec/asof_join_node.cc index 1475537afad3..837a17d9cf9d 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node.cc @@ -1198,9 +1198,6 @@ class AsofJoinNode : public ExecNode { bool may_rehash); Status Init() override { - if (plan()->query_context()->exec_context()->executor() == nullptr) { - return Status::Invalid("AsOfJoinNode requires a non-null executor"); - } auto inputs = this->inputs(); for (size_t i = 0; i < inputs.size(); i++) { RETURN_NOT_OK(key_hashers_[i]->Init(plan()->query_context()->exec_context(), @@ -1222,12 +1219,8 @@ class AsofJoinNode : public ExecNode { } virtual ~AsofJoinNode() { - process_.Push(false); // poison pill - if (process_thread_.get_id() != std::this_thread::get_id()) { // avoid deadlock - process_thread_.join(); - } else { - process_thread_.detach(); - } + process_.Push(false); // poison pill + process_thread_.join(); } const std::vector& indices_of_on_key() { return indices_of_on_key_; } diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index c056227b4447..10020d39d2e1 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -798,8 +798,7 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& substrait::RelCommon::EmitKindCase::kEmit; if (!ext_rel_info.field_output_indices) { if (!has_emit) { - return ProcessEmitProject(ext_common_opt, ext_decl_info, - ext_decl_info.output_schema); + return ext_decl_info; } return Status::NotImplemented("Emit not supported by ", ext_decl_info.declaration.factory_name); @@ -810,7 +809,7 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& // from these input fields. std::vector emit_order; if (has_emit) { - // the emit order in defined in the Substrait plan - pick it up + // the emit order is defined in the Substrait plan - pick it up const auto& emit_info = ext_common_opt->emit(); emit_order.reserve(emit_info.output_mapping_size()); for (const auto& emit_idx : emit_info.output_mapping()) { diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index 971d529284db..203d1e4b48a2 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -169,19 +169,6 @@ inline compute::Expression UseBoringRefs(const compute::Expression& expr) { return compute::Expression{std::move(modified_call)}; } -Status CheckTable(std::shared_ptr& buf, - const ConversionOptions& conversion_options = {}) { - std::shared_ptr sp_ext_id_reg = MakeExtensionIdRegistry(); - ExtensionIdRegistry* ext_id_reg = sp_ext_id_reg.get(); - ExtensionSet ext_set(ext_id_reg); - ARROW_ASSIGN_OR_RAISE(auto sink_decls, DeserializePlans( - *buf, [] { return kNullConsumer; }, - ext_id_reg, &ext_set, conversion_options)); - auto& other_declrs = std::get(sink_decls[0].inputs[0]); - ARROW_RETURN_NOT_OK(compute::DeclarationToTable(other_declrs)); - return Status::OK(); -} - void CheckRoundTripResult(const std::shared_ptr
expected_table, std::shared_ptr& buf, const std::vector& include_columns = {}, @@ -4793,8 +4780,6 @@ TEST(Substrait, AsOfJoinDefaultEmit) { auto expected_table = TableFromJSON( out_schema, {"[[2, 1, 1.1, 2, 1, 1.2], [4, 1, 2.1, 4, 1, 1.2], [6, 2, 3.1, 6, 2, 3.2]]"}); - ASSERT_OK_AND_ASSIGN(auto io_executor, arrow::internal::ThreadPool::Make(1)); - compute::ExecContext exec_context(default_memory_pool(), io_executor.get()); CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); }