diff --git a/cpp/src/arrow/compute/exec.cc b/cpp/src/arrow/compute/exec.cc index ee02b26845bd..6100d87d74d9 100644 --- a/cpp/src/arrow/compute/exec.cc +++ b/cpp/src/arrow/compute/exec.cc @@ -179,7 +179,7 @@ Result ExecBatch::Make(std::vector values) { Result> ExecBatch::ToRecordBatch( std::shared_ptr schema, MemoryPool* pool) const { if (static_cast(schema->num_fields()) > values.size()) { - return Status::Invalid("ExecBatch::ToTRecordBatch mismatching schema size"); + return Status::Invalid("ExecBatch::ToRecordBatch mismatching schema size"); } ArrayVector columns(schema->num_fields()); diff --git a/cpp/src/arrow/compute/exec/asof_join_node.cc b/cpp/src/arrow/compute/exec/asof_join_node.cc index f53dc07ac594..40e75efce173 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.cc +++ b/cpp/src/arrow/compute/exec/asof_join_node.cc @@ -61,6 +61,12 @@ inline bool std_has(const T& container, const V& val) { return container.end() != std_find(container, val); } +template +inline D std_index(const T& container, const V& val) { + return std_find(container, val) - container.begin(); +} + typedef uint64_t ByType; typedef uint64_t OnType; typedef uint64_t HashType; @@ -954,20 +960,29 @@ class CompositeReferenceTable { void AddRecordBatchRef(const std::shared_ptr& ref) { if (!_ptr2ref.count((uintptr_t)ref.get())) _ptr2ref[(uintptr_t)ref.get()] = ref; } + template ::BuilderType> - enable_if_fixed_width_type static BuilderAppend( + enable_if_boolean static BuilderAppend( Builder& builder, const std::shared_ptr& source, row_index_t row) { if (source->IsNull(row)) { builder.UnsafeAppendNull(); return Status::OK(); } + builder.UnsafeAppend(bit_util::GetBit(source->template GetValues(1), row)); + return Status::OK(); + } - if constexpr (is_boolean_type::value) { - builder.UnsafeAppend(bit_util::GetBit(source->template GetValues(1), row)); - } else { - using CType = typename TypeTraits::CType; - builder.UnsafeAppend(source->template GetValues(1)[row]); + template ::BuilderType> + enable_if_t::value && !is_boolean_type::value, + Status> static BuilderAppend(Builder& builder, + const std::shared_ptr& source, + row_index_t row) { + if (source->IsNull(row)) { + builder.UnsafeAppendNull(); + return Status::OK(); } + using CType = typename TypeTraits::CType; + builder.UnsafeAppend(source->template GetValues(1)[row]); return Status::OK(); } @@ -1291,16 +1306,27 @@ class AsofJoinNode : public ExecNode { } } + /// \brief Make the output schema of an as-of-join node + /// + /// Optionally, also provides the field output indices for this node. + /// \see arrow::engine::RelationInfo + /// + /// \param[in] input_schema the schema of each input to the node + /// \param[in] indices_of_on_key the on-key index of each input to the node + /// \param[in] indices_of_by_key the by-key indices of each input to the node + /// \param[out] field_output_indices the output index of each field static arrow::Result> MakeOutputSchema( const std::vector> input_schema, const std::vector& indices_of_on_key, - const std::vector>& indices_of_by_key) { + const std::vector>& indices_of_by_key, + std::vector* field_output_indices = nullptr) { std::vector> fields; size_t n_by = indices_of_by_key.size() == 0 ? 0 : indices_of_by_key[0].size(); const DataType* on_key_type = NULLPTR; std::vector by_key_type(n_by, NULLPTR); // Take all non-key, non-time RHS fields + int output_field_idx = 0; for (size_t j = 0; j < input_schema.size(); ++j) { const auto& on_field_ix = indices_of_on_key[j]; const auto& by_field_ix = indices_of_by_key[j]; @@ -1334,22 +1360,30 @@ class AsofJoinNode : public ExecNode { for (int i = 0; i < input_schema[j]->num_fields(); ++i) { const auto field = input_schema[j]->field(i); + bool as_output; // true if the field appears as an output + int final_output_idx; // the final output index for the field if (i == on_field_ix) { ARROW_RETURN_NOT_OK(is_valid_on_field(field)); // Only add on field from the left table - if (j == 0) { - fields.push_back(field); - } + as_output = (j == 0); + final_output_idx = as_output ? output_field_idx++ : indices_of_on_key[0]; } else if (std_has(by_field_ix, i)) { ARROW_RETURN_NOT_OK(is_valid_by_field(field)); // Only add by field from the left table - if (j == 0) { - fields.push_back(field); - } + as_output = (j == 0); + final_output_idx = as_output ? output_field_idx++ + : indices_of_by_key[0][std_index(by_field_ix, i)]; } else { ARROW_RETURN_NOT_OK(is_valid_data_field(field)); + as_output = true; + final_output_idx = output_field_idx++; + } + if (as_output) { fields.push_back(field); } + if (field_output_indices) { + field_output_indices->push_back(final_output_idx); + } } } return std::make_shared(fields); @@ -1559,13 +1593,13 @@ namespace asofjoin { Result> MakeOutputSchema( const std::vector>& input_schema, - const std::vector& input_keys) { + const std::vector& input_keys, std::vector* field_output_indices) { ARROW_ASSIGN_OR_RAISE(std::vector indices_of_on_key, AsofJoinNode::GetIndicesOfOnKey(input_schema, input_keys)); ARROW_ASSIGN_OR_RAISE(std::vector> indices_of_by_key, AsofJoinNode::GetIndicesOfByKey(input_schema, input_keys)); return AsofJoinNode::MakeOutputSchema(input_schema, indices_of_on_key, - indices_of_by_key); + indices_of_by_key, field_output_indices); } } // namespace asofjoin diff --git a/cpp/src/arrow/compute/exec/asof_join_node.h b/cpp/src/arrow/compute/exec/asof_join_node.h index 27777090d3d4..e8a4f62900f2 100644 --- a/cpp/src/arrow/compute/exec/asof_join_node.h +++ b/cpp/src/arrow/compute/exec/asof_join_node.h @@ -28,9 +28,18 @@ namespace asofjoin { using AsofJoinKeys = AsofJoinNodeOptions::Keys; +/// \brief Make the output schema of an as-of-join node +/// +/// Optionally, also provides the field output indices for this node. +/// \see arrow::engine::RelationInfo +/// +/// \param[in] input_schema the schema of each input to the node +/// \param[in] input_keys the key of each input to the node +/// \param[out] field_output_indices the output index of each field ARROW_EXPORT Result> MakeOutputSchema( const std::vector>& input_schema, - const std::vector& input_keys); + const std::vector& input_keys, + std::vector* field_output_indices = NULLPTR); } // namespace asofjoin } // namespace compute diff --git a/cpp/src/arrow/engine/substrait/extension_set.h b/cpp/src/arrow/engine/substrait/extension_set.h index 8b6b5d5159bf..4df64abba075 100644 --- a/cpp/src/arrow/engine/substrait/extension_set.h +++ b/cpp/src/arrow/engine/substrait/extension_set.h @@ -67,12 +67,12 @@ constexpr const char* kSubstraitAggregateGenericFunctionsUri = "https://github.com/substrait-io/substrait/blob/main/extensions/" "functions_aggregate_generic.yaml"; -struct Id { +struct ARROW_ENGINE_EXPORT Id { std::string_view uri, name; bool empty() const { return uri.empty() && name.empty(); } std::string ToString() const; }; -struct IdHashEq { +struct ARROW_ENGINE_EXPORT IdHashEq { size_t operator()(Id id) const; bool operator()(Id l, Id r) const; }; @@ -84,7 +84,7 @@ struct IdHashEq { /// convert a plan from Substrait to Arrow we need to copy these strings out of /// the Substrait buffer and into owned storage. This class serves as that owned /// storage. -class IdStorage { +class ARROW_ENGINE_EXPORT IdStorage { public: virtual ~IdStorage() = default; /// \brief Get an equivalent id pointing into this storage @@ -112,7 +112,7 @@ class IdStorage { /// Substrait call expressions contain a list of arguments which can either /// be enum arguments (which are serialized as strings), value arguments (which) /// are Arrow expressions, or type arguments (not yet implemented) -class SubstraitCall { +class ARROW_ENGINE_EXPORT SubstraitCall { public: SubstraitCall(Id id, std::shared_ptr output_type, bool output_nullable, bool is_hash = false) diff --git a/cpp/src/arrow/engine/substrait/options.cc b/cpp/src/arrow/engine/substrait/options.cc index 9dfd4d7856a1..643ce2296aa7 100644 --- a/cpp/src/arrow/engine/substrait/options.cc +++ b/cpp/src/arrow/engine/substrait/options.cc @@ -14,7 +14,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include #include "arrow/engine/substrait/options.h" @@ -29,13 +28,27 @@ namespace arrow { namespace engine { -class DefaultExtensionProvider : public ExtensionProvider { +class BaseExtensionProvider : public ExtensionProvider { public: - Result MakeRel(const std::vector& inputs, - const google::protobuf::Any& rel, - const ExtensionSet& ext_set) override { - if (rel.Is()) { - arrow::substrait_ext::AsOfJoinRel as_of_join_rel; + Result MakeRel(const std::vector& inputs, + const ExtensionDetails& ext_details, + const ExtensionSet& ext_set) override { + auto details = dynamic_cast(ext_details); + return MakeRel(inputs, details.rel, ext_set); + } + + virtual Result MakeRel(const std::vector& inputs, + const google::protobuf::Any& rel, + const ExtensionSet& ext_set) = 0; +}; + +class DefaultExtensionProvider : public BaseExtensionProvider { + public: + Result MakeRel(const std::vector& inputs, + const google::protobuf::Any& rel, + const ExtensionSet& ext_set) override { + if (rel.Is()) { + substrait_ext::AsOfJoinRel as_of_join_rel; rel.UnpackTo(&as_of_join_rel); return MakeAsOfJoinRel(inputs, as_of_join_rel, ext_set); } @@ -44,10 +57,9 @@ class DefaultExtensionProvider : public ExtensionProvider { } private: - Result MakeAsOfJoinRel( - const std::vector& inputs, - const arrow::substrait_ext::AsOfJoinRel& as_of_join_rel, - const ExtensionSet& ext_set) { + Result MakeAsOfJoinRel(const std::vector& inputs, + const substrait_ext::AsOfJoinRel& as_of_join_rel, + const ExtensionSet& ext_set) { if (inputs.size() < 2) { return Status::Invalid("substrait_ext::AsOfJoinNode too few input tables: ", inputs.size()); @@ -92,8 +104,10 @@ class DefaultExtensionProvider : public ExtensionProvider { for (size_t i = 0; i < inputs.size(); i++) { input_schema[i] = inputs[i].output_schema; } + std::vector field_output_indices; ARROW_ASSIGN_OR_RAISE(auto schema, - compute::asofjoin::MakeOutputSchema(input_schema, input_keys)); + compute::asofjoin::MakeOutputSchema(input_schema, input_keys, + &field_output_indices)); compute::AsofJoinNodeOptions asofjoin_node_opts{std::move(input_keys), tolerance}; // declaration @@ -101,9 +115,10 @@ class DefaultExtensionProvider : public ExtensionProvider { for (size_t i = 0; i < inputs.size(); i++) { input_decls[i] = inputs[i].declaration; } - return DeclarationInfo{ - compute::Declaration("asofjoin", input_decls, std::move(asofjoin_node_opts)), - std::move(schema)}; + return RelationInfo{ + {compute::Declaration("asofjoin", input_decls, std::move(asofjoin_node_opts)), + std::move(schema)}, + std::move(field_output_indices)}; } }; diff --git a/cpp/src/arrow/engine/substrait/options.h b/cpp/src/arrow/engine/substrait/options.h index 41d0792a8d20..bf32610ad417 100644 --- a/cpp/src/arrow/engine/substrait/options.h +++ b/cpp/src/arrow/engine/substrait/options.h @@ -67,7 +67,19 @@ using NamedTableProvider = std::function(const std::vector&)>; static NamedTableProvider kDefaultNamedTableProvider; -class ExtensionProvider; +class ARROW_ENGINE_EXPORT ExtensionDetails { + public: + virtual ~ExtensionDetails() = default; +}; + +class ARROW_ENGINE_EXPORT ExtensionProvider { + public: + static std::shared_ptr kDefaultExtensionProvider; + virtual ~ExtensionProvider() = default; + virtual Result MakeRel(const std::vector& inputs, + const ExtensionDetails& ext_details, + const ExtensionSet& ext_set) = 0; +}; ARROW_ENGINE_EXPORT std::shared_ptr default_extension_provider(); diff --git a/cpp/src/arrow/engine/substrait/options_internal.h b/cpp/src/arrow/engine/substrait/options_internal.h index 0d186147a9a0..9983efbc91d4 100644 --- a/cpp/src/arrow/engine/substrait/options_internal.h +++ b/cpp/src/arrow/engine/substrait/options_internal.h @@ -19,25 +19,16 @@ #pragma once -#include - -#include - -#include "arrow/compute/type_fwd.h" -#include "arrow/engine/substrait/type_fwd.h" -#include "arrow/engine/substrait/visibility.h" -#include "arrow/type_fwd.h" +#include "arrow/engine/substrait/options.h" namespace arrow { namespace engine { -class ARROW_ENGINE_EXPORT ExtensionProvider { +class DefaultExtensionDetails : public ExtensionDetails { public: - static std::shared_ptr kDefaultExtensionProvider; - virtual ~ExtensionProvider() = default; - virtual Result MakeRel(const std::vector& inputs, - const google::protobuf::Any& rel, - const ExtensionSet& ext_set) = 0; + explicit DefaultExtensionDetails(google::protobuf::Any rel) : rel(rel) {} + + google::protobuf::Any rel; }; } // namespace engine diff --git a/cpp/src/arrow/engine/substrait/relation.h b/cpp/src/arrow/engine/substrait/relation.h index 06bb3d58545a..39f7b650697b 100644 --- a/cpp/src/arrow/engine/substrait/relation.h +++ b/cpp/src/arrow/engine/substrait/relation.h @@ -26,7 +26,7 @@ namespace arrow { namespace engine { -/// Information resulting from converting a Substrait relation. +/// Execution information resulting from converting a Substrait relation. struct ARROW_ENGINE_EXPORT DeclarationInfo { /// The compute declaration produced thus far. compute::Declaration declaration; @@ -34,5 +34,21 @@ struct ARROW_ENGINE_EXPORT DeclarationInfo { std::shared_ptr output_schema; }; +/// Information resulting from converting a Substrait relation. +/// +/// RelationInfo adds the "output indices" field for the extension to define how the +/// fields should be mapped to get the standard indices expected by Substrait. +struct ARROW_ENGINE_EXPORT RelationInfo { + /// The execution information produced thus far. + DeclarationInfo decl_info; + /// A vector of indices, one per input field per input in order, each index referring + /// to the corresponding field within the output schema, if it is in the output, or -1 + /// otherwise. Each location in this vector is a field input index. This vector is + /// useful for translating selected field input indices (often from an output mapping in + /// a Substrait plan) of a join-type relation to their locations in the output schema of + /// the relation. This vector is undefined if the translation is unsupported. + std::optional> field_output_indices; +}; + } // namespace engine } // namespace arrow diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index 95b0ef16ba5e..10020d39d2e1 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -92,6 +92,42 @@ Result GetEmitInfo(const RelMessage& rel, return std::move(emit_info); } +Result ProcessEmitProject( + std::optional rel_common_opt, + const DeclarationInfo& project_declr, const std::shared_ptr& input_schema) { + if (rel_common_opt) { + switch (rel_common_opt->emit_kind_case()) { + case substrait::RelCommon::EmitKindCase::kDirect: + return project_declr; + case substrait::RelCommon::EmitKindCase::kEmit: { + const auto& emit = rel_common_opt->emit(); + int emit_size = emit.output_mapping_size(); + const auto& proj_options = checked_cast( + *project_declr.declaration.options); + FieldVector emit_fields(emit_size); + std::vector emit_proj_exprs(emit_size); + for (int i = 0; i < emit_size; i++) { + int32_t map_id = emit.output_mapping(i); + emit_fields[i] = input_schema->field(map_id); + emit_proj_exprs[i] = std::move(proj_options.expressions[map_id]); + } + // Note: DeclarationInfo is created by considering the input to the + // ProjectRel and the ProjectNodeOptions are set by only considering + // what is in the emit expression in Substrait. + return DeclarationInfo{ + compute::Declaration::Sequence( + {std::get(project_declr.declaration.inputs[0]), + {"project", compute::ProjectNodeOptions{std::move(emit_proj_exprs)}}}), + schema(std::move(emit_fields))}; + } + default: + return Status::Invalid("Invalid emit case"); + } + } else { + return project_declr; + } +} + template Result ProcessEmit(const RelMessage& rel, const DeclarationInfo& no_emit_declr, @@ -120,38 +156,104 @@ Result ProcessEmit(const RelMessage& rel, /// get the Acero relation with or without emit. template <> Result ProcessEmit(const substrait::ProjectRel& rel, - const DeclarationInfo& project_declr, - const std::shared_ptr& input_schema) { - if (rel.has_common()) { - switch (rel.common().emit_kind_case()) { - case substrait::RelCommon::EmitKindCase::kDirect: - return project_declr; - case substrait::RelCommon::EmitKindCase::kEmit: { - const auto& emit = rel.common().emit(); - int emit_size = emit.output_mapping_size(); - const auto& proj_options = checked_cast( - *project_declr.declaration.options); - FieldVector emit_fields(emit_size); - std::vector emit_proj_exprs(emit_size); - for (int i = 0; i < emit_size; i++) { - int32_t map_id = emit.output_mapping(i); - emit_fields[i] = input_schema->field(map_id); - emit_proj_exprs[i] = std::move(proj_options.expressions[map_id]); - } - // Note: DeclarationInfo is created by considering the input to the - // ProjectRel and the ProjectNodeOptions are set by only considering - // what is in the emit expression in Substrait. - return DeclarationInfo{ - compute::Declaration::Sequence( - {std::get(project_declr.declaration.inputs[0]), - {"project", compute::ProjectNodeOptions{std::move(emit_proj_exprs)}}}), - schema(std::move(emit_fields))}; + const DeclarationInfo& no_emit_declr, + const std::shared_ptr& schema) { + return ProcessEmitProject(rel.has_common() ? std::optional(rel.common()) : std::nullopt, + no_emit_declr, schema); +} + +Result ProcessExtensionEmit( + const DeclarationInfo& no_emit_declr, const std::vector& emit_order, + const std::vector& field_output_indices) { + const std::shared_ptr& input_schema = no_emit_declr.output_schema; + std::vector proj_field_refs; + proj_field_refs.reserve(emit_order.size()); + FieldVector emit_fields; + emit_fields.reserve(emit_order.size()); + + for (int emit_idx : emit_order) { + if (emit_idx < 0 || static_cast(emit_idx) >= field_output_indices.size()) { + return Status::Invalid("Out of bounds emit index ", emit_idx); + } + int field_idx = field_output_indices[emit_idx]; + if (field_idx < 0) { + return Status::Invalid("Non-output emit index ", emit_idx); + } + proj_field_refs.push_back(compute::field_ref(FieldRef(field_idx))); + emit_fields.push_back(input_schema->field(field_idx)); + } + + std::shared_ptr emit_schema = schema(std::move(emit_fields)); + + return DeclarationInfo{ + compute::Declaration::Sequence( + {no_emit_declr.declaration, + {"project", compute::ProjectNodeOptions{std::move(proj_field_refs)}}}), + std::move(emit_schema)}; +} + +Result GetExtensionRelationInfo(const substrait::Rel& rel, + const ExtensionSet& ext_set, + const ConversionOptions& conv_opts, + std::vector* inputs_arg) { + if (inputs_arg == nullptr) { + std::vector inputs_tmp; + return GetExtensionRelationInfo(rel, ext_set, conv_opts, &inputs_tmp); + } + std::vector& inputs = *inputs_arg; + inputs.clear(); + switch (rel.rel_type_case()) { + case substrait::Rel::RelTypeCase::kExtensionLeaf: { + const auto& ext = rel.extension_leaf(); + DefaultExtensionDetails detail{ext.detail()}; + return conv_opts.extension_provider->MakeRel(inputs, detail, ext_set); + } + + case substrait::Rel::RelTypeCase::kExtensionSingle: { + const auto& ext = rel.extension_single(); + ARROW_ASSIGN_OR_RAISE(DeclarationInfo input_info, + FromProto(ext.input(), ext_set, conv_opts)); + inputs.push_back(std::move(input_info)); + DefaultExtensionDetails detail{ext.detail()}; + return conv_opts.extension_provider->MakeRel(inputs, detail, ext_set); + } + + case substrait::Rel::RelTypeCase::kExtensionMulti: { + const auto& ext = rel.extension_multi(); + for (const auto& input : ext.inputs()) { + ARROW_ASSIGN_OR_RAISE(auto input_info, FromProto(input, ext_set, conv_opts)); + inputs.push_back(std::move(input_info)); } - default: - return Status::Invalid("Invalid emit case"); + DefaultExtensionDetails detail{ext.detail()}; + return conv_opts.extension_provider->MakeRel(inputs, detail, ext_set); + } + + default: { + return Status::Invalid("Invalid extension relation case ", rel.rel_type_case()); + } + } +} + +std::optional GetExtensionRelCommon(const substrait::Rel& rel) { + switch (rel.rel_type_case()) { + case substrait::Rel::RelTypeCase::kExtensionLeaf: { + const auto& ext = rel.extension_leaf(); + return ext.has_common() ? std::optional(ext.common()) : std::nullopt; + } + + case substrait::Rel::RelTypeCase::kExtensionSingle: { + const auto& ext = rel.extension_single(); + return ext.has_common() ? std::optional(ext.common()) : std::nullopt; + } + + case substrait::Rel::RelTypeCase::kExtensionMulti: { + const auto& ext = rel.extension_multi(); + return ext.has_common() ? std::optional(ext.common()) : std::nullopt; + } + + default: { + return std::nullopt; } - } else { - return project_declr; } } @@ -683,6 +785,51 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& std::move(aggregate_schema)); } + case substrait::Rel::RelTypeCase::kExtensionLeaf: + case substrait::Rel::RelTypeCase::kExtensionSingle: + case substrait::Rel::RelTypeCase::kExtensionMulti: { + std::vector ext_rel_inputs; + ARROW_ASSIGN_OR_RAISE( + auto ext_rel_info, + GetExtensionRelationInfo(rel, ext_set, conversion_options, &ext_rel_inputs)); + const auto& ext_decl_info = ext_rel_info.decl_info; + auto ext_common_opt = GetExtensionRelCommon(rel); + bool has_emit = ext_common_opt && ext_common_opt->emit_kind_case() == + substrait::RelCommon::EmitKindCase::kEmit; + if (!ext_rel_info.field_output_indices) { + if (!has_emit) { + return ext_decl_info; + } + return Status::NotImplemented("Emit not supported by ", + ext_decl_info.declaration.factory_name); + } + // Set up the emit order - an ordered list of indices that specifies an output + // mapping as expected by Substrait. This is a sublist of [0..N), where N is the + // total number of input fields across all inputs of the relation, that selects + // from these input fields. + std::vector emit_order; + if (has_emit) { + // the emit order is defined in the Substrait plan - pick it up + const auto& emit_info = ext_common_opt->emit(); + emit_order.reserve(emit_info.output_mapping_size()); + for (const auto& emit_idx : emit_info.output_mapping()) { + emit_order.push_back(emit_idx); + } + } else { + // the emit order is the default output mapping [0..N) + int emit_size = 0; + for (const auto& input : ext_rel_inputs) { + emit_size += input.output_schema->num_fields(); + } + emit_order.reserve(emit_size); + for (int emit_idx = 0; emit_idx < emit_size; emit_idx++) { + emit_order.push_back(emit_idx); + } + } + return ProcessExtensionEmit(std::move(ext_decl_info), emit_order, + *ext_rel_info.field_output_indices); + } + case substrait::Rel::RelTypeCase::kSet: { const auto& set = rel.set(); RETURN_NOT_OK(CheckRelCommon(set, conversion_options)); @@ -725,35 +872,6 @@ Result FromProto(const substrait::Rel& rel, const ExtensionSet& return ProcessEmit(std::move(set), std::move(set_declaration), std::move(union_schema)); } - case substrait::Rel::RelTypeCase::kExtensionLeaf: { - const auto& ext = rel.extension_leaf(); - ARROW_ASSIGN_OR_RAISE( - auto ext_leaf_decl, - conversion_options.extension_provider->MakeRel({}, ext.detail(), ext_set)); - return ProcessEmit(ext, std::move(ext_leaf_decl), ext_leaf_decl.output_schema); - } - case substrait::Rel::RelTypeCase::kExtensionSingle: { - const auto& ext = rel.extension_single(); - ARROW_ASSIGN_OR_RAISE(DeclarationInfo input, - FromProto(ext.input(), ext_set, conversion_options)); - ARROW_ASSIGN_OR_RAISE( - auto ext_single_decl, - conversion_options.extension_provider->MakeRel({input}, ext.detail(), ext_set)); - return ProcessEmit(ext, std::move(ext_single_decl), ext_single_decl.output_schema); - } - case substrait::Rel::RelTypeCase::kExtensionMulti: { - const auto& ext = rel.extension_multi(); - std::vector inputs; - for (const auto& input : ext.inputs()) { - ARROW_ASSIGN_OR_RAISE(auto input_info, - FromProto(input, ext_set, conversion_options)); - inputs.push_back(std::move(input_info)); - } - ARROW_ASSIGN_OR_RAISE( - auto ext_multi_decl, - conversion_options.extension_provider->MakeRel(inputs, ext.detail(), ext_set)); - return ProcessEmit(ext, std::move(ext_multi_decl), ext_multi_decl.output_schema); - } default: break; diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index e17267d4f570..b08130e2ba7e 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -4195,7 +4195,7 @@ TEST(Substrait, PlanWithAsOfJoinExtension) { "extension_multi": { "common": { "emit": { - "outputMapping": [0, 1, 2, 3] + "outputMapping": [0, 1, 2, 5] } }, "inputs": [ @@ -4984,5 +4984,376 @@ TEST(Substrait, CompoundEmitWithFilter) { CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); } +TEST(Substrait, PlanWithExtension) { + // This demos an extension relation + std::string substrait_json = R"({ + "extensionUris": [], + "extensions": [], + "relations": [{ + "root": { + "input": { + "extension_multi": { + "common": { + "emit": { + "outputMapping": [0, 1, 2, 5] + } + }, + "inputs": [ + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value1"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T1"] + } + } + }, + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value2"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T2"] + } + } + } + ], + "detail": { + "@type": "/arrow.substrait_ext.AsOfJoinRel", + "keys" : [ + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + }, + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + } + ], + "tolerance": 1000 + } + } + }, + "names": ["time", "key", "value1", "value2"] + } + }], + "expectedTypeUrls": [] + })"; + + std::vector> input_schema = { + schema({field("time", int32()), field("key", int32()), field("value1", float64())}), + schema( + {field("time", int32()), field("key", int32()), field("value2", float64())})}; + NamedTableProvider table_provider = ProvideMadeTable( + [&input_schema]( + const std::vector& names) -> Result> { + if (names.size() != 1) { + return Status::Invalid("Multiple test table names"); + } + if (names[0] == "T1") { + return TableFromJSON(input_schema[0], + {"[[2, 1, 1.1], [4, 1, 2.1], [6, 2, 3.1]]"}); + } + if (names[0] == "T2") { + return TableFromJSON(input_schema[1], + {"[[1, 1, 1.2], [3, 2, 2.2], [5, 2, 3.2]]"}); + } + return Status::Invalid("Unknown test table name ", names[0]); + }); + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + ASSERT_OK_AND_ASSIGN(auto buf, internal::SubstraitFromJSON("Plan", substrait_json)); + + ASSERT_OK_AND_ASSIGN( + auto out_schema, + compute::asofjoin::MakeOutputSchema( + input_schema, {{FieldRef(0), {FieldRef(1)}}, {FieldRef(0), {FieldRef(1)}}})); + auto expected_table = TableFromJSON( + out_schema, {"[[2, 1, 1.1, 1.2], [4, 1, 2.1, 1.2], [6, 2, 3.1, 3.2]]"}); + CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); +} + +TEST(Substrait, AsOfJoinDefaultEmit) { + std::string substrait_json = R"({ + "extensionUris": [], + "extensions": [], + "relations": [{ + "root": { + "input": { + "extension_multi": { + "inputs": [ + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value1"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T1"] + } + } + }, + { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["time", "key", "value2"], + "struct": { + "types": [ + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "i32": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "fp64": { + "typeVariationReference": 0, + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "typeVariationReference": 0, + "nullability": "NULLABILITY_REQUIRED" + } + }, + "namedTable": { + "names": ["T2"] + } + } + } + ], + "detail": { + "@type": "/arrow.substrait_ext.AsOfJoinRel", + "keys" : [ + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + }, + { + "on": { + "selection": { + "directReference": { + "structField": { + "field": 0, + } + }, + "rootReference": {} + } + }, + "by": [ + { + "selection": { + "directReference": { + "structField": { + "field": 1, + } + }, + "rootReference": {} + } + } + ] + } + ], + "tolerance": 1000 + } + } + }, + "names": ["time", "key", "value1", "time2", "key2", "value2"] + } + }], + "expectedTypeUrls": [] + })"; + + std::vector> input_schema = { + schema({field("time", int32()), field("key", int32()), field("value1", float64())}), + schema( + {field("time", int32()), field("key", int32()), field("value2", float64())})}; + NamedTableProvider table_provider = ProvideMadeTable( + [&input_schema]( + const std::vector& names) -> Result> { + if (names.size() != 1) { + return Status::Invalid("Multiple test table names"); + } + if (names[0] == "T1") { + return TableFromJSON(input_schema[0], + {"[[2, 1, 1.1], [4, 1, 2.1], [6, 2, 3.1]]"}); + } + if (names[0] == "T2") { + return TableFromJSON(input_schema[1], + {"[[1, 1, 1.2], [3, 2, 2.2], [5, 2, 3.2]]"}); + } + return Status::Invalid("Unknown test table name ", names[0]); + }); + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + ASSERT_OK_AND_ASSIGN(auto buf, internal::SubstraitFromJSON("Plan", substrait_json)); + + auto out_schema = schema({field("time", int32()), field("key", int32()), + field("value1", float64()), field("time2", int32()), + field("key2", int32()), field("value2", float64())}); + + auto expected_table = TableFromJSON( + out_schema, + {"[[2, 1, 1.1, 2, 1, 1.2], [4, 1, 2.1, 4, 1, 1.2], [6, 2, 3.1, 6, 2, 3.2]]"}); + CheckRoundTripResult(std::move(expected_table), buf, {}, conversion_options); +} + } // namespace engine } // namespace arrow diff --git a/cpp/src/arrow/engine/substrait/type_fwd.h b/cpp/src/arrow/engine/substrait/type_fwd.h index 6089d3f747a8..bbeb4ef6897a 100644 --- a/cpp/src/arrow/engine/substrait/type_fwd.h +++ b/cpp/src/arrow/engine/substrait/type_fwd.h @@ -27,6 +27,7 @@ class ExtensionSet; struct ConversionOptions; struct DeclarationInfo; +struct RelationInfo; } // namespace engine } // namespace arrow