Skip to content
Merged
2 changes: 1 addition & 1 deletion cpp/src/arrow/compute/exec.cc
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ Result<ExecBatch> ExecBatch::Make(std::vector<Datum> values) {
Result<std::shared_ptr<RecordBatch>> ExecBatch::ToRecordBatch(
std::shared_ptr<Schema> schema, MemoryPool* pool) const {
if (static_cast<size_t>(schema->num_fields()) > values.size()) {
return Status::Invalid("ExecBatch::ToTRecordBatch mismatching schema size");
return Status::Invalid("ExecBatch::ToRecordBatch mismatching schema size");
}
ArrayVector columns(schema->num_fields());

Expand Down
64 changes: 49 additions & 15 deletions cpp/src/arrow/compute/exec/asof_join_node.cc
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ inline bool std_has(const T& container, const V& val) {
return container.end() != std_find(container, val);
}

template <typename T, typename V = typename T::value_type,
typename D = typename T::difference_type>
inline D std_index(const T& container, const V& val) {
return std_find(container, val) - container.begin();
}

typedef uint64_t ByType;
typedef uint64_t OnType;
typedef uint64_t HashType;
Expand Down Expand Up @@ -954,20 +960,29 @@ class CompositeReferenceTable {
void AddRecordBatchRef(const std::shared_ptr<RecordBatch>& ref) {
if (!_ptr2ref.count((uintptr_t)ref.get())) _ptr2ref[(uintptr_t)ref.get()] = ref;
}

template <class Type, class Builder = typename TypeTraits<Type>::BuilderType>
enable_if_fixed_width_type<Type, Status> static BuilderAppend(
enable_if_boolean<Type, Status> static BuilderAppend(
Builder& builder, const std::shared_ptr<ArrayData>& source, row_index_t row) {
if (source->IsNull(row)) {
builder.UnsafeAppendNull();
return Status::OK();
}
builder.UnsafeAppend(bit_util::GetBit(source->template GetValues<uint8_t>(1), row));
return Status::OK();
}

if constexpr (is_boolean_type<Type>::value) {
builder.UnsafeAppend(bit_util::GetBit(source->template GetValues<uint8_t>(1), row));
} else {
using CType = typename TypeTraits<Type>::CType;
builder.UnsafeAppend(source->template GetValues<CType>(1)[row]);
template <class Type, class Builder = typename TypeTraits<Type>::BuilderType>
enable_if_t<is_fixed_width_type<Type>::value && !is_boolean_type<Type>::value,
Status> static BuilderAppend(Builder& builder,
const std::shared_ptr<ArrayData>& source,
row_index_t row) {
if (source->IsNull(row)) {
builder.UnsafeAppendNull();
return Status::OK();
}
using CType = typename TypeTraits<Type>::CType;
builder.UnsafeAppend(source->template GetValues<CType>(1)[row]);
return Status::OK();
}

Expand Down Expand Up @@ -1291,16 +1306,27 @@ class AsofJoinNode : public ExecNode {
}
}

/// \brief Make the output schema of an as-of-join node
///
/// Optionally, also provides the field output indices for this node.
/// \see arrow::engine::RelationInfo
///
/// \param[in] input_schema the schema of each input to the node
/// \param[in] indices_of_on_key the on-key index of each input to the node
/// \param[in] indices_of_by_key the by-key indices of each input to the node
/// \param[out] field_output_indices the output index of each field
static arrow::Result<std::shared_ptr<Schema>> MakeOutputSchema(
const std::vector<std::shared_ptr<Schema>> input_schema,
const std::vector<col_index_t>& indices_of_on_key,
const std::vector<std::vector<col_index_t>>& indices_of_by_key) {
const std::vector<std::vector<col_index_t>>& indices_of_by_key,
std::vector<int>* field_output_indices = nullptr) {
std::vector<std::shared_ptr<arrow::Field>> fields;

size_t n_by = indices_of_by_key.size() == 0 ? 0 : indices_of_by_key[0].size();
const DataType* on_key_type = NULLPTR;
std::vector<const DataType*> by_key_type(n_by, NULLPTR);
// Take all non-key, non-time RHS fields
int output_field_idx = 0;
for (size_t j = 0; j < input_schema.size(); ++j) {
const auto& on_field_ix = indices_of_on_key[j];
const auto& by_field_ix = indices_of_by_key[j];
Expand Down Expand Up @@ -1334,22 +1360,30 @@ class AsofJoinNode : public ExecNode {

for (int i = 0; i < input_schema[j]->num_fields(); ++i) {
const auto field = input_schema[j]->field(i);
bool as_output; // true if the field appears as an output
int final_output_idx; // the final output index for the field
if (i == on_field_ix) {
ARROW_RETURN_NOT_OK(is_valid_on_field(field));
// Only add on field from the left table
if (j == 0) {
fields.push_back(field);
}
as_output = (j == 0);
final_output_idx = as_output ? output_field_idx++ : indices_of_on_key[0];
} else if (std_has(by_field_ix, i)) {
ARROW_RETURN_NOT_OK(is_valid_by_field(field));
// Only add by field from the left table
if (j == 0) {
fields.push_back(field);
}
as_output = (j == 0);
final_output_idx = as_output ? output_field_idx++
: indices_of_by_key[0][std_index(by_field_ix, i)];
} else {
ARROW_RETURN_NOT_OK(is_valid_data_field(field));
as_output = true;
final_output_idx = output_field_idx++;
}
if (as_output) {
fields.push_back(field);
}
if (field_output_indices) {
field_output_indices->push_back(final_output_idx);
}
}
}
return std::make_shared<arrow::Schema>(fields);
Expand Down Expand Up @@ -1559,13 +1593,13 @@ namespace asofjoin {

Result<std::shared_ptr<Schema>> MakeOutputSchema(
const std::vector<std::shared_ptr<Schema>>& input_schema,
const std::vector<AsofJoinKeys>& input_keys) {
const std::vector<AsofJoinKeys>& input_keys, std::vector<int>* field_output_indices) {
ARROW_ASSIGN_OR_RAISE(std::vector<col_index_t> indices_of_on_key,
AsofJoinNode::GetIndicesOfOnKey(input_schema, input_keys));
ARROW_ASSIGN_OR_RAISE(std::vector<std::vector<col_index_t>> indices_of_by_key,
AsofJoinNode::GetIndicesOfByKey(input_schema, input_keys));
return AsofJoinNode::MakeOutputSchema(input_schema, indices_of_on_key,
indices_of_by_key);
indices_of_by_key, field_output_indices);
}

} // namespace asofjoin
Expand Down
11 changes: 10 additions & 1 deletion cpp/src/arrow/compute/exec/asof_join_node.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,18 @@ namespace asofjoin {

using AsofJoinKeys = AsofJoinNodeOptions::Keys;

/// \brief Make the output schema of an as-of-join node
///
/// Optionally, also provides the field output indices for this node.
/// \see arrow::engine::RelationInfo
///
/// \param[in] input_schema the schema of each input to the node
/// \param[in] input_keys the key of each input to the node
/// \param[out] field_output_indices the output index of each field
ARROW_EXPORT Result<std::shared_ptr<Schema>> MakeOutputSchema(
const std::vector<std::shared_ptr<Schema>>& input_schema,
const std::vector<AsofJoinKeys>& input_keys);
const std::vector<AsofJoinKeys>& input_keys,
std::vector<int>* field_output_indices = NULLPTR);

} // namespace asofjoin
} // namespace compute
Expand Down
8 changes: 4 additions & 4 deletions cpp/src/arrow/engine/substrait/extension_set.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,12 @@ constexpr const char* kSubstraitAggregateGenericFunctionsUri =
"https://github.com/substrait-io/substrait/blob/main/extensions/"
"functions_aggregate_generic.yaml";

struct Id {
struct ARROW_ENGINE_EXPORT Id {
std::string_view uri, name;
bool empty() const { return uri.empty() && name.empty(); }
std::string ToString() const;
};
struct IdHashEq {
struct ARROW_ENGINE_EXPORT IdHashEq {
size_t operator()(Id id) const;
bool operator()(Id l, Id r) const;
};
Expand All @@ -84,7 +84,7 @@ struct IdHashEq {
/// convert a plan from Substrait to Arrow we need to copy these strings out of
/// the Substrait buffer and into owned storage. This class serves as that owned
/// storage.
class IdStorage {
class ARROW_ENGINE_EXPORT IdStorage {
public:
virtual ~IdStorage() = default;
/// \brief Get an equivalent id pointing into this storage
Expand Down Expand Up @@ -112,7 +112,7 @@ class IdStorage {
/// Substrait call expressions contain a list of arguments which can either
/// be enum arguments (which are serialized as strings), value arguments (which)
/// are Arrow expressions, or type arguments (not yet implemented)
class SubstraitCall {
class ARROW_ENGINE_EXPORT SubstraitCall {
public:
SubstraitCall(Id id, std::shared_ptr<DataType> output_type, bool output_nullable,
bool is_hash = false)
Expand Down
45 changes: 30 additions & 15 deletions cpp/src/arrow/engine/substrait/options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <iostream>

#include "arrow/engine/substrait/options.h"

Expand All @@ -29,13 +28,27 @@
namespace arrow {
namespace engine {

class DefaultExtensionProvider : public ExtensionProvider {
class BaseExtensionProvider : public ExtensionProvider {
public:
Result<DeclarationInfo> MakeRel(const std::vector<DeclarationInfo>& inputs,
const google::protobuf::Any& rel,
const ExtensionSet& ext_set) override {
if (rel.Is<arrow::substrait_ext::AsOfJoinRel>()) {
arrow::substrait_ext::AsOfJoinRel as_of_join_rel;
Result<RelationInfo> MakeRel(const std::vector<DeclarationInfo>& inputs,
const ExtensionDetails& ext_details,
const ExtensionSet& ext_set) override {
auto details = dynamic_cast<const DefaultExtensionDetails&>(ext_details);
return MakeRel(inputs, details.rel, ext_set);
}

virtual Result<RelationInfo> MakeRel(const std::vector<DeclarationInfo>& inputs,
const google::protobuf::Any& rel,
const ExtensionSet& ext_set) = 0;
};

class DefaultExtensionProvider : public BaseExtensionProvider {
public:
Result<RelationInfo> MakeRel(const std::vector<DeclarationInfo>& inputs,
const google::protobuf::Any& rel,
const ExtensionSet& ext_set) override {
if (rel.Is<substrait_ext::AsOfJoinRel>()) {
substrait_ext::AsOfJoinRel as_of_join_rel;
rel.UnpackTo(&as_of_join_rel);
return MakeAsOfJoinRel(inputs, as_of_join_rel, ext_set);
}
Expand All @@ -44,10 +57,9 @@ class DefaultExtensionProvider : public ExtensionProvider {
}

private:
Result<DeclarationInfo> MakeAsOfJoinRel(
const std::vector<DeclarationInfo>& inputs,
const arrow::substrait_ext::AsOfJoinRel& as_of_join_rel,
const ExtensionSet& ext_set) {
Result<RelationInfo> MakeAsOfJoinRel(const std::vector<DeclarationInfo>& inputs,
const substrait_ext::AsOfJoinRel& as_of_join_rel,
const ExtensionSet& ext_set) {
if (inputs.size() < 2) {
return Status::Invalid("substrait_ext::AsOfJoinNode too few input tables: ",
inputs.size());
Expand Down Expand Up @@ -92,18 +104,21 @@ class DefaultExtensionProvider : public ExtensionProvider {
for (size_t i = 0; i < inputs.size(); i++) {
input_schema[i] = inputs[i].output_schema;
}
std::vector<int> field_output_indices;
ARROW_ASSIGN_OR_RAISE(auto schema,
compute::asofjoin::MakeOutputSchema(input_schema, input_keys));
compute::asofjoin::MakeOutputSchema(input_schema, input_keys,
&field_output_indices));
compute::AsofJoinNodeOptions asofjoin_node_opts{std::move(input_keys), tolerance};

// declaration
std::vector<compute::Declaration::Input> input_decls(inputs.size());
for (size_t i = 0; i < inputs.size(); i++) {
input_decls[i] = inputs[i].declaration;
}
return DeclarationInfo{
compute::Declaration("asofjoin", input_decls, std::move(asofjoin_node_opts)),
std::move(schema)};
return RelationInfo{
{compute::Declaration("asofjoin", input_decls, std::move(asofjoin_node_opts)),
std::move(schema)},
std::move(field_output_indices)};
}
};

Expand Down
14 changes: 13 additions & 1 deletion cpp/src/arrow/engine/substrait/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,19 @@ using NamedTableProvider =
std::function<Result<compute::Declaration>(const std::vector<std::string>&)>;
static NamedTableProvider kDefaultNamedTableProvider;

class ExtensionProvider;
class ARROW_ENGINE_EXPORT ExtensionDetails {
public:
virtual ~ExtensionDetails() = default;
};

class ARROW_ENGINE_EXPORT ExtensionProvider {
public:
static std::shared_ptr<ExtensionProvider> kDefaultExtensionProvider;
virtual ~ExtensionProvider() = default;
virtual Result<RelationInfo> MakeRel(const std::vector<DeclarationInfo>& inputs,
const ExtensionDetails& ext_details,
const ExtensionSet& ext_set) = 0;
};

ARROW_ENGINE_EXPORT std::shared_ptr<ExtensionProvider> default_extension_provider();

Expand Down
19 changes: 5 additions & 14 deletions cpp/src/arrow/engine/substrait/options_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,16 @@

#pragma once

#include <vector>

#include <google/protobuf/any.pb.h>

#include "arrow/compute/type_fwd.h"
#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/engine/substrait/visibility.h"
#include "arrow/type_fwd.h"
#include "arrow/engine/substrait/options.h"

namespace arrow {
namespace engine {

class ARROW_ENGINE_EXPORT ExtensionProvider {
class DefaultExtensionDetails : public ExtensionDetails {
public:
static std::shared_ptr<ExtensionProvider> kDefaultExtensionProvider;
virtual ~ExtensionProvider() = default;
virtual Result<DeclarationInfo> MakeRel(const std::vector<DeclarationInfo>& inputs,
const google::protobuf::Any& rel,
const ExtensionSet& ext_set) = 0;
explicit DefaultExtensionDetails(google::protobuf::Any rel) : rel(rel) {}

google::protobuf::Any rel;
};

} // namespace engine
Expand Down
18 changes: 17 additions & 1 deletion cpp/src/arrow/engine/substrait/relation.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,29 @@
namespace arrow {
namespace engine {

/// Information resulting from converting a Substrait relation.
/// Execution information resulting from converting a Substrait relation.
struct ARROW_ENGINE_EXPORT DeclarationInfo {
/// The compute declaration produced thus far.
compute::Declaration declaration;

std::shared_ptr<Schema> output_schema;
};

/// Information resulting from converting a Substrait relation.
///
/// RelationInfo adds the "output indices" field for the extension to define how the
/// fields should be mapped to get the standard indices expected by Substrait.
struct ARROW_ENGINE_EXPORT RelationInfo {
/// The execution information produced thus far.
DeclarationInfo decl_info;
/// A vector of indices, one per input field per input in order, each index referring
/// to the corresponding field within the output schema, if it is in the output, or -1
/// otherwise. Each location in this vector is a field input index. This vector is
/// useful for translating selected field input indices (often from an output mapping in
/// a Substrait plan) of a join-type relation to their locations in the output schema of
/// the relation. This vector is undefined if the translation is unsupported.
std::optional<std::vector<int>> field_output_indices;
};

} // namespace engine
} // namespace arrow
Loading