diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index fb5d603d6e86..58fc66709061 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -395,6 +395,7 @@ if(ARROW_COMPUTE) compute/exec/options.cc compute/exec/order_by_impl.cc compute/exec/partition_util.cc + compute/exec/options.cc compute/exec/project_node.cc compute/exec/sink_node.cc compute/exec/source_node.cc diff --git a/cpp/src/arrow/compute/exec/options.cc b/cpp/src/arrow/compute/exec/options.cc index 585cc7a6a64a..c09ab1c1b68c 100644 --- a/cpp/src/arrow/compute/exec/options.cc +++ b/cpp/src/arrow/compute/exec/options.cc @@ -16,6 +16,10 @@ // under the License. #include "arrow/compute/exec/options.h" +#include "arrow/compute/exec/exec_plan.h" +#include "arrow/io/util_internal.h" +#include "arrow/table.h" +#include "arrow/util/async_generator.h" #include "arrow/util/logging.h" namespace arrow { @@ -44,5 +48,18 @@ std::string ToString(JoinType t) { std::abort(); } +Result> SourceNodeOptions::FromTable( + const Table& table, arrow::internal::Executor* exc) { + std::shared_ptr reader = std::make_shared(table); + + if (exc == nullptr) return Status::TypeError("No executor provided."); + + // Map the RecordBatchReader to a SourceNode + ARROW_ASSIGN_OR_RAISE(auto batch_gen, MakeReaderGenerator(std::move(reader), exc)); + + return std::shared_ptr( + new SourceNodeOptions(table.schema(), batch_gen)); +} + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index 448c609146ea..d5780753254b 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -26,6 +26,8 @@ #include "arrow/compute/api_vector.h" #include "arrow/compute/exec.h" #include "arrow/compute/exec/expression.h" +#include "arrow/result.h" +#include "arrow/util/async_generator.h" #include "arrow/util/async_util.h" #include "arrow/util/optional.h" #include "arrow/util/visibility.h" @@ -33,9 +35,10 @@ namespace arrow { namespace compute { +using AsyncExecBatchGenerator = AsyncGenerator>; + /// \addtogroup execnode-options /// @{ - class ARROW_EXPORT ExecNodeOptions { public: virtual ~ExecNodeOptions() = default; @@ -51,6 +54,9 @@ class ARROW_EXPORT SourceNodeOptions : public ExecNodeOptions { std::function>()> generator) : output_schema(std::move(output_schema)), generator(std::move(generator)) {} + static Result> FromTable(const Table& table, + arrow::internal::Executor*); + std::shared_ptr output_schema; std::function>()> generator; }; diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b4b713ba9914..668b7b788c81 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -399,6 +399,7 @@ set(CYTHON_EXTENSIONS lib _compute _csv + _exec_plan _feather _fs _hdfsio diff --git a/python/pyarrow/_compute.pxd b/python/pyarrow/_compute.pxd index c10867a75b76..ebf2066d0466 100644 --- a/python/pyarrow/_compute.pxd +++ b/python/pyarrow/_compute.pxd @@ -49,3 +49,6 @@ cdef class Expression(_Weakrefable): @staticmethod cdef Expression _expr_or_scalar(object expr) + + +cdef CExpression _true diff --git a/python/pyarrow/_compute.pyx b/python/pyarrow/_compute.pyx index 40fc3b4fff00..2f18ab99866c 100644 --- a/python/pyarrow/_compute.pyx +++ b/python/pyarrow/_compute.pyx @@ -2218,8 +2218,11 @@ cdef class Expression(_Weakrefable): return Expression._call("is_in", [self], options) @staticmethod - def _field(str name not None): - return Expression.wrap(CMakeFieldExpression(tobytes(name))) + def _field(name_or_idx not None): + if isinstance(name_or_idx, str): + return Expression.wrap(CMakeFieldExpression(tobytes(name_or_idx))) + else: + return Expression.wrap(CMakeFieldExpressionByIndex(name_or_idx)) @staticmethod def _scalar(value): diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx new file mode 100644 index 000000000000..350409291526 --- /dev/null +++ b/python/pyarrow/_exec_plan.pyx @@ -0,0 +1,313 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# --------------------------------------------------------------------- +# Implement Internal ExecPlan bindings + +# cython: profile=False +# distutils: language = c++ +# cython: language_level = 3 + +from cython.operator cimport dereference as deref, preincrement as inc + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) +from pyarrow.lib import tobytes, _pc +from pyarrow._compute cimport Expression, _true + + +cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads=True): + """ + Internal Function to create an ExecPlan and run it. + + Parameters + ---------- + inputs : list of Table or Dataset + The sources from which the ExecPlan should fetch data. + In most cases this is only one, unless the first node of the + plan is able to get data from multiple different sources. + output_type : Table or Dataset + In which format the output should be provided. + plan : vector[CDeclaration] + The nodes of the plan that should be applied to the sources + to produce the output. + use_threads : bool, default True + Whenever to use multithreading or not. + """ + cdef: + CExecutor *c_executor + shared_ptr[CThreadPool] c_executor_sptr + shared_ptr[CExecContext] c_exec_context + shared_ptr[CExecPlan] c_exec_plan + vector[CDeclaration] c_decls + vector[CExecNode*] _empty + vector[CExecNode*] c_final_node_vec + CExecNode *c_node + CTable* c_table + shared_ptr[CTable] c_out_table + shared_ptr[CSourceNodeOptions] c_sourceopts + shared_ptr[CSinkNodeOptions] c_sinkopts + shared_ptr[CAsyncExecBatchGenerator] c_async_exec_batch_gen + shared_ptr[CRecordBatchReader] c_recordbatchreader + vector[CDeclaration].iterator plan_iter + + if use_threads: + c_executor = GetCpuThreadPool() + else: + c_executor_sptr = GetResultValue(CThreadPool.Make(1)) + c_executor = c_executor_sptr.get() + + c_exec_context = make_shared[CExecContext]( + c_default_memory_pool(), c_executor) + c_exec_plan = GetResultValue(CExecPlan.Make(c_exec_context.get())) + + plan_iter = plan.begin() + + # Create source nodes for each input + for ipt in inputs: + if isinstance(ipt, Table): + c_in_table = pyarrow_unwrap_table(ipt).get() + c_sourceopts = GetResultValue( + CSourceNodeOptions.FromTable(deref(c_in_table), deref(c_exec_context).executor())) + else: + raise TypeError("Unsupported type") + + if plan_iter != plan.end(): + # Flag the source as the input of the first plan node. + deref(plan_iter).inputs.push_back(CDeclaration.Input( + CDeclaration(tobytes("source"), deref(c_sourceopts)) + )) + else: + # Empty plan, make the source the first plan node. + c_decls.push_back( + CDeclaration(tobytes("source"), deref(c_sourceopts)) + ) + + # Add Here additional nodes + while plan_iter != plan.end(): + c_decls.push_back(deref(plan_iter)) + inc(plan_iter) + + # Add all CDeclarations to the plan + c_node = GetResultValue( + CDeclaration.Sequence(c_decls).AddToPlan(&deref(c_exec_plan)) + ) + c_final_node_vec.push_back(c_node) + + # Create the output node + c_async_exec_batch_gen = make_shared[CAsyncExecBatchGenerator]() + c_sinkopts = make_shared[CSinkNodeOptions](c_async_exec_batch_gen.get()) + GetResultValue( + MakeExecNode(tobytes("sink"), &deref(c_exec_plan), + c_final_node_vec, deref(c_sinkopts)) + ) + + # Convert the asyncgenerator to a sync batch reader + c_recordbatchreader = MakeGeneratorReader(c_node.output_schema(), + deref(c_async_exec_batch_gen), + deref(c_exec_context).memory_pool()) + + # Start execution of the ExecPlan + deref(c_exec_plan).Validate() + deref(c_exec_plan).StartProducing() + + # Convert output to the expected one. + if output_type == Table: + c_out_table = GetResultValue( + CTable.FromRecordBatchReader(c_recordbatchreader.get())) + output = pyarrow_wrap_table(c_out_table) + else: + raise TypeError("Unsupported output type") + + deref(c_exec_plan).StopProducing() + + return output + + +def tables_join(join_type, left_table not None, left_keys, + right_table not None, right_keys, + left_suffix=None, right_suffix=None, + use_threads=True, coalesce_keys=False): + """ + Perform join of two tables. + + The result will be an output table with the result of the join operation + + Parameters + ---------- + join_type : str + One of supported join types. + left_table : Table + The left table for the join operation. + left_keys : str or list[str] + The left table key (or keys) on which the join operation should be performed. + right_table : Table + The right table for the join operation. + right_keys : str or list[str] + The right table key (or keys) on which the join operation should be performed. + left_suffix : str, default None + Which suffix to add to right column names. This prevents confusion + when the columns in left and right tables have colliding names. + right_suffix : str, default None + Which suffic to add to the left column names. This prevents confusion + when the columns in left and right tables have colliding names. + use_threads : bool, default True + Whenever to use multithreading or not. + coalesce_keys : bool, default False + If the duplicated keys should be omitted from one of the sides + in the join result. + + Returns + ------- + result_table : Table + """ + cdef: + vector[CFieldRef] c_left_keys + vector[CFieldRef] c_right_keys + vector[CFieldRef] c_left_columns + vector[CFieldRef] c_right_columns + vector[CDeclaration] c_decl_plan + vector[CExpression] c_projections + vector[c_string] c_projected_col_names + CJoinType c_join_type + + # Prepare left and right tables Keys to send them to the C++ function + left_keys_order = {} + if isinstance(left_keys, str): + left_keys = [left_keys] + for idx, key in enumerate(left_keys): + left_keys_order[key] = idx + c_left_keys.push_back(CFieldRef(tobytes(key))) + + right_keys_order = {} + if isinstance(right_keys, str): + right_keys = [right_keys] + for idx, key in enumerate(right_keys): + right_keys_order[key] = idx + c_right_keys.push_back(CFieldRef(tobytes(key))) + + # By default expose all columns on both left and right table + left_columns = left_table.column_names + right_columns = right_table.column_names + + # Pick the join type + if join_type == "left semi": + c_join_type = CJoinType_LEFT_SEMI + right_columns = [] + elif join_type == "right semi": + c_join_type = CJoinType_RIGHT_SEMI + left_columns = [] + elif join_type == "left anti": + c_join_type = CJoinType_LEFT_ANTI + right_columns = [] + elif join_type == "right anti": + c_join_type = CJoinType_RIGHT_ANTI + left_columns = [] + elif join_type == "inner": + c_join_type = CJoinType_INNER + right_columns = set(right_columns) - set(right_keys) + elif join_type == "left outer": + c_join_type = CJoinType_LEFT_OUTER + right_columns = set(right_columns) - set(right_keys) + elif join_type == "right outer": + c_join_type = CJoinType_RIGHT_OUTER + left_columns = set(left_columns) - set(left_keys) + elif join_type == "full outer": + c_join_type = CJoinType_FULL_OUTER + else: + raise ValueError("Unsupported join type") + + # Turn the columns to vectors of FieldRefs + # and set aside indices of keys. + left_column_keys_indices = {} + for idx, colname in enumerate(left_columns): + c_left_columns.push_back(CFieldRef(tobytes(colname))) + if colname in left_keys: + left_column_keys_indices[colname] = idx + right_column_keys_indices = {} + for idx, colname in enumerate(right_columns): + c_right_columns.push_back(CFieldRef(tobytes(colname))) + if colname in right_keys: + right_column_keys_indices[colname] = idx + + # Add the join node to the execplan + if coalesce_keys: + c_decl_plan.push_back( + CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys, + c_left_columns, c_right_columns, + _true, + tobytes(left_suffix or ""), + tobytes(right_suffix or "") + )) + ) + if join_type == "full outer": + # In case of full outer joins, the join operation will output all columns + # so that we can coalesce the keys and exclude duplicates in a subsequent projection. + left_columns_set = set(left_columns) + right_columns_set = set(right_columns) + # Where the right table columns start. + right_table_index = len(left_columns) + for idx, col in enumerate(left_columns + right_columns): + if idx < len(left_columns) and col in left_column_keys_indices: + # Include keys only once and coalesce left+right table keys. + c_projected_col_names.push_back(tobytes(col)) + # Get the index of the right key that is being paired + # with this left key. We do so by retrieving the name + # of the right key that is in the same position in the provided keys + # and then looking up the index for that name in the right table. + right_key_index = right_column_keys_indices[right_keys[left_keys_order[col]]] + c_projections.push_back(Expression.unwrap( + Expression._call("coalesce", [ + Expression._field(idx), Expression._field( + right_table_index+right_key_index) + ]) + )) + elif idx >= right_table_index and col in right_column_keys_indices: + # Do not include right table keys. As they would lead to duplicated keys. + continue + else: + # For all the other columns incude them as they are. + # Just recompute the suffixes that the join produced as the projection + # would lose them otherwise. + if left_suffix and idx < right_table_index and col in right_columns_set: + col += left_suffix + if right_suffix and idx >= right_table_index and col in left_columns_set: + col += right_suffix + c_projected_col_names.push_back(tobytes(col)) + c_projections.push_back( + Expression.unwrap(Expression._field(idx))) + c_decl_plan.push_back( + CDeclaration(tobytes("project"), CProjectNodeOptions( + c_projections, c_projected_col_names)) + ) + else: + c_decl_plan.push_back( + CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys, + _true, + tobytes(left_suffix or ""), + tobytes(right_suffix or "") + )) + ) + + result_table = execplan([left_table, right_table], + output_type=Table, + plan=c_decl_plan) + + return result_table diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index d66c3dd5df3a..0da61dcec0a7 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -79,6 +79,7 @@ # Expressions Expression, ) +from pyarrow import _exec_plan # noqa from collections import namedtuple import inspect @@ -591,22 +592,22 @@ def bottom_k_unstable(values, k, sort_keys=None, *, memory_pool=None): return call_function("select_k_unstable", [values], options, memory_pool) -def field(name): - """Reference a named column of the dataset. +def field(name_or_index): + """Reference a column of the dataset. Stores only the field's name. Type and other information is known only when the expression is bound to a dataset having an explicit scheme. Parameters ---------- - name : string - The name of the field the expression references to. + name_or_index : string or int + The name or index of the field the expression references to. Returns ------- field_expr : Expression """ - return Expression._field(name) + return Expression._field(name_or_index) def scalar(value): diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index b67914ae880d..6246a15fcff9 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -812,6 +812,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const shared_ptr[CSchema]& schema, const vector[shared_ptr[CArray]]& arrays) + @staticmethod + CResult[shared_ptr[CTable]] FromRecordBatchReader( + CRecordBatchReader *reader) + @staticmethod CResult[shared_ptr[CTable]] FromRecordBatches( const shared_ptr[CSchema]& schema, @@ -1787,11 +1791,27 @@ cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil: CResult[shared_ptr[CTable]] Read() +cdef extern from "arrow/util/thread_pool.h" namespace "arrow::internal" nogil: + + cdef cppclass CExecutor "arrow::internal::Executor": + pass + + cdef cppclass CThreadPool "arrow::internal::ThreadPool"(CExecutor): + @staticmethod + CResult[shared_ptr[CThreadPool]] Make(int threads) + + CThreadPool* GetCpuThreadPool() + + cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: cdef cppclass CExecContext" arrow::compute::ExecContext": CExecContext() CExecContext(CMemoryPool* pool) + CExecContext(CMemoryPool* pool, CExecutor* exc) + + CMemoryPool* memory_pool() const + CExecutor* executor() cdef cppclass CKernelSignature" arrow::compute::KernelSignature": c_string ToString() const @@ -2393,6 +2413,9 @@ cdef extern from "arrow/compute/exec/expression.h" \ cdef CExpression CMakeFieldExpression \ "arrow::compute::field_ref"(c_string name) + cdef CExpression CMakeFieldExpressionByIndex \ + "arrow::compute::field_ref"(int idx) + cdef CExpression CMakeCallExpression \ "arrow::compute::call"(c_string function, vector[CExpression] arguments, @@ -2423,6 +2446,97 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": const CMonthDayNanoIntervalScalar& scalar) +cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil: + cdef enum CJoinType "arrow::compute::JoinType": + CJoinType_LEFT_SEMI "arrow::compute::JoinType::LEFT_SEMI" + CJoinType_RIGHT_SEMI "arrow::compute::JoinType::RIGHT_SEMI" + CJoinType_LEFT_ANTI "arrow::compute::JoinType::LEFT_ANTI" + CJoinType_RIGHT_ANTI "arrow::compute::JoinType::RIGHT_ANTI" + CJoinType_INNER "arrow::compute::JoinType::INNER" + CJoinType_LEFT_OUTER "arrow::compute::JoinType::LEFT_OUTER" + CJoinType_RIGHT_OUTER "arrow::compute::JoinType::RIGHT_OUTER" + CJoinType_FULL_OUTER "arrow::compute::JoinType::FULL_OUTER" + + cdef cppclass CAsyncExecBatchGenerator "arrow::compute::AsyncExecBatchGenerator": + pass + + cdef cppclass CExecNodeOptions "arrow::compute::ExecNodeOptions": + pass + + cdef cppclass CSourceNodeOptions "arrow::compute::SourceNodeOptions"(CExecNodeOptions): + @staticmethod + CResult[shared_ptr[CSourceNodeOptions]] FromTable(const CTable& table, CExecutor*) + + cdef cppclass CSinkNodeOptions "arrow::compute::SinkNodeOptions"(CExecNodeOptions): + pass + + cdef cppclass CProjectNodeOptions "arrow::compute::ProjectNodeOptions"(CExecNodeOptions): + CProjectNodeOptions(vector[CExpression] expressions) + CProjectNodeOptions(vector[CExpression] expressions, + vector[c_string] names) + + cdef cppclass CHashJoinNodeOptions "arrow::compute::HashJoinNodeOptions"(CExecNodeOptions): + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys) + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys, + CExpression filter, + c_string output_suffix_for_left, + c_string output_suffix_for_right) + CHashJoinNodeOptions(CJoinType join_type, + vector[CFieldRef] left_keys, + vector[CFieldRef] right_keys, + vector[CFieldRef] left_output, + vector[CFieldRef] right_output, + CExpression filter, + c_string output_suffix_for_left, + c_string output_suffix_for_right) + + +cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: + cdef cppclass CDeclaration "arrow::compute::Declaration": + cppclass Input: + Input(CExecNode*) + Input(CDeclaration) + + c_string label + vector[Input] inputs + + CDeclaration(c_string factory_name, CExecNodeOptions options) + + @staticmethod + CDeclaration Sequence(vector[CDeclaration] decls) + + CResult[CExecNode*] AddToPlan(CExecPlan* plan) const + + cdef cppclass CExecPlan "arrow::compute::ExecPlan": + @staticmethod + CResult[shared_ptr[CExecPlan]] Make(CExecContext* exec_context) + + CStatus StartProducing() + CStatus Validate() + CStatus StopProducing() + + vector[CExecNode*] sinks() const + vector[CExecNode*] sources() const + + cdef cppclass CExecNode "arrow::compute::ExecNode": + const vector[CExecNode*]& inputs() const + const shared_ptr[CSchema]& output_schema() const + + cdef cppclass CExecBatch "arrow::compute::ExecBatch": + pass + + shared_ptr[CRecordBatchReader] MakeGeneratorReader( + shared_ptr[CSchema] schema, + CAsyncExecBatchGenerator gen, + CMemoryPool* memory_pool + ) + CResult[CExecNode*] MakeExecNode(c_string factory_name, CExecPlan* plan, + vector[CExecNode*] inputs, + const CExecNodeOptions& options) + + cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CDataType] GetPrimitiveType(Type type) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index f93c20f49d46..af3d3a340ffe 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2600,6 +2600,53 @@ cdef class Table(_PandasConvertible): return table + def join(self, right_table, keys, right_keys=None, join_type="left outer", + left_suffix=None, right_suffix=None, coalesce_keys=True, + use_threads=True): + """ + Perform a join between this table and another one. + + Result of the join will be a new Table, where further + operations can be applied. + + Parameters + ---------- + right_table : Table + The table to join to the current one, acting as the right table + in the join operation. + keys : str or list[str] + The columns from current table that should be used as keys + of the join operation left side. + right_keys : str or list[str], default None + The columns from the right_table that should be used as keys + on the join operation right side. + When ``None`` use the same key names as the left table. + join_type : str, default "left outer" + The kind of join that should be performed, one of + ("left semi", "right semi", "left anti", "right anti", + "inner", "left outer", "right outer", "full outer") + left_suffix : str, default None + Which suffix to add to right column names. This prevents confusion + when the columns in left and right tables have colliding names. + right_suffix : str, default None + Which suffic to add to the left column names. This prevents confusion + when the columns in left and right tables have colliding names. + coalesce_keys : bool, default True + If the duplicated keys should be omitted from one of the sides + in the join result. + use_threads : bool, default True + Whenever to use multithreading or not. + + Returns + ------- + Table + """ + if right_keys is None: + right_keys = keys + return _pc()._exec_plan.tables_join(join_type, self, keys, right_table, right_keys, + left_suffix=left_suffix, right_suffix=right_suffix, + use_threads=use_threads, coalesce_keys=coalesce_keys) + def group_by(self, keys): """Declare a grouping over the columns of the table. diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py new file mode 100644 index 000000000000..8111660b027a --- /dev/null +++ b/python/pyarrow/tests/test_exec_plan.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import pyarrow as pa +import pyarrow._exec_plan as ep + + +def test_joins_corner_cases(): + t1 = pa.Table.from_pydict({ + "colA": [1, 2, 3, 4, 5, 6], + "col2": ["a", "b", "c", "d", "e", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [1, 2, 3, 4, 5], + "col3": ["A", "B", "C", "D", "E"] + }) + + with pytest.raises(pa.ArrowInvalid): + ep.tables_join("left outer", t1, "", t2, "") + + with pytest.raises(TypeError): + ep.tables_join("left outer", None, "colA", t2, "colB") + + with pytest.raises(ValueError): + ep.tables_join("super mario join", t1, "colA", t2, "colB") + + +@pytest.mark.parametrize("jointype,expected", [ + ("left semi", { + "colA": [1, 2], + "col2": ["a", "b"] + }), + ("right semi", { + "colB": [1, 2], + "col3": ["A", "B"] + }), + ("left anti", { + "colA": [6], + "col2": ["f"] + }), + ("right anti", { + "colB": [99], + "col3": ["Z"] + }), + ("inner", { + "colA": [1, 2], + "col2": ["a", "b"], + "col3": ["A", "B"] + }), + ("left outer", { + "colA": [1, 2, 6], + "col2": ["a", "b", "f"], + "col3": ["A", "B", None] + }), + ("right outer", { + "col2": ["a", "b", None], + "colB": [1, 2, 99], + "col3": ["A", "B", "Z"] + }), + ("full outer", { + "colA": [1, 2, 6, 99], + "col2": ["a", "b", "f", None], + "col3": ["A", "B", None, "Z"] + }) +]) +@pytest.mark.parametrize("use_threads", [True, False]) +def test_joins(jointype, expected, use_threads): + # Allocate table here instead of using parametrize + # this prevents having arrow allocated memory forever around. + expected = pa.table(expected) + + t1 = pa.Table.from_pydict({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [99, 2, 1], + "col3": ["Z", "B", "A"] + }) + + r = ep.tables_join(jointype, t1, "colA", t2, "colB", + use_threads=use_threads, coalesce_keys=True) + r = r.combine_chunks() + if "right" in jointype: + r = r.sort_by("colB") + else: + r = r.sort_by("colA") + assert r == expected + + +def test_table_join_collisions(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + }) + + result = ep.tables_join( + "full outer", t1, ["colA", "colB"], t2, ["colA", "colB"]) + assert result.combine_chunks() == pa.table([ + [1, 2, 6, None], + [10, 20, 60, None], + ["a", "b", "f", None], + [10, 20, None, 99], + ["A", "B", None, "Z"], + [300, 200, None, 100], + [1, 2, None, 99], + ], names=["colA", "colB", "colVals", "colB", "colVals", "colUniq", "colA"]) + + result = ep.tables_join("full outer", t1, "colA", + t2, "colA", right_suffix="_r", coalesce_keys=False) + assert result.combine_chunks() == pa.table({ + "colA": [1, 2, 6, None], + "colB": [10, 20, 60, None], + "colVals": ["a", "b", "f", None], + "colB_r": [10, 20, None, 99], + "colVals_r": ["A", "B", None, "Z"], + "colUniq": [300, 200, None, 100], + "colA_r": [1, 2, None, 99], + }) + + result = ep.tables_join("full outer", t1, "colA", + t2, "colA", right_suffix="_r", coalesce_keys=True) + assert result.combine_chunks() == pa.table({ + "colA": [1, 2, 6, 99], + "colB": [10, 20, 60, None], + "colVals": ["a", "b", "f", None], + "colB_r": [10, 20, None, 99], + "colVals_r": ["A", "B", None, "Z"], + "colUniq": [300, 200, None, 100] + }) + + +def test_table_join_keys_order(): + t1 = pa.table({ + "colB": [10, 20, 60], + "colA": [1, 2, 6], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colX": [99, 2, 1], + }) + + result = ep.tables_join("full outer", t1, "colA", t2, "colX", + left_suffix="_l", right_suffix="_r", + coalesce_keys=True) + assert result.combine_chunks() == pa.table({ + "colB": [10, 20, 60, None], + "colA": [1, 2, 6, 99], + "colVals_l": ["a", "b", "f", None], + "colVals_r": ["A", "B", None, "Z"], + }) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 9080d14ebc6b..0bcf715b91f5 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2043,3 +2043,78 @@ def test_table_to_recordbatchreader(): reader = table.to_reader(max_chunksize=2) assert reader.read_next_batch().num_rows == 2 assert reader.read_next_batch().num_rows == 1 + + +def test_table_join(): + t1 = pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colB": [99, 2, 1], + "col3": ["Z", "B", "A"] + }) + + result = t1.join(t2, "colA", "colB") + assert result.combine_chunks() == pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"], + "col3": ["A", "B", None] + }) + + result = t1.join(t2, "colA", "colB", join_type="full outer") + assert result.combine_chunks().sort_by("colA") == pa.table({ + "colA": [1, 2, 6, 99], + "col2": ["a", "b", "f", None], + "col3": ["A", "B", None, "Z"] + }) + + +def test_table_join_unique_key(): + t1 = pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colA": [99, 2, 1], + "col3": ["Z", "B", "A"] + }) + + result = t1.join(t2, "colA") + assert result.combine_chunks() == pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"], + "col3": ["A", "B", None] + }) + + result = t1.join(t2, "colA", join_type="full outer", right_suffix="_r") + assert result.combine_chunks().sort_by("colA") == pa.table({ + "colA": [1, 2, 6, 99], + "col2": ["a", "b", "f", None], + "col3": ["A", "B", None, "Z"] + }) + + +def test_table_join_collisions(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colA": [99, 2, 1], + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"] + }) + + result = t1.join(t2, "colA", join_type="full outer") + assert result.combine_chunks().sort_by("colA") == pa.table([ + [1, 2, 6, 99], + [10, 20, 60, None], + ["a", "b", "f", None], + [10, 20, None, 99], + ["A", "B", None, "Z"], + ], names=["colA", "colB", "colVals", "colB", "colVals"]) diff --git a/python/setup.py b/python/setup.py index 7288c5c684b9..6a5e7372068b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -207,6 +207,7 @@ def initialize_options(self): '_dataset', '_dataset_orc', '_dataset_parquet', + '_exec_plan', '_feather', '_parquet', '_parquet_encryption',