From f1d835627da563e17e3bb1f492efec74ba5f7048 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 11 Feb 2022 16:30:16 +0100 Subject: [PATCH 01/24] Test case for execplan --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/compute/exec/options.h | 5 ++ python/CMakeLists.txt | 1 + python/pyarrow/_exec_plan.pyx | 94 ++++++++++++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 46 ++++++++++++++ python/setup.py | 1 + 6 files changed, 148 insertions(+) create mode 100644 python/pyarrow/_exec_plan.pyx diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index fb5d603d6e86..58fc66709061 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -395,6 +395,7 @@ if(ARROW_COMPUTE) compute/exec/options.cc compute/exec/order_by_impl.cc compute/exec/partition_util.cc + compute/exec/options.cc compute/exec/project_node.cc compute/exec/sink_node.cc compute/exec/source_node.cc diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index 448c609146ea..a2db8e6e6a24 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -29,6 +29,7 @@ #include "arrow/util/async_util.h" #include "arrow/util/optional.h" #include "arrow/util/visibility.h" +#include "arrow/result.h" namespace arrow { namespace compute { @@ -51,6 +52,8 @@ class ARROW_EXPORT SourceNodeOptions : public ExecNodeOptions { std::function>()> generator) : output_schema(std::move(output_schema)), generator(std::move(generator)) {} + static Result> FromTable(const Table& table); + std::shared_ptr output_schema; std::function>()> generator; }; @@ -132,6 +135,8 @@ class ARROW_EXPORT SinkNodeOptions : public ExecNodeOptions { util::BackpressureOptions backpressure = {}) : generator(generator), backpressure(std::move(backpressure)) {} + static std::pair, std::shared_ptr> MakeForRecordBatchReader(std::shared_ptr schema); + std::function>()>* generator; util::BackpressureOptions backpressure; }; diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b4b713ba9914..668b7b788c81 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -399,6 +399,7 @@ set(CYTHON_EXTENSIONS lib _compute _csv + _exec_plan _feather _fs _hdfsio diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx new file mode 100644 index 000000000000..100cc9776223 --- /dev/null +++ b/python/pyarrow/_exec_plan.pyx @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# --------------------------------------------------------------------- +# Implement Internal ExecPlan bindings + +# cython: profile=False +# distutils: language = c++ +# cython: language_level = 3 + +from cython.operator cimport dereference as deref + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) +from pyarrow.lib import tobytes + +cdef pair[shared_ptr[CRecordBatchReader], shared_ptr[CExecPlan]] execplan(t): + cdef: + CExecContext c_exec_context = CExecContext(c_default_memory_pool()) + shared_ptr[CExecPlan] c_exec_plan = GetResultValue(CExecPlan.Make(&c_exec_context)) + vector[CDeclaration] c_decls + vector[CExecNode*] c_final_node_vec + CExecNode *c_final_node + CExecNode *c_sink_node + CTable* c_table + shared_ptr[CSourceNodeOptions] c_sourceopts + shared_ptr[CSinkNodeOptions] c_sinkopts + shared_ptr[CRecordBatchReader] c_recordbatchreader + + if isinstance(t, Table): + c_table = pyarrow_unwrap_table(t).get() + else: + raise ValueError("Unsupproted type") + + c_sourceopts = GetResultValue(CSourceNodeOptions.FromTable(deref(c_table))) + c_decls.push_back(CDeclaration(tobytes("source"), deref(c_sourceopts))) + c_decls[0].label = tobytes("source") + + # Add Here additional nodes + CDeclaration.Sequence(c_decls).AddToPlan(&deref(c_exec_plan)) + + c_final_node_vec = deref(c_exec_plan).sinks() + if c_final_node_vec.size() == 0: + c_final_node_vec = deref(c_exec_plan).sources() + c_final_node = c_final_node_vec[0] + + res = CSinkNodeOptions.MakeForRecordBatchReader(c_final_node.output_schema()) + c_sinkopts = res.first + c_recordbatchreader = res.second + c_sink_node = GetResultValue( + MakeExecNode(tobytes("sink"), &deref(c_exec_plan), c_final_node_vec, deref(c_sinkopts)) + ) + + deref(c_exec_plan).Validate() + deref(c_exec_plan).StartProducing() + + return pair[shared_ptr[CRecordBatchReader], shared_ptr[CExecPlan]](c_recordbatchreader, c_exec_plan) + +def test(): + cdef: + shared_ptr[CTable] c_table + shared_ptr[CExecPlan] c_exec_plan + shared_ptr[CRecordBatchReader] c_recordbatchreader + + t = Table.from_pydict({ + "col1": [1, 2, 3, 4, 5], + "col2": ["a", "b", "c", "d", "e"] + }) + + res = execplan(t) + c_recordbatchreader = res.first + c_exec_plan = res.second + + c_table = GetResultValue(CTable.FromRecordBatchReader(c_recordbatchreader.get())) + table = pyarrow_wrap_table(c_table) + + deref(c_exec_plan).StopProducing() + + print(table) \ No newline at end of file diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index b67914ae880d..168789802363 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -812,6 +812,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const shared_ptr[CSchema]& schema, const vector[shared_ptr[CArray]]& arrays) + @staticmethod + CResult[shared_ptr[CTable]] FromRecordBatchReader( + CRecordBatchReader *reader) + @staticmethod CResult[shared_ptr[CTable]] FromRecordBatches( const shared_ptr[CSchema]& schema, @@ -2423,6 +2427,48 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": const CMonthDayNanoIntervalScalar& scalar) +cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil: + cdef cppclass CExecNodeOptions "arrow::compute::ExecNodeOptions": + pass + + cdef cppclass CSourceNodeOptions "arrow::compute::SourceNodeOptions"(CExecNodeOptions): + @staticmethod + CResult[shared_ptr[CSourceNodeOptions]] FromTable(const CTable& table) + + cdef cppclass CSinkNodeOptions "arrow::compute::SinkNodeOptions"(CExecNodeOptions): + @staticmethod + pair[shared_ptr[CSinkNodeOptions], shared_ptr[CRecordBatchReader]] MakeForRecordBatchReader(shared_ptr[CSchema] schema) + + +cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: + cdef cppclass CDeclaration "arrow::compute::Declaration": + c_string label + CDeclaration(c_string factory_name, CExecNodeOptions options) + + @staticmethod + CDeclaration Sequence(vector[CDeclaration] decls) + + CResult[CExecNode*] AddToPlan(CExecPlan* plan) const + + cdef cppclass CExecPlan "arrow::compute::ExecPlan": + @staticmethod + CResult[shared_ptr[CExecPlan]] Make(CExecContext* exec_context) + + CStatus StartProducing() + CStatus Validate() + CStatus StopProducing() + + vector[CExecNode*] sinks() const + vector[CExecNode*] sources() const + + cdef cppclass CExecNode "arrow::compute::ExecNode": + const vector[CExecNode*]& inputs() const + const shared_ptr[CSchema]& output_schema() const + + CResult[CExecNode*] MakeExecNode(c_string factory_name, CExecPlan* plan, + vector[CExecNode*] inputs, + const CExecNodeOptions& options) + cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CDataType] GetPrimitiveType(Type type) diff --git a/python/setup.py b/python/setup.py index 7288c5c684b9..6a5e7372068b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -207,6 +207,7 @@ def initialize_options(self): '_dataset', '_dataset_orc', '_dataset_parquet', + '_exec_plan', '_feather', '_parquet', '_parquet_encryption', From c4a5cfe7423585e6084622add6a08a3fdffee40a Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 11 Feb 2022 16:35:47 +0100 Subject: [PATCH 02/24] Add options --- cpp/src/arrow/compute/exec/options.cc | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cpp/src/arrow/compute/exec/options.cc b/cpp/src/arrow/compute/exec/options.cc index 585cc7a6a64a..19d8651a11d7 100644 --- a/cpp/src/arrow/compute/exec/options.cc +++ b/cpp/src/arrow/compute/exec/options.cc @@ -17,6 +17,11 @@ #include "arrow/compute/exec/options.h" #include "arrow/util/logging.h" +#include "arrow/compute/exec/exec_plan.h" +#include "arrow/compute/exec/options.h" +#include "arrow/io/util_internal.h" +#include "arrow/util/async_generator.h" +#include "arrow/table.h" namespace arrow { namespace compute { @@ -44,5 +49,29 @@ std::string ToString(JoinType t) { std::abort(); } +Result> SourceNodeOptions::FromTable(const Table& table) { + std::shared_ptr reader = std::make_shared(table); + + // Map the RecordBatchReader to a SourceNode + ARROW_ASSIGN_OR_RAISE( + auto batch_gen, + MakeReaderGenerator(std::move(reader), arrow::io::internal::GetIOThreadPool())); + + return std::shared_ptr(new SourceNodeOptions(table.schema(), batch_gen)); +} + + +std::pair, std::shared_ptr> SinkNodeOptions::MakeForRecordBatchReader(std::shared_ptr schema) { + // finally, pipe the project node into a sink node + AsyncGenerator> sink_gen; + auto node_options = std::shared_ptr(new compute::SinkNodeOptions(&sink_gen)); + + // translate sink_gen (async) to sink_reader (sync) + std::shared_ptr sink_reader = compute::MakeGeneratorReader( + schema, std::move(sink_gen), default_memory_pool()); + + return std::make_pair(node_options, sink_reader); +} + } // namespace compute } // namespace arrow From 01ab8b489008aa58ad06133cb0ef46cec713b71f Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 11 Feb 2022 17:59:26 +0100 Subject: [PATCH 03/24] Working execplan --- cpp/src/arrow/compute/exec/options.cc | 13 ++++--------- cpp/src/arrow/compute/exec/options.h | 6 ++++-- python/pyarrow/_exec_plan.pyx | 20 ++++++++++++++------ python/pyarrow/includes/libarrow.pxd | 16 +++++++++++++++- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/cpp/src/arrow/compute/exec/options.cc b/cpp/src/arrow/compute/exec/options.cc index 19d8651a11d7..4c94163ddc15 100644 --- a/cpp/src/arrow/compute/exec/options.cc +++ b/cpp/src/arrow/compute/exec/options.cc @@ -61,16 +61,11 @@ Result> SourceNodeOptions::FromTable(const Ta } -std::pair, std::shared_ptr> SinkNodeOptions::MakeForRecordBatchReader(std::shared_ptr schema) { - // finally, pipe the project node into a sink node - AsyncGenerator> sink_gen; - auto node_options = std::shared_ptr(new compute::SinkNodeOptions(&sink_gen)); +std::pair, std::shared_ptr>>> SinkNodeOptions::MakeWithAsyncGenerator() { + auto sink_gen = std::make_shared>>(); + auto node_options = std::shared_ptr(new compute::SinkNodeOptions(sink_gen.get())); - // translate sink_gen (async) to sink_reader (sync) - std::shared_ptr sink_reader = compute::MakeGeneratorReader( - schema, std::move(sink_gen), default_memory_pool()); - - return std::make_pair(node_options, sink_reader); + return std::make_pair(node_options, sink_gen); } } // namespace compute diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index a2db8e6e6a24..c10834ae5f14 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -29,14 +29,16 @@ #include "arrow/util/async_util.h" #include "arrow/util/optional.h" #include "arrow/util/visibility.h" +#include "arrow/util/async_generator.h" #include "arrow/result.h" namespace arrow { namespace compute { +using AsyncExecBatchGenerator = AsyncGenerator>; + /// \addtogroup execnode-options /// @{ - class ARROW_EXPORT ExecNodeOptions { public: virtual ~ExecNodeOptions() = default; @@ -135,7 +137,7 @@ class ARROW_EXPORT SinkNodeOptions : public ExecNodeOptions { util::BackpressureOptions backpressure = {}) : generator(generator), backpressure(std::move(backpressure)) {} - static std::pair, std::shared_ptr> MakeForRecordBatchReader(std::shared_ptr schema); + static std::pair, std::shared_ptr>>> MakeWithAsyncGenerator(); std::function>()>* generator; util::BackpressureOptions backpressure; diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 100cc9776223..85005d132847 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -29,7 +29,7 @@ from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) from pyarrow.lib import tobytes -cdef pair[shared_ptr[CRecordBatchReader], shared_ptr[CExecPlan]] execplan(t): +cdef (shared_ptr[CAsyncExecBatchGenerator], shared_ptr[CExecPlan], shared_ptr[CRecordBatchReader]) execplan(t): cdef: CExecContext c_exec_context = CExecContext(c_default_memory_pool()) shared_ptr[CExecPlan] c_exec_plan = GetResultValue(CExecPlan.Make(&c_exec_context)) @@ -40,6 +40,7 @@ cdef pair[shared_ptr[CRecordBatchReader], shared_ptr[CExecPlan]] execplan(t): CTable* c_table shared_ptr[CSourceNodeOptions] c_sourceopts shared_ptr[CSinkNodeOptions] c_sinkopts + shared_ptr[CAsyncExecBatchGenerator] c_asyncexecbatchgen shared_ptr[CRecordBatchReader] c_recordbatchreader if isinstance(t, Table): @@ -59,23 +60,29 @@ cdef pair[shared_ptr[CRecordBatchReader], shared_ptr[CExecPlan]] execplan(t): c_final_node_vec = deref(c_exec_plan).sources() c_final_node = c_final_node_vec[0] - res = CSinkNodeOptions.MakeForRecordBatchReader(c_final_node.output_schema()) + res = CSinkNodeOptions.MakeWithAsyncGenerator() c_sinkopts = res.first - c_recordbatchreader = res.second + c_asyncexecbatchgen = res.second c_sink_node = GetResultValue( MakeExecNode(tobytes("sink"), &deref(c_exec_plan), c_final_node_vec, deref(c_sinkopts)) ) + c_recordbatchreader = MakeGeneratorReader(c_final_node.output_schema(), + deref(c_asyncexecbatchgen), + c_exec_context.memory_pool()) + deref(c_exec_plan).Validate() deref(c_exec_plan).StartProducing() - return pair[shared_ptr[CRecordBatchReader], shared_ptr[CExecPlan]](c_recordbatchreader, c_exec_plan) + return (c_asyncexecbatchgen, c_exec_plan, c_recordbatchreader) def test(): cdef: shared_ptr[CTable] c_table shared_ptr[CExecPlan] c_exec_plan + shared_ptr[CAsyncExecBatchGenerator] c_asyncexecbatchgen shared_ptr[CRecordBatchReader] c_recordbatchreader + shared_ptr[CSchema] c_schema t = Table.from_pydict({ "col1": [1, 2, 3, 4, 5], @@ -83,8 +90,9 @@ def test(): }) res = execplan(t) - c_recordbatchreader = res.first - c_exec_plan = res.second + c_asyncexecbatchgen = res[0] + c_exec_plan = res[1] + c_recordbatchreader = res[2] c_table = GetResultValue(CTable.FromRecordBatchReader(c_recordbatchreader.get())) table = pyarrow_wrap_table(c_table) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 168789802363..d359382be13f 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1796,6 +1796,8 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: cdef cppclass CExecContext" arrow::compute::ExecContext": CExecContext() CExecContext(CMemoryPool* pool) + + CMemoryPool* memory_pool() const cdef cppclass CKernelSignature" arrow::compute::KernelSignature": c_string ToString() const @@ -2428,6 +2430,9 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil: + cdef cppclass CAsyncExecBatchGenerator "arrow::compute::AsyncExecBatchGenerator": + pass + cdef cppclass CExecNodeOptions "arrow::compute::ExecNodeOptions": pass @@ -2437,7 +2442,7 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil cdef cppclass CSinkNodeOptions "arrow::compute::SinkNodeOptions"(CExecNodeOptions): @staticmethod - pair[shared_ptr[CSinkNodeOptions], shared_ptr[CRecordBatchReader]] MakeForRecordBatchReader(shared_ptr[CSchema] schema) + pair[shared_ptr[CSinkNodeOptions], shared_ptr[CAsyncExecBatchGenerator]] MakeWithAsyncGenerator() cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: @@ -2465,10 +2470,19 @@ cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nog const vector[CExecNode*]& inputs() const const shared_ptr[CSchema]& output_schema() const + cdef cppclass CExecBatch "arrow::compute::ExecBatch": + pass + + shared_ptr[CRecordBatchReader] MakeGeneratorReader( + shared_ptr[CSchema] schema, + CAsyncExecBatchGenerator gen, + CMemoryPool* memory_pool + ) CResult[CExecNode*] MakeExecNode(c_string factory_name, CExecPlan* plan, vector[CExecNode*] inputs, const CExecNodeOptions& options) + cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CDataType] GetPrimitiveType(Type type) From 5032dc848f9451fd942a283ef15a1c35d484463a Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 15 Feb 2022 11:58:17 +0100 Subject: [PATCH 04/24] Accept additional nodes for the execplan --- cpp/src/arrow/compute/exec/options.cc | 8 ---- cpp/src/arrow/compute/exec/options.h | 2 - python/pyarrow/_exec_plan.pyx | 65 +++++++++++++++------------ python/pyarrow/includes/libarrow.pxd | 7 ++- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/cpp/src/arrow/compute/exec/options.cc b/cpp/src/arrow/compute/exec/options.cc index 4c94163ddc15..8c851ca4ccd2 100644 --- a/cpp/src/arrow/compute/exec/options.cc +++ b/cpp/src/arrow/compute/exec/options.cc @@ -60,13 +60,5 @@ Result> SourceNodeOptions::FromTable(const Ta return std::shared_ptr(new SourceNodeOptions(table.schema(), batch_gen)); } - -std::pair, std::shared_ptr>>> SinkNodeOptions::MakeWithAsyncGenerator() { - auto sink_gen = std::make_shared>>(); - auto node_options = std::shared_ptr(new compute::SinkNodeOptions(sink_gen.get())); - - return std::make_pair(node_options, sink_gen); -} - } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index c10834ae5f14..654a5ab56a57 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -137,8 +137,6 @@ class ARROW_EXPORT SinkNodeOptions : public ExecNodeOptions { util::BackpressureOptions backpressure = {}) : generator(generator), backpressure(std::move(backpressure)) {} - static std::pair, std::shared_ptr>>> MakeWithAsyncGenerator(); - std::function>()>* generator; util::BackpressureOptions backpressure; }; diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 85005d132847..b89a1f7c1993 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -22,14 +22,14 @@ # distutils: language = c++ # cython: language_level = 3 -from cython.operator cimport dereference as deref +from cython.operator cimport dereference as deref, preincrement as inc from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) from pyarrow.lib import tobytes -cdef (shared_ptr[CAsyncExecBatchGenerator], shared_ptr[CExecPlan], shared_ptr[CRecordBatchReader]) execplan(t): +cdef execplan(input_t, output_type, vector[CDeclaration] plan): cdef: CExecContext c_exec_context = CExecContext(c_default_memory_pool()) shared_ptr[CExecPlan] c_exec_plan = GetResultValue(CExecPlan.Make(&c_exec_context)) @@ -38,31 +38,36 @@ cdef (shared_ptr[CAsyncExecBatchGenerator], shared_ptr[CExecPlan], shared_ptr[CR CExecNode *c_final_node CExecNode *c_sink_node CTable* c_table + shared_ptr[CTable] c_out_table shared_ptr[CSourceNodeOptions] c_sourceopts shared_ptr[CSinkNodeOptions] c_sinkopts shared_ptr[CAsyncExecBatchGenerator] c_asyncexecbatchgen shared_ptr[CRecordBatchReader] c_recordbatchreader + vector[CDeclaration].iterator plan_iter - if isinstance(t, Table): - c_table = pyarrow_unwrap_table(t).get() + if isinstance(input_t, Table): + c_in_table = pyarrow_unwrap_table(input_t).get() + c_sourceopts = GetResultValue(CSourceNodeOptions.FromTable(deref(c_in_table))) else: raise ValueError("Unsupproted type") - c_sourceopts = GetResultValue(CSourceNodeOptions.FromTable(deref(c_table))) c_decls.push_back(CDeclaration(tobytes("source"), deref(c_sourceopts))) c_decls[0].label = tobytes("source") # Add Here additional nodes - CDeclaration.Sequence(c_decls).AddToPlan(&deref(c_exec_plan)) + plan_iter = plan.begin() + while plan_iter != plan.end(): + c_decls.push_back(deref(plan_iter)) + inc(plan_iter) - c_final_node_vec = deref(c_exec_plan).sinks() - if c_final_node_vec.size() == 0: - c_final_node_vec = deref(c_exec_plan).sources() - c_final_node = c_final_node_vec[0] + # Add all CDeclarations to the plan + c_final_node = GetResultValue( + CDeclaration.Sequence(c_decls).AddToPlan(&deref(c_exec_plan)) + ) + c_final_node_vec.push_back(c_final_node) - res = CSinkNodeOptions.MakeWithAsyncGenerator() - c_sinkopts = res.first - c_asyncexecbatchgen = res.second + c_asyncexecbatchgen = make_shared[CAsyncExecBatchGenerator]() + c_sinkopts = make_shared[CSinkNodeOptions](c_asyncexecbatchgen.get()) c_sink_node = GetResultValue( MakeExecNode(tobytes("sink"), &deref(c_exec_plan), c_final_node_vec, deref(c_sinkopts)) ) @@ -74,29 +79,33 @@ cdef (shared_ptr[CAsyncExecBatchGenerator], shared_ptr[CExecPlan], shared_ptr[CR deref(c_exec_plan).Validate() deref(c_exec_plan).StartProducing() - return (c_asyncexecbatchgen, c_exec_plan, c_recordbatchreader) + if output_type == Table: + c_out_table = GetResultValue(CTable.FromRecordBatchReader(c_recordbatchreader.get())) + output = pyarrow_wrap_table(c_out_table) + else: + raise TypeError("Unsupported output type") + + deref(c_exec_plan).StopProducing() + + return output def test(): + # Currently run with + # python -c "__import__('pyarrow._exec_plan')._exec_plan.test()" cdef: - shared_ptr[CTable] c_table - shared_ptr[CExecPlan] c_exec_plan - shared_ptr[CAsyncExecBatchGenerator] c_asyncexecbatchgen - shared_ptr[CRecordBatchReader] c_recordbatchreader - shared_ptr[CSchema] c_schema + vector[CExpression] c_project_expr + vector[c_string] c_project_names + vector[CDeclaration] c_decl_plan t = Table.from_pydict({ "col1": [1, 2, 3, 4, 5], "col2": ["a", "b", "c", "d", "e"] }) - res = execplan(t) - c_asyncexecbatchgen = res[0] - c_exec_plan = res[1] - c_recordbatchreader = res[2] - - c_table = GetResultValue(CTable.FromRecordBatchReader(c_recordbatchreader.get())) - table = pyarrow_wrap_table(c_table) - - deref(c_exec_plan).StopProducing() + c_project_expr.push_back(CMakeFieldExpression(tobytes("col1"))) + c_decl_plan.push_back( + CDeclaration(tobytes("project"), CProjectNodeOptions(c_project_expr)) + ) + table = execplan(t, output_type=Table, plan=c_decl_plan) print(table) \ No newline at end of file diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index d359382be13f..2ac0c9ea2790 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2441,9 +2441,12 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil CResult[shared_ptr[CSourceNodeOptions]] FromTable(const CTable& table) cdef cppclass CSinkNodeOptions "arrow::compute::SinkNodeOptions"(CExecNodeOptions): - @staticmethod - pair[shared_ptr[CSinkNodeOptions], shared_ptr[CAsyncExecBatchGenerator]] MakeWithAsyncGenerator() + pass + cdef cppclass CProjectNodeOptions "arrow::compute::ProjectNodeOptions"(CExecNodeOptions): + CProjectNodeOptions(vector[CExpression] expressions) + CProjectNodeOptions(vector[CExpression] expressions, + vector[c_string] names) cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: cdef cppclass CDeclaration "arrow::compute::Declaration": From 81b5b58fa3fda769338c474fb6d36a0081d7adc1 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 15 Feb 2022 17:18:16 +0100 Subject: [PATCH 05/24] Add support for multiple sources --- python/pyarrow/_exec_plan.pyx | 69 +++++++++++++++++++--------- python/pyarrow/includes/libarrow.pxd | 14 ++++-- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index b89a1f7c1993..fb379ded96c7 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -29,14 +29,15 @@ from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) from pyarrow.lib import tobytes -cdef execplan(input_t, output_type, vector[CDeclaration] plan): + +cdef execplan(inputs, output_type, vector[CDeclaration] plan): cdef: CExecContext c_exec_context = CExecContext(c_default_memory_pool()) shared_ptr[CExecPlan] c_exec_plan = GetResultValue(CExecPlan.Make(&c_exec_context)) vector[CDeclaration] c_decls + vector[CExecNode*] _empty vector[CExecNode*] c_final_node_vec - CExecNode *c_final_node - CExecNode *c_sink_node + CExecNode *c_node CTable* c_table shared_ptr[CTable] c_out_table shared_ptr[CSourceNodeOptions] c_sourceopts @@ -45,42 +46,60 @@ cdef execplan(input_t, output_type, vector[CDeclaration] plan): shared_ptr[CRecordBatchReader] c_recordbatchreader vector[CDeclaration].iterator plan_iter - if isinstance(input_t, Table): - c_in_table = pyarrow_unwrap_table(input_t).get() - c_sourceopts = GetResultValue(CSourceNodeOptions.FromTable(deref(c_in_table))) - else: - raise ValueError("Unsupproted type") - - c_decls.push_back(CDeclaration(tobytes("source"), deref(c_sourceopts))) - c_decls[0].label = tobytes("source") - - # Add Here additional nodes plan_iter = plan.begin() + + # Create source nodes for each input + for idx, ipt in enumerate(inputs): + if isinstance(ipt, Table): + c_in_table = pyarrow_unwrap_table(ipt).get() + c_sourceopts = GetResultValue( + CSourceNodeOptions.FromTable(deref(c_in_table))) + else: + raise ValueError("Unsupproted type") + + if plan_iter != plan.end(): + # Flag the source as the input of the first plan node. + deref(plan_iter).inputs.push_back(CDeclaration.Input( + CDeclaration(tobytes("source"), deref(c_sourceopts)) + )) + else: + # Empty plan, make the source the first plan node. + c_decls.push_back( + CDeclaration(tobytes("source"), deref(c_sourceopts)) + ) + + # Add Here additional nodes while plan_iter != plan.end(): c_decls.push_back(deref(plan_iter)) inc(plan_iter) # Add all CDeclarations to the plan - c_final_node = GetResultValue( + c_node = GetResultValue( CDeclaration.Sequence(c_decls).AddToPlan(&deref(c_exec_plan)) ) - c_final_node_vec.push_back(c_final_node) + c_final_node_vec.push_back(c_node) + # Create the output node c_asyncexecbatchgen = make_shared[CAsyncExecBatchGenerator]() c_sinkopts = make_shared[CSinkNodeOptions](c_asyncexecbatchgen.get()) - c_sink_node = GetResultValue( - MakeExecNode(tobytes("sink"), &deref(c_exec_plan), c_final_node_vec, deref(c_sinkopts)) + GetResultValue( + MakeExecNode(tobytes("sink"), &deref(c_exec_plan), + c_final_node_vec, deref(c_sinkopts)) ) - c_recordbatchreader = MakeGeneratorReader(c_final_node.output_schema(), + # Convert the asyncgenerator to a sync batch reader + c_recordbatchreader = MakeGeneratorReader(c_node.output_schema(), deref(c_asyncexecbatchgen), c_exec_context.memory_pool()) + # Start execution of the ExecPlan deref(c_exec_plan).Validate() deref(c_exec_plan).StartProducing() + # Convert output to the expected one. if output_type == Table: - c_out_table = GetResultValue(CTable.FromRecordBatchReader(c_recordbatchreader.get())) + c_out_table = GetResultValue( + CTable.FromRecordBatchReader(c_recordbatchreader.get())) output = pyarrow_wrap_table(c_out_table) else: raise TypeError("Unsupported output type") @@ -89,6 +108,7 @@ cdef execplan(input_t, output_type, vector[CDeclaration] plan): return output + def test(): # Currently run with # python -c "__import__('pyarrow._exec_plan')._exec_plan.test()" @@ -101,11 +121,16 @@ def test(): "col1": [1, 2, 3, 4, 5], "col2": ["a", "b", "c", "d", "e"] }) - + + t2 = Table.from_pydict({ + "col1": [1, 2, 3, 4, 5], + "col2": ["a", "b", "c", "d", "e"] + }) + c_project_expr.push_back(CMakeFieldExpression(tobytes("col1"))) c_decl_plan.push_back( CDeclaration(tobytes("project"), CProjectNodeOptions(c_project_expr)) ) - table = execplan(t, output_type=Table, plan=c_decl_plan) + table = execplan([t], output_type=Table, plan=c_decl_plan) - print(table) \ No newline at end of file + print(table) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 2ac0c9ea2790..273dee72909c 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1796,7 +1796,7 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: cdef cppclass CExecContext" arrow::compute::ExecContext": CExecContext() CExecContext(CMemoryPool* pool) - + CMemoryPool* memory_pool() const cdef cppclass CKernelSignature" arrow::compute::KernelSignature": @@ -2435,7 +2435,7 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil cdef cppclass CExecNodeOptions "arrow::compute::ExecNodeOptions": pass - + cdef cppclass CSourceNodeOptions "arrow::compute::SourceNodeOptions"(CExecNodeOptions): @staticmethod CResult[shared_ptr[CSourceNodeOptions]] FromTable(const CTable& table) @@ -2450,7 +2450,13 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: cdef cppclass CDeclaration "arrow::compute::Declaration": + cppclass Input: + Input(CExecNode*) + Input(CDeclaration) + c_string label + vector[Input] inputs + CDeclaration(c_string factory_name, CExecNodeOptions options) @staticmethod @@ -2475,13 +2481,13 @@ cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nog cdef cppclass CExecBatch "arrow::compute::ExecBatch": pass - + shared_ptr[CRecordBatchReader] MakeGeneratorReader( shared_ptr[CSchema] schema, CAsyncExecBatchGenerator gen, CMemoryPool* memory_pool ) - CResult[CExecNode*] MakeExecNode(c_string factory_name, CExecPlan* plan, + CResult[CExecNode*] MakeExecNode(c_string factory_name, CExecPlan* plan, vector[CExecNode*] inputs, const CExecNodeOptions& options) From 721f00978f60899964e1de946a367610f1b66376 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 16 Feb 2022 12:29:21 +0100 Subject: [PATCH 06/24] table Join support --- cpp/src/arrow/compute/exec/options.cc | 17 ++-- cpp/src/arrow/compute/exec/options.h | 7 +- python/pyarrow/_exec_plan.pyx | 113 +++++++++++++++++++------ python/pyarrow/compute.py | 3 + python/pyarrow/includes/libarrow.pxd | 38 ++++++++- python/pyarrow/table.pxi | 29 +++++++ python/pyarrow/tests/test_exec_plan.py | 103 ++++++++++++++++++++++ python/pyarrow/tests/test_table.py | 28 ++++++ 8 files changed, 301 insertions(+), 37 deletions(-) create mode 100644 python/pyarrow/tests/test_exec_plan.py diff --git a/cpp/src/arrow/compute/exec/options.cc b/cpp/src/arrow/compute/exec/options.cc index 8c851ca4ccd2..c09ab1c1b68c 100644 --- a/cpp/src/arrow/compute/exec/options.cc +++ b/cpp/src/arrow/compute/exec/options.cc @@ -16,12 +16,11 @@ // under the License. #include "arrow/compute/exec/options.h" -#include "arrow/util/logging.h" #include "arrow/compute/exec/exec_plan.h" -#include "arrow/compute/exec/options.h" #include "arrow/io/util_internal.h" -#include "arrow/util/async_generator.h" #include "arrow/table.h" +#include "arrow/util/async_generator.h" +#include "arrow/util/logging.h" namespace arrow { namespace compute { @@ -49,15 +48,17 @@ std::string ToString(JoinType t) { std::abort(); } -Result> SourceNodeOptions::FromTable(const Table& table) { +Result> SourceNodeOptions::FromTable( + const Table& table, arrow::internal::Executor* exc) { std::shared_ptr reader = std::make_shared(table); + if (exc == nullptr) return Status::TypeError("No executor provided."); + // Map the RecordBatchReader to a SourceNode - ARROW_ASSIGN_OR_RAISE( - auto batch_gen, - MakeReaderGenerator(std::move(reader), arrow::io::internal::GetIOThreadPool())); + ARROW_ASSIGN_OR_RAISE(auto batch_gen, MakeReaderGenerator(std::move(reader), exc)); - return std::shared_ptr(new SourceNodeOptions(table.schema(), batch_gen)); + return std::shared_ptr( + new SourceNodeOptions(table.schema(), batch_gen)); } } // namespace compute diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index 654a5ab56a57..d5780753254b 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -26,11 +26,11 @@ #include "arrow/compute/api_vector.h" #include "arrow/compute/exec.h" #include "arrow/compute/exec/expression.h" +#include "arrow/result.h" +#include "arrow/util/async_generator.h" #include "arrow/util/async_util.h" #include "arrow/util/optional.h" #include "arrow/util/visibility.h" -#include "arrow/util/async_generator.h" -#include "arrow/result.h" namespace arrow { namespace compute { @@ -54,7 +54,8 @@ class ARROW_EXPORT SourceNodeOptions : public ExecNodeOptions { std::function>()> generator) : output_schema(std::move(output_schema)), generator(std::move(generator)) {} - static Result> FromTable(const Table& table); + static Result> FromTable(const Table& table, + arrow::internal::Executor*); std::shared_ptr output_schema; std::function>()> generator; diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index fb379ded96c7..fef306743d03 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -31,8 +31,24 @@ from pyarrow.lib import tobytes cdef execplan(inputs, output_type, vector[CDeclaration] plan): + """ + Internal Function to create and ExecPlan and run it. + + Parameters + ---------- + inputs : list of Table or Dataset + The sources from which the ExecPlan should fetch data. + In most cases this is only one, unless the first node of the + plan is able to get data from multiple different sources. + output_type : Table or Dataset + In which format the output should be provided. + plan : vector[CDeclaration] + The nodes of the plan that should be applied to the sources + to produce the output. + """ cdef: - CExecContext c_exec_context = CExecContext(c_default_memory_pool()) + CExecContext c_exec_context = CExecContext(c_default_memory_pool(), + GetCpuThreadPool()) shared_ptr[CExecPlan] c_exec_plan = GetResultValue(CExecPlan.Make(&c_exec_context)) vector[CDeclaration] c_decls vector[CExecNode*] _empty @@ -49,13 +65,13 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): plan_iter = plan.begin() # Create source nodes for each input - for idx, ipt in enumerate(inputs): + for ipt in inputs: if isinstance(ipt, Table): c_in_table = pyarrow_unwrap_table(ipt).get() c_sourceopts = GetResultValue( - CSourceNodeOptions.FromTable(deref(c_in_table))) + CSourceNodeOptions.FromTable(deref(c_in_table), c_exec_context.executor())) else: - raise ValueError("Unsupproted type") + raise TypeError("Unsupproted type") if plan_iter != plan.end(): # Flag the source as the input of the first plan node. @@ -79,7 +95,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): ) c_final_node_vec.push_back(c_node) - # Create the output node + # Create the output node c_asyncexecbatchgen = make_shared[CAsyncExecBatchGenerator]() c_sinkopts = make_shared[CSinkNodeOptions](c_asyncexecbatchgen.get()) GetResultValue( @@ -92,7 +108,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): deref(c_asyncexecbatchgen), c_exec_context.memory_pool()) - # Start execution of the ExecPlan + # Start execution of the ExecPlan deref(c_exec_plan).Validate() deref(c_exec_plan).StartProducing() @@ -109,28 +125,75 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): return output -def test(): - # Currently run with - # python -c "__import__('pyarrow._exec_plan')._exec_plan.test()" +def tables_join(join_type, left_table, left_keys, right_table, right_keys): + """ + Perform join of two tables. + + The result will be an output table with the result of the join operation + + Parameters + ---------- + join_type : str + One of supported join types. + left_table : Table + The left table for the join operation + left_keys : str or list[str] + The left table key (or keys) on which the join operation should be performed + right_table : Table + The right table for the join operation + right_keys : str or list[str] + The right table key (or keys) on which the join operation should be performed + + Returns + ------- + result_table : Table + """ cdef: - vector[CExpression] c_project_expr - vector[c_string] c_project_names + vector[CFieldRef] c_left_keys + vector[CFieldRef] c_right_keys vector[CDeclaration] c_decl_plan + CJoinType c_join_type + + # Prepare left and right tables Keys to send them to the C++ function + if isinstance(left_keys, str): + left_keys = [left_keys] + for key in left_keys: + c_left_keys.push_back(CFieldRef(tobytes(key))) + + if isinstance(right_keys, str): + right_keys = [right_keys] + for key in right_keys: + c_right_keys.push_back(CFieldRef(tobytes(key))) + + # Pick the join type + if join_type == "left semi": + c_join_type = CJoinType_LEFT_SEMI + elif join_type == "right semi": + c_join_type = CJoinType_RIGHT_SEMI + elif join_type == "left anti": + c_join_type = CJoinType_LEFT_ANTI + elif join_type == "right anti": + c_join_type = CJoinType_RIGHT_ANTI + elif join_type == "inner": + c_join_type = CJoinType_INNER + elif join_type == "left outer": + c_join_type = CJoinType_LEFT_OUTER + elif join_type == "right outer": + c_join_type = CJoinType_RIGHT_OUTER + elif join_type == "full outer": + c_join_type = CJoinType_FULL_OUTER + else: + raise ValueError("Unsupporter join type") - t = Table.from_pydict({ - "col1": [1, 2, 3, 4, 5], - "col2": ["a", "b", "c", "d", "e"] - }) - - t2 = Table.from_pydict({ - "col1": [1, 2, 3, 4, 5], - "col2": ["a", "b", "c", "d", "e"] - }) - - c_project_expr.push_back(CMakeFieldExpression(tobytes("col1"))) + # Add the join node to the execplan c_decl_plan.push_back( - CDeclaration(tobytes("project"), CProjectNodeOptions(c_project_expr)) + CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys + )) ) - table = execplan([t], output_type=Table, plan=c_decl_plan) - print(table) + result_table = execplan([left_table, right_table], + output_type=Table, + plan=c_decl_plan) + + return result_table diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index d66c3dd5df3a..dc8eac9f6268 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -79,6 +79,9 @@ # Expressions Expression, ) +from pyarrow._exec_plan import ( # noqa + tables_join +) from collections import namedtuple import inspect diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 273dee72909c..941613749c03 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1791,13 +1791,26 @@ cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil: CResult[shared_ptr[CTable]] Read() +cdef extern from "arrow/util/thread_pool.h" namespace "arrow::internal" nogil: + + cdef cppclass CExecutor "arrow::internal::Executor": + pass + + cdef cppclass CThreadPool "arrow::internal::ThreadPool"(CExecutor): + pass + + CThreadPool* GetCpuThreadPool() + + cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: cdef cppclass CExecContext" arrow::compute::ExecContext": CExecContext() CExecContext(CMemoryPool* pool) + CExecContext(CMemoryPool* pool, CExecutor* exc) CMemoryPool* memory_pool() const + CExecutor* executor() cdef cppclass CKernelSignature" arrow::compute::KernelSignature": c_string ToString() const @@ -2430,6 +2443,16 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil: + cdef enum CJoinType "arrow::compute::JoinType": + CJoinType_LEFT_SEMI "arrow::compute::JoinType::LEFT_SEMI" + CJoinType_RIGHT_SEMI "arrow::compute::JoinType::RIGHT_SEMI" + CJoinType_LEFT_ANTI "arrow::compute::JoinType::LEFT_ANTI" + CJoinType_RIGHT_ANTI "arrow::compute::JoinType::RIGHT_ANTI" + CJoinType_INNER "arrow::compute::JoinType::INNER" + CJoinType_LEFT_OUTER "arrow::compute::JoinType::LEFT_OUTER" + CJoinType_RIGHT_OUTER "arrow::compute::JoinType::RIGHT_OUTER" + CJoinType_FULL_OUTER "arrow::compute::JoinType::FULL_OUTER" + cdef cppclass CAsyncExecBatchGenerator "arrow::compute::AsyncExecBatchGenerator": pass @@ -2438,7 +2461,7 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil cdef cppclass CSourceNodeOptions "arrow::compute::SourceNodeOptions"(CExecNodeOptions): @staticmethod - CResult[shared_ptr[CSourceNodeOptions]] FromTable(const CTable& table) + CResult[shared_ptr[CSourceNodeOptions]] FromTable(const CTable& table, CExecutor*) cdef cppclass CSinkNodeOptions "arrow::compute::SinkNodeOptions"(CExecNodeOptions): pass @@ -2448,6 +2471,19 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil CProjectNodeOptions(vector[CExpression] expressions, vector[c_string] names) + cdef cppclass CHashJoinNodeOptions "arrow::compute::HashJoinNodeOptions"(CExecNodeOptions): + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys) + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys, + CExpression filter) + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys, + CExpression filter, + c_string output_prefix_for_left, + c_string output_prefix_for_right) + + cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: cdef cppclass CDeclaration "arrow::compute::Declaration": cppclass Input: diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index f93c20f49d46..e65b1de8709a 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2600,6 +2600,35 @@ cdef class Table(_PandasConvertible): return table + def join(self, keys, right_table, right_keys, join_type="left outer"): + """ + Perform a join between this table and another one. + + Result of the join will be a new Table, where further + operations can be applied. + + Parameters + ---------- + keys : str or list[str] + The columns from current table that should be used as keys + of the join operation left side. + right_table : Table + The table to join to the current one, acting as the right table + in the join operation. + right_keys : str or list[str] + The columns from the right_table that should be used as keys + on the join operation right side. + join_type : str, default "left outer" + The kind of join that should be performed, one of + ("left semi", "right semi", "left anti", "right anti", + "inner", "left outer", "right outer", "full outer") + + Returns + ------- + Table + """ + return _pc().tables_join(join_type, self, keys, right_table, right_keys) + def group_by(self, keys): """Declare a grouping over the columns of the table. diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py new file mode 100644 index 000000000000..bd88c3d7817b --- /dev/null +++ b/python/pyarrow/tests/test_exec_plan.py @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import pyarrow as pa +import pyarrow.compute as pc + + +def test_joins_corner_cases(): + t1 = pa.Table.from_pydict({ + "colA": [1, 2, 3, 4, 5, 6], + "col2": ["a", "b", "c", "d", "e", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [1, 2, 3, 4, 5], + "col3": ["A", "B", "C", "D", "E"] + }) + + with pytest.raises(pa.ArrowInvalid): + pc.tables_join("left outer", t1, "", t2, "") + + with pytest.raises(TypeError): + pc.tables_join("left outer", None, "colA", t2, "colB") + + with pytest.raises(ValueError): + pc.tables_join("super mario join", t1, "colA", t2, "colB") + + +@pytest.mark.parametrize("jointype,expected", [ + ("left semi", { + "colA": [1, 2], + "col2": ["a", "b"] + }), + ("right semi", { + "colB": [2, 1], + "col3": ["B", "A"] + }), + ("left anti", { + "colA": [6], + "col2": ["f"] + }), + ("right anti", { + "colB": [99], + "col3": ["Z"] + }), + ("inner", { + "colA": [1, 2], + "col2": ["a", "b"], + "colB": [1, 2], + "col3": ["A", "B"] + }), + ("left outer", { + "colA": [1, 2, 6], + "col2": ["a", "b", "f"], + "colB": [1, 2, None], + "col3": ["A", "B", None] + }), + ("right outer", { + "colA": [1, 2, None], + "col2": ["a", "b", None], + "colB": [1, 2, 99], + "col3": ["A", "B", "Z"] + }), + ("full outer", { + "colA": [1, 2, 6, None], + "col2": ["a", "b", "f", None], + "colB": [1, 2, None, 99], + "col3": ["A", "B", None, "Z"] + }) +]) +def test_joins(jointype, expected): + # Allocate table here instead of using parametrize + # this prevents having arrow allocated memory forever around. + expected = pa.table(expected) + + t1 = pa.Table.from_pydict({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [99, 2, 1], + "col3": ["Z", "B", "A"] + }) + + r = pc.tables_join(jointype, t1, "colA", t2, "colB") + assert r == expected diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 9080d14ebc6b..86f99716749d 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2043,3 +2043,31 @@ def test_table_to_recordbatchreader(): reader = table.to_reader(max_chunksize=2) assert reader.read_next_batch().num_rows == 2 assert reader.read_next_batch().num_rows == 1 + + +def test_table_join(): + t1 = pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colB": [99, 2, 1], + "col3": ["Z", "B", "A"] + }) + + result = t1.join("colA", t2, "colB") + assert result == pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"], + "colB": [1, 2, None], + "col3": ["A", "B", None] + }) + + result = t1.join("colA", t2, "colB", join_type="full outer") + assert result == pa.table({ + "colA": [1, 2, 6, None], + "col2": ["a", "b", "f", None], + "colB": [1, 2, None, 99], + "col3": ["A", "B", None, "Z"] + }) From f40d06628c98a7ce3336012de4cdd4c7310db75c Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 1 Mar 2022 11:32:15 +0100 Subject: [PATCH 07/24] Apply suggestions from code review Co-authored-by: Joris Van den Bossche --- python/pyarrow/_exec_plan.pyx | 94 ++++++++++++++++++++------ python/pyarrow/includes/libarrow.pxd | 15 +++- python/pyarrow/table.pxi | 22 ++++-- python/pyarrow/tests/test_exec_plan.py | 9 ++- python/pyarrow/tests/test_table.py | 28 +++++++- 5 files changed, 134 insertions(+), 34 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index fef306743d03..e49d419bc456 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -30,26 +30,29 @@ from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) from pyarrow.lib import tobytes -cdef execplan(inputs, output_type, vector[CDeclaration] plan): +cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads=True): """ - Internal Function to create and ExecPlan and run it. + Internal Function to create an ExecPlan and run it. Parameters ---------- inputs : list of Table or Dataset - The sources from which the ExecPlan should fetch data. - In most cases this is only one, unless the first node of the - plan is able to get data from multiple different sources. + The sources from which the ExecPlan should fetch data. + In most cases this is only one, unless the first node of the + plan is able to get data from multiple different sources. output_type : Table or Dataset - In which format the output should be provided. + In which format the output should be provided. plan : vector[CDeclaration] - The nodes of the plan that should be applied to the sources - to produce the output. + The nodes of the plan that should be applied to the sources + to produce the output. + use_threads : bool, default True + Whenever to use multithreading or not. """ cdef: - CExecContext c_exec_context = CExecContext(c_default_memory_pool(), - GetCpuThreadPool()) - shared_ptr[CExecPlan] c_exec_plan = GetResultValue(CExecPlan.Make(&c_exec_context)) + CExecutor *c_executor + shared_ptr[CThreadPool] c_executor_sptr + shared_ptr[CExecContext] c_exec_context + shared_ptr[CExecPlan] c_exec_plan vector[CDeclaration] c_decls vector[CExecNode*] _empty vector[CExecNode*] c_final_node_vec @@ -62,6 +65,16 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): shared_ptr[CRecordBatchReader] c_recordbatchreader vector[CDeclaration].iterator plan_iter + if use_threads: + c_executor = GetCpuThreadPool() + else: + c_executor_sptr = GetResultValue(CThreadPool.Make(1)) + c_executor = c_executor_sptr.get() + + c_exec_context = make_shared[CExecContext]( + c_default_memory_pool(), c_executor) + c_exec_plan = GetResultValue(CExecPlan.Make(c_exec_context.get())) + plan_iter = plan.begin() # Create source nodes for each input @@ -69,9 +82,9 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): if isinstance(ipt, Table): c_in_table = pyarrow_unwrap_table(ipt).get() c_sourceopts = GetResultValue( - CSourceNodeOptions.FromTable(deref(c_in_table), c_exec_context.executor())) + CSourceNodeOptions.FromTable(deref(c_in_table), deref(c_exec_context).executor())) else: - raise TypeError("Unsupproted type") + raise TypeError("Unsupported type") if plan_iter != plan.end(): # Flag the source as the input of the first plan node. @@ -106,7 +119,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): # Convert the asyncgenerator to a sync batch reader c_recordbatchreader = MakeGeneratorReader(c_node.output_schema(), deref(c_asyncexecbatchgen), - c_exec_context.memory_pool()) + deref(c_exec_context).memory_pool()) # Start execution of the ExecPlan deref(c_exec_plan).Validate() @@ -125,7 +138,15 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan): return output -def tables_join(join_type, left_table, left_keys, right_table, right_keys): +# Default filter for the JOIN operation, means to pick every row. +cdef CExpression _true = CMakeScalarExpression( + make_shared[CBooleanScalar](True) +) + + +def tables_join(join_type, left_table not None, left_keys, + right_table not None, right_keys, + left_suffix=None, right_suffix=None, use_threads=True): """ Perform join of two tables. @@ -136,13 +157,21 @@ def tables_join(join_type, left_table, left_keys, right_table, right_keys): join_type : str One of supported join types. left_table : Table - The left table for the join operation + The left table for the join operation. left_keys : str or list[str] - The left table key (or keys) on which the join operation should be performed + The left table key (or keys) on which the join operation should be performed. right_table : Table - The right table for the join operation + The right table for the join operation. right_keys : str or list[str] - The right table key (or keys) on which the join operation should be performed + The right table key (or keys) on which the join operation should be performed. + left_suffix : str, default None + Which suffix to add to right column names. This prevents confusion + when the columns in left and right tables have colliding names. + right_suffix : str, default None + Which suffic to add to the left column names. This prevents confusion + when the columns in left and right tables have colliding names. + use_threads : bool, default True + Whenever to use multithreading or not. Returns ------- @@ -151,6 +180,8 @@ def tables_join(join_type, left_table, left_keys, right_table, right_keys): cdef: vector[CFieldRef] c_left_keys vector[CFieldRef] c_right_keys + vector[CFieldRef] c_left_columns + vector[CFieldRef] c_right_columns vector[CDeclaration] c_decl_plan CJoinType c_join_type @@ -165,30 +196,51 @@ def tables_join(join_type, left_table, left_keys, right_table, right_keys): for key in right_keys: c_right_keys.push_back(CFieldRef(tobytes(key))) + # By default expose all columns on both left and right table + left_columns = left_table.column_names + right_columns = right_table.column_names + # Pick the join type if join_type == "left semi": c_join_type = CJoinType_LEFT_SEMI + right_columns = [] elif join_type == "right semi": c_join_type = CJoinType_RIGHT_SEMI + left_columns = [] elif join_type == "left anti": c_join_type = CJoinType_LEFT_ANTI + right_columns = [] elif join_type == "right anti": c_join_type = CJoinType_RIGHT_ANTI + left_columns = [] elif join_type == "inner": c_join_type = CJoinType_INNER + right_columns = set(right_columns) - set(right_keys) elif join_type == "left outer": c_join_type = CJoinType_LEFT_OUTER + right_columns = set(right_columns) - set(right_keys) elif join_type == "right outer": c_join_type = CJoinType_RIGHT_OUTER + left_columns = set(left_columns) - set(left_keys) elif join_type == "full outer": c_join_type = CJoinType_FULL_OUTER else: - raise ValueError("Unsupporter join type") + raise ValueError("Unsupported join type") + + # Turn the columns to vectors of FieldRefs + for colname in left_columns: + c_left_columns.push_back(CFieldRef(tobytes(colname))) + for colname in right_columns: + c_right_columns.push_back(CFieldRef(tobytes(colname))) # Add the join node to the execplan c_decl_plan.push_back( CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( - c_join_type, c_left_keys, c_right_keys + c_join_type, c_left_keys, c_right_keys, + c_left_columns, c_right_columns, + _true, + tobytes(left_suffix or ""), + tobytes(right_suffix or "") )) ) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 941613749c03..62a9f399d0fd 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1797,7 +1797,8 @@ cdef extern from "arrow/util/thread_pool.h" namespace "arrow::internal" nogil: pass cdef cppclass CThreadPool "arrow::internal::ThreadPool"(CExecutor): - pass + @staticmethod + CResult[shared_ptr[CThreadPool]] Make(int threads) CThreadPool* GetCpuThreadPool() @@ -2480,8 +2481,16 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, vector[CFieldRef] in_right_keys, CExpression filter, - c_string output_prefix_for_left, - c_string output_prefix_for_right) + c_string output_suffix_for_left, + c_string output_suffix_for_right) + CHashJoinNodeOptions(CJoinType join_type, + vector[CFieldRef] left_keys, + vector[CFieldRef] right_keys, + vector[CFieldRef] left_output, + vector[CFieldRef] right_output, + CExpression filter, + c_string output_suffix_for_left, + c_string output_suffix_for_right) cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index e65b1de8709a..1e46d6e4b4b5 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2600,7 +2600,8 @@ cdef class Table(_PandasConvertible): return table - def join(self, keys, right_table, right_keys, join_type="left outer"): + def join(self, keys, right_table, right_keys=None, join_type="left outer", + left_suffix=None, right_suffix=None, use_threads=True): """ Perform a join between this table and another one. @@ -2615,19 +2616,32 @@ cdef class Table(_PandasConvertible): right_table : Table The table to join to the current one, acting as the right table in the join operation. - right_keys : str or list[str] + right_keys : str or list[str], default None The columns from the right_table that should be used as keys - on the join operation right side. + on the join operation right side. + When ``None`` use the same key names as the left table. join_type : str, default "left outer" The kind of join that should be performed, one of ("left semi", "right semi", "left anti", "right anti", "inner", "left outer", "right outer", "full outer") + left_suffix : str, default None + Which suffix to add to right column names. This prevents confusion + when the columns in left and right tables have colliding names. + right_suffix : str, default None + Which suffic to add to the left column names. This prevents confusion + when the columns in left and right tables have colliding names. + use_threads : bool, default True + Whenever to use multithreading or not. Returns ------- Table """ - return _pc().tables_join(join_type, self, keys, right_table, right_keys) + if right_keys is None: + right_keys = keys + return _pc().tables_join(join_type, self, keys, right_table, right_keys, + left_suffix=left_suffix, right_suffix=right_suffix, + use_threads=use_threads) def group_by(self, keys): """Declare a grouping over the columns of the table. diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index bd88c3d7817b..85ca465e3aa3 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -62,17 +62,14 @@ def test_joins_corner_cases(): ("inner", { "colA": [1, 2], "col2": ["a", "b"], - "colB": [1, 2], "col3": ["A", "B"] }), ("left outer", { "colA": [1, 2, 6], "col2": ["a", "b", "f"], - "colB": [1, 2, None], "col3": ["A", "B", None] }), ("right outer", { - "colA": [1, 2, None], "col2": ["a", "b", None], "colB": [1, 2, 99], "col3": ["A", "B", "Z"] @@ -84,7 +81,8 @@ def test_joins_corner_cases(): "col3": ["A", "B", None, "Z"] }) ]) -def test_joins(jointype, expected): +@pytest.mark.parametrize("use_threads", [True, False]) +def test_joins(jointype, expected, use_threads): # Allocate table here instead of using parametrize # this prevents having arrow allocated memory forever around. expected = pa.table(expected) @@ -99,5 +97,6 @@ def test_joins(jointype, expected): "col3": ["Z", "B", "A"] }) - r = pc.tables_join(jointype, t1, "colA", t2, "colB") + r = pc.tables_join(jointype, t1, "colA", t2, "colB", + use_threads=use_threads) assert r == expected diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 86f99716749d..dc6ff6e6c1d3 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2060,7 +2060,6 @@ def test_table_join(): assert result == pa.table({ "colA": [1, 2, 6], "col2": ["a", "b", "f"], - "colB": [1, 2, None], "col3": ["A", "B", None] }) @@ -2071,3 +2070,30 @@ def test_table_join(): "colB": [1, 2, None, 99], "col3": ["A", "B", None, "Z"] }) + + +def test_table_join_unique_key(): + t1 = pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colA": [99, 2, 1], + "col3": ["Z", "B", "A"] + }) + + result = t1.join("colA", t2) + assert result == pa.table({ + "colA": [1, 2, 6], + "col2": ["a", "b", "f"], + "col3": ["A", "B", None] + }) + + result = t1.join("colA", t2, join_type="full outer", right_suffix="_r") + assert result == pa.table({ + "colA": [1, 2, 6, None], + "col2": ["a", "b", "f", None], + "colA_r": [1, 2, None, 99], + "col3": ["A", "B", None, "Z"] + }) From 1068ca708435279acb64d0c820cef8068cc0f59b Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 16 Mar 2022 16:47:18 +0100 Subject: [PATCH 08/24] Do not expose tables_join directly --- python/pyarrow/compute.py | 4 +--- python/pyarrow/table.pxi | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index dc8eac9f6268..460fa66d061a 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -79,9 +79,7 @@ # Expressions Expression, ) -from pyarrow._exec_plan import ( # noqa - tables_join -) +from pyarrow import _exec_plan from collections import namedtuple import inspect diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 1e46d6e4b4b5..9998eb5391ea 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2639,9 +2639,9 @@ cdef class Table(_PandasConvertible): """ if right_keys is None: right_keys = keys - return _pc().tables_join(join_type, self, keys, right_table, right_keys, - left_suffix=left_suffix, right_suffix=right_suffix, - use_threads=use_threads) + return _pc()._exec_plan.tables_join(join_type, self, keys, right_table, right_keys, + left_suffix=left_suffix, right_suffix=right_suffix, + use_threads=use_threads) def group_by(self, keys): """Declare a grouping over the columns of the table. From b08480a1a2466bc93228b4875ae5587f78257ce3 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 16 Mar 2022 17:26:27 +0100 Subject: [PATCH 09/24] Fix imports --- python/pyarrow/tests/test_exec_plan.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 85ca465e3aa3..6b415fed7044 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -18,7 +18,7 @@ import pytest import pyarrow as pa -import pyarrow.compute as pc +import pyarrow._exec_plan as ep def test_joins_corner_cases(): @@ -33,13 +33,13 @@ def test_joins_corner_cases(): }) with pytest.raises(pa.ArrowInvalid): - pc.tables_join("left outer", t1, "", t2, "") + ep.tables_join("left outer", t1, "", t2, "") with pytest.raises(TypeError): - pc.tables_join("left outer", None, "colA", t2, "colB") + ep.tables_join("left outer", None, "colA", t2, "colB") with pytest.raises(ValueError): - pc.tables_join("super mario join", t1, "colA", t2, "colB") + ep.tables_join("super mario join", t1, "colA", t2, "colB") @pytest.mark.parametrize("jointype,expected", [ @@ -97,6 +97,6 @@ def test_joins(jointype, expected, use_threads): "col3": ["Z", "B", "A"] }) - r = pc.tables_join(jointype, t1, "colA", t2, "colB", + r = ep.tables_join(jointype, t1, "colA", t2, "colB", use_threads=use_threads) assert r == expected From c867997b24256a280683e410f0d0bf2ff488d559 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 16 Mar 2022 18:06:40 +0100 Subject: [PATCH 10/24] fix linter --- python/pyarrow/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index 460fa66d061a..d27b0f10ef21 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -79,7 +79,7 @@ # Expressions Expression, ) -from pyarrow import _exec_plan +from pyarrow import _exec_plan # noqa from collections import namedtuple import inspect From e638ffa5b0ded6aa60a48c92347a1e0f88bfbf17 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 16 Mar 2022 18:07:10 +0100 Subject: [PATCH 11/24] coalesce keys in full outer join, needs more tests --- python/pyarrow/_exec_plan.pyx | 52 ++++++++++++++++++++------ python/pyarrow/includes/libarrow.pxd | 3 -- python/pyarrow/table.pxi | 2 +- python/pyarrow/tests/test_exec_plan.py | 5 +-- python/pyarrow/tests/test_table.py | 6 +-- 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index e49d419bc456..bfd5e9f8e467 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -27,7 +27,7 @@ from cython.operator cimport dereference as deref, preincrement as inc from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) -from pyarrow.lib import tobytes +from pyarrow.lib import tobytes, _pc cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads=True): @@ -146,7 +146,8 @@ cdef CExpression _true = CMakeScalarExpression( def tables_join(join_type, left_table not None, left_keys, right_table not None, right_keys, - left_suffix=None, right_suffix=None, use_threads=True): + left_suffix=None, right_suffix=None, + use_threads=True, deduplicate=False): """ Perform join of two tables. @@ -172,6 +173,9 @@ def tables_join(join_type, left_table not None, left_keys, when the columns in left and right tables have colliding names. use_threads : bool, default True Whenever to use multithreading or not. + deduplicate : bool, default False + If the duplicated keys should be omitted from one of the sides + in the join result. Returns ------- @@ -234,18 +238,44 @@ def tables_join(join_type, left_table not None, left_keys, c_right_columns.push_back(CFieldRef(tobytes(colname))) # Add the join node to the execplan - c_decl_plan.push_back( - CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( - c_join_type, c_left_keys, c_right_keys, - c_left_columns, c_right_columns, - _true, - tobytes(left_suffix or ""), - tobytes(right_suffix or "") - )) - ) + if deduplicate: + c_decl_plan.push_back( + CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys, + c_left_columns, c_right_columns, + _true, + tobytes(left_suffix or ""), + tobytes(right_suffix or "") + )) + ) + else: + c_decl_plan.push_back( + CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys, + _true, + tobytes(left_suffix or ""), + tobytes(right_suffix or "") + )) + ) result_table = execplan([left_table, right_table], output_type=Table, plan=c_decl_plan) + if deduplicate and join_type == "full outer": + suffixed_left_keys = left_keys + if left_suffix: + suffixed_left_keys = [ + lk+left_suffix for lk in left_keys if lk in right_keys] + suffixed_right_keys = right_keys + if right_suffix: + suffixed_right_keys = [ + rk+right_suffix for rk in right_keys if rk in left_keys] + for leftkey, rightkey in zip(reversed(suffixed_left_keys), reversed(suffixed_right_keys)): + result_table = result_table.set_column( + 0, leftkey, + _pc().coalesce(result_table[leftkey], result_table[rightkey]) + ) + result_table = result_table.drop(suffixed_right_keys) + return result_table diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 62a9f399d0fd..5dbcae4c9c4c 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2475,9 +2475,6 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil cdef cppclass CHashJoinNodeOptions "arrow::compute::HashJoinNodeOptions"(CExecNodeOptions): CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, vector[CFieldRef] in_right_keys) - CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, - vector[CFieldRef] in_right_keys, - CExpression filter) CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, vector[CFieldRef] in_right_keys, CExpression filter, diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 9998eb5391ea..5d2cd94d35f0 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2641,7 +2641,7 @@ cdef class Table(_PandasConvertible): right_keys = keys return _pc()._exec_plan.tables_join(join_type, self, keys, right_table, right_keys, left_suffix=left_suffix, right_suffix=right_suffix, - use_threads=use_threads) + use_threads=use_threads, deduplicate=True) def group_by(self, keys): """Declare a grouping over the columns of the table. diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 6b415fed7044..5a43b6f41b31 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -75,9 +75,8 @@ def test_joins_corner_cases(): "col3": ["A", "B", "Z"] }), ("full outer", { - "colA": [1, 2, 6, None], + "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], - "colB": [1, 2, None, 99], "col3": ["A", "B", None, "Z"] }) ]) @@ -98,5 +97,5 @@ def test_joins(jointype, expected, use_threads): }) r = ep.tables_join(jointype, t1, "colA", t2, "colB", - use_threads=use_threads) + use_threads=use_threads, deduplicate=True) assert r == expected diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index dc6ff6e6c1d3..988cb78f25a9 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2065,9 +2065,8 @@ def test_table_join(): result = t1.join("colA", t2, "colB", join_type="full outer") assert result == pa.table({ - "colA": [1, 2, 6, None], + "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], - "colB": [1, 2, None, 99], "col3": ["A", "B", None, "Z"] }) @@ -2092,8 +2091,7 @@ def test_table_join_unique_key(): result = t1.join("colA", t2, join_type="full outer", right_suffix="_r") assert result == pa.table({ - "colA": [1, 2, 6, None], + "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], - "colA_r": [1, 2, None, 99], "col3": ["A", "B", None, "Z"] }) From 2cd2b2a4bd43f51d5f4b1fae7d2e4d1106e682b5 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 18 Mar 2022 19:01:36 +0100 Subject: [PATCH 12/24] Use projection to coalesce join columns --- python/pyarrow/_compute.pxd | 3 +++ python/pyarrow/_compute.pyx | 7 ++++-- python/pyarrow/_exec_plan.pyx | 32 +++++++++++++++++++++----- python/pyarrow/includes/libarrow.pxd | 3 +++ python/pyarrow/tests/test_exec_plan.py | 26 ++++++++++++++++++++- python/pyarrow/tests/test_table.py | 31 +++++++++++++++++++++---- 6 files changed, 89 insertions(+), 13 deletions(-) diff --git a/python/pyarrow/_compute.pxd b/python/pyarrow/_compute.pxd index c10867a75b76..063b6318de49 100644 --- a/python/pyarrow/_compute.pxd +++ b/python/pyarrow/_compute.pxd @@ -49,3 +49,6 @@ cdef class Expression(_Weakrefable): @staticmethod cdef Expression _expr_or_scalar(object expr) + + +cdef CExpression _true \ No newline at end of file diff --git a/python/pyarrow/_compute.pyx b/python/pyarrow/_compute.pyx index 40fc3b4fff00..2f18ab99866c 100644 --- a/python/pyarrow/_compute.pyx +++ b/python/pyarrow/_compute.pyx @@ -2218,8 +2218,11 @@ cdef class Expression(_Weakrefable): return Expression._call("is_in", [self], options) @staticmethod - def _field(str name not None): - return Expression.wrap(CMakeFieldExpression(tobytes(name))) + def _field(name_or_idx not None): + if isinstance(name_or_idx, str): + return Expression.wrap(CMakeFieldExpression(tobytes(name_or_idx))) + else: + return Expression.wrap(CMakeFieldExpressionByIndex(name_or_idx)) @staticmethod def _scalar(value): diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index bfd5e9f8e467..1020d078c130 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -28,6 +28,7 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table) from pyarrow.lib import tobytes, _pc +from pyarrow._compute cimport Expression, _true cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads=True): @@ -138,12 +139,6 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads return output -# Default filter for the JOIN operation, means to pick every row. -cdef CExpression _true = CMakeScalarExpression( - make_shared[CBooleanScalar](True) -) - - def tables_join(join_type, left_table not None, left_keys, right_table not None, right_keys, left_suffix=None, right_suffix=None, @@ -187,6 +182,8 @@ def tables_join(join_type, left_table not None, left_keys, vector[CFieldRef] c_left_columns vector[CFieldRef] c_right_columns vector[CDeclaration] c_decl_plan + vector[CExpression] c_projections + vector[c_string] c_projected_col_names CJoinType c_join_type # Prepare left and right tables Keys to send them to the C++ function @@ -248,6 +245,29 @@ def tables_join(join_type, left_table not None, left_keys, tobytes(right_suffix or "") )) ) + if join_type == "full outer": + # In case of full outer joins, the join operation will output all columns + # so that we can coalesce the keys and exclude duplicates in a subsequent projection. + right_table_index = len(left_columns) + for idx, col in enumerate(left_columns + right_columns): + if idx < len(left_keys): + # Include keys only once and coalesce left+right table keys. + c_projected_col_names.push_back(tobytes(col)) + c_projections.push_back(Expression.unwrap( + Expression._call("coalesce", [ + Expression._field(idx), Expression._field(right_table_index+idx) + ]) + )) + elif right_table_index <= idx < right_table_index+len(right_keys): + # Do not include right table keys. As they would lead to duplicated keys. + continue + else: + # For all the other columns incude them as they are. + c_projected_col_names.push_back(tobytes(col)) + c_projections.push_back(Expression.unwrap(Expression._field(idx))) + c_decl_plan.push_back( + CDeclaration(tobytes("project"), CProjectNodeOptions(c_projections, c_projected_col_names)) + ) else: c_decl_plan.push_back( CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 5dbcae4c9c4c..6246a15fcff9 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2413,6 +2413,9 @@ cdef extern from "arrow/compute/exec/expression.h" \ cdef CExpression CMakeFieldExpression \ "arrow::compute::field_ref"(c_string name) + cdef CExpression CMakeFieldExpressionByIndex \ + "arrow::compute::field_ref"(int idx) + cdef CExpression CMakeCallExpression \ "arrow::compute::call"(c_string function, vector[CExpression] arguments, diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 5a43b6f41b31..c19634534712 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -98,4 +98,28 @@ def test_joins(jointype, expected, use_threads): r = ep.tables_join(jointype, t1, "colA", t2, "colB", use_threads=use_threads, deduplicate=True) - assert r == expected + assert r.combine_chunks() == expected + + +def test_table_join_all_columns(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colA": [99, 2, 1], + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"] + }) + + result = ep.tables_join("full outer", t1, ["colA", "colB"], t2, ["colA", "colB"]) + assert result.combine_chunks() == pa.table([ + [1, 2, 6, None], + [10, 20, 60, None], + ["a", "b", "f", None], + [1, 2, None, 99], + [10, 20, None, 99], + ["A", "B", None, "Z"], + ], names=["colA", "colB", "colVals", "colA", "colB", "colVals"]) \ No newline at end of file diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 988cb78f25a9..06320ade432a 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2057,14 +2057,14 @@ def test_table_join(): }) result = t1.join("colA", t2, "colB") - assert result == pa.table({ + assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6], "col2": ["a", "b", "f"], "col3": ["A", "B", None] }) result = t1.join("colA", t2, "colB", join_type="full outer") - assert result == pa.table({ + assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], "col3": ["A", "B", None, "Z"] @@ -2083,15 +2083,38 @@ def test_table_join_unique_key(): }) result = t1.join("colA", t2) - assert result == pa.table({ + assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6], "col2": ["a", "b", "f"], "col3": ["A", "B", None] }) result = t1.join("colA", t2, join_type="full outer", right_suffix="_r") - assert result == pa.table({ + assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], "col3": ["A", "B", None, "Z"] }) + + +def test_table_join_collisions(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colA": [99, 2, 1], + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"] + }) + + result = t1.join("colA", t2, join_type="full outer") + assert result.combine_chunks() == pa.table([ + [1, 2, 6, 99], + [10, 20, 60, None], + ["a", "b", "f", None], + [10, 20, None, 99], + ["A", "B", None, "Z"], + ], names=["colA", "colB", "colVals", "colB", "colVals"]) \ No newline at end of file From 3bb14c1da358dc9a9a69efcd2724e2548c83ee50 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 21 Mar 2022 12:06:04 +0100 Subject: [PATCH 13/24] Sort tests, to cover for unstable sorting of joins --- python/pyarrow/_exec_plan.pyx | 16 ---------------- python/pyarrow/tests/test_exec_plan.py | 11 ++++++++--- python/pyarrow/tests/test_table.py | 4 ++-- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 1020d078c130..46a48d26663c 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -282,20 +282,4 @@ def tables_join(join_type, left_table not None, left_keys, output_type=Table, plan=c_decl_plan) - if deduplicate and join_type == "full outer": - suffixed_left_keys = left_keys - if left_suffix: - suffixed_left_keys = [ - lk+left_suffix for lk in left_keys if lk in right_keys] - suffixed_right_keys = right_keys - if right_suffix: - suffixed_right_keys = [ - rk+right_suffix for rk in right_keys if rk in left_keys] - for leftkey, rightkey in zip(reversed(suffixed_left_keys), reversed(suffixed_right_keys)): - result_table = result_table.set_column( - 0, leftkey, - _pc().coalesce(result_table[leftkey], result_table[rightkey]) - ) - result_table = result_table.drop(suffixed_right_keys) - return result_table diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index c19634534712..f9780514f623 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -48,8 +48,8 @@ def test_joins_corner_cases(): "col2": ["a", "b"] }), ("right semi", { - "colB": [2, 1], - "col3": ["B", "A"] + "colB": [1, 2], + "col3": ["A", "B"] }), ("left anti", { "colA": [6], @@ -98,7 +98,12 @@ def test_joins(jointype, expected, use_threads): r = ep.tables_join(jointype, t1, "colA", t2, "colB", use_threads=use_threads, deduplicate=True) - assert r.combine_chunks() == expected + r = r.combine_chunks() + if "right" in jointype: + r = r.sort_by("colB") + else: + r = r.sort_by("colA") + assert r == expected def test_table_join_all_columns(): diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 06320ade432a..a953e00bd341 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2064,7 +2064,7 @@ def test_table_join(): }) result = t1.join("colA", t2, "colB", join_type="full outer") - assert result.combine_chunks() == pa.table({ + assert result.combine_chunks().sort_by("colA") == pa.table({ "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], "col3": ["A", "B", None, "Z"] @@ -2111,7 +2111,7 @@ def test_table_join_collisions(): }) result = t1.join("colA", t2, join_type="full outer") - assert result.combine_chunks() == pa.table([ + assert result.combine_chunks().sort_by("colA") == pa.table([ [1, 2, 6, 99], [10, 20, 60, None], ["a", "b", "f", None], From 3411322e5ae4fafc5a957ca66b8ad9f68cb44457 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 21 Mar 2022 12:28:48 +0100 Subject: [PATCH 14/24] Lint and reverse order of Table.join arguments --- python/pyarrow/_compute.pxd | 2 +- python/pyarrow/_exec_plan.pyx | 9 ++++++--- python/pyarrow/table.pxi | 8 ++++---- python/pyarrow/tests/test_exec_plan.py | 5 +++-- python/pyarrow/tests/test_table.py | 12 ++++++------ 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/python/pyarrow/_compute.pxd b/python/pyarrow/_compute.pxd index 063b6318de49..ebf2066d0466 100644 --- a/python/pyarrow/_compute.pxd +++ b/python/pyarrow/_compute.pxd @@ -51,4 +51,4 @@ cdef class Expression(_Weakrefable): cdef Expression _expr_or_scalar(object expr) -cdef CExpression _true \ No newline at end of file +cdef CExpression _true diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 46a48d26663c..b3744a799827 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -255,7 +255,8 @@ def tables_join(join_type, left_table not None, left_keys, c_projected_col_names.push_back(tobytes(col)) c_projections.push_back(Expression.unwrap( Expression._call("coalesce", [ - Expression._field(idx), Expression._field(right_table_index+idx) + Expression._field(idx), Expression._field( + right_table_index+idx) ]) )) elif right_table_index <= idx < right_table_index+len(right_keys): @@ -264,9 +265,11 @@ def tables_join(join_type, left_table not None, left_keys, else: # For all the other columns incude them as they are. c_projected_col_names.push_back(tobytes(col)) - c_projections.push_back(Expression.unwrap(Expression._field(idx))) + c_projections.push_back( + Expression.unwrap(Expression._field(idx))) c_decl_plan.push_back( - CDeclaration(tobytes("project"), CProjectNodeOptions(c_projections, c_projected_col_names)) + CDeclaration(tobytes("project"), CProjectNodeOptions( + c_projections, c_projected_col_names)) ) else: c_decl_plan.push_back( diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5d2cd94d35f0..68a12723644f 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2600,7 +2600,7 @@ cdef class Table(_PandasConvertible): return table - def join(self, keys, right_table, right_keys=None, join_type="left outer", + def join(self, right_table, keys, right_keys=None, join_type="left outer", left_suffix=None, right_suffix=None, use_threads=True): """ Perform a join between this table and another one. @@ -2610,12 +2610,12 @@ cdef class Table(_PandasConvertible): Parameters ---------- - keys : str or list[str] - The columns from current table that should be used as keys - of the join operation left side. right_table : Table The table to join to the current one, acting as the right table in the join operation. + keys : str or list[str] + The columns from current table that should be used as keys + of the join operation left side. right_keys : str or list[str], default None The columns from the right_table that should be used as keys on the join operation right side. diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index f9780514f623..430ba26132f4 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -119,7 +119,8 @@ def test_table_join_all_columns(): "colVals": ["Z", "B", "A"] }) - result = ep.tables_join("full outer", t1, ["colA", "colB"], t2, ["colA", "colB"]) + result = ep.tables_join( + "full outer", t1, ["colA", "colB"], t2, ["colA", "colB"]) assert result.combine_chunks() == pa.table([ [1, 2, 6, None], [10, 20, 60, None], @@ -127,4 +128,4 @@ def test_table_join_all_columns(): [1, 2, None, 99], [10, 20, None, 99], ["A", "B", None, "Z"], - ], names=["colA", "colB", "colVals", "colA", "colB", "colVals"]) \ No newline at end of file + ], names=["colA", "colB", "colVals", "colA", "colB", "colVals"]) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index a953e00bd341..34ad1837cf68 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2056,14 +2056,14 @@ def test_table_join(): "col3": ["Z", "B", "A"] }) - result = t1.join("colA", t2, "colB") + result = t1.join(t2, "colA", "colB") assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6], "col2": ["a", "b", "f"], "col3": ["A", "B", None] }) - result = t1.join("colA", t2, "colB", join_type="full outer") + result = t1.join(t2, "colA", "colB", join_type="full outer") assert result.combine_chunks().sort_by("colA") == pa.table({ "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], @@ -2082,14 +2082,14 @@ def test_table_join_unique_key(): "col3": ["Z", "B", "A"] }) - result = t1.join("colA", t2) + result = t1.join(t2, "colA") assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6], "col2": ["a", "b", "f"], "col3": ["A", "B", None] }) - result = t1.join("colA", t2, join_type="full outer", right_suffix="_r") + result = t1.join(t2, "colA", join_type="full outer", right_suffix="_r") assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], @@ -2110,11 +2110,11 @@ def test_table_join_collisions(): "colVals": ["Z", "B", "A"] }) - result = t1.join("colA", t2, join_type="full outer") + result = t1.join(t2, "colA", join_type="full outer") assert result.combine_chunks().sort_by("colA") == pa.table([ [1, 2, 6, 99], [10, 20, 60, None], ["a", "b", "f", None], [10, 20, None, 99], ["A", "B", None, "Z"], - ], names=["colA", "colB", "colVals", "colB", "colVals"]) \ No newline at end of file + ], names=["colA", "colB", "colVals", "colB", "colVals"]) From 23e11c173fb68dcc9ce5673ec1fd590e2b0b736e Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 22 Mar 2022 12:35:04 +0100 Subject: [PATCH 15/24] Ensure proper suffixes --- python/pyarrow/_exec_plan.pyx | 8 +++++++ python/pyarrow/tests/test_exec_plan.py | 29 +++++++++++++++++++++++--- python/pyarrow/tests/test_table.py | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index b3744a799827..d645f2ee9273 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -245,6 +245,8 @@ def tables_join(join_type, left_table not None, left_keys, tobytes(right_suffix or "") )) ) + left_columns_set = set(left_columns) + right_columns_set = set(right_columns) if join_type == "full outer": # In case of full outer joins, the join operation will output all columns # so that we can coalesce the keys and exclude duplicates in a subsequent projection. @@ -264,6 +266,12 @@ def tables_join(join_type, left_table not None, left_keys, continue else: # For all the other columns incude them as they are. + # Just recompute the suffixes that the join produced as the projection + # would lose them otherwise. + if left_suffix and idx < right_table_index and col in right_columns_set: + col += left_suffix + if right_suffix and idx >= right_table_index and col in left_columns_set: + col += right_suffix c_projected_col_names.push_back(tobytes(col)) c_projections.push_back( Expression.unwrap(Expression._field(idx))) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 430ba26132f4..66af7a0285f4 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -106,7 +106,7 @@ def test_joins(jointype, expected, use_threads): assert r == expected -def test_table_join_all_columns(): +def test_table_join_collisions(): t1 = pa.table({ "colA": [1, 2, 6], "colB": [10, 20, 60], @@ -116,7 +116,8 @@ def test_table_join_all_columns(): t2 = pa.table({ "colA": [99, 2, 1], "colB": [99, 20, 10], - "colVals": ["Z", "B", "A"] + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300] }) result = ep.tables_join( @@ -128,4 +129,26 @@ def test_table_join_all_columns(): [1, 2, None, 99], [10, 20, None, 99], ["A", "B", None, "Z"], - ], names=["colA", "colB", "colVals", "colA", "colB", "colVals"]) + [300, 200, None, 100] + ], names=["colA", "colB", "colVals", "colA", "colB", "colVals", "colUniq"]) + + result = ep.tables_join("full outer", t1, "colA", t2, "colA", right_suffix="_r", deduplicate=False) + assert result.combine_chunks() == pa.table({ + "colA": [1, 2, 6, None], + "colB": [10, 20, 60, None], + "colVals": ["a", "b", "f", None], + "colA_r": [1, 2, None, 99], + "colB_r": [10, 20, None, 99], + "colVals_r": ["A", "B", None, "Z"], + "colUniq": [300, 200, None, 100] + }) + + result = ep.tables_join("full outer", t1, "colA", t2, "colA", right_suffix="_r", deduplicate=True) + assert result.combine_chunks() == pa.table({ + "colA": [1, 2, 6, 99], + "colB": [10, 20, 60, None], + "colVals": ["a", "b", "f", None], + "colB_r": [10, 20, None, 99], + "colVals_r": ["A", "B", None, "Z"], + "colUniq": [300, 200, None, 100] + }) \ No newline at end of file diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 34ad1837cf68..0bcf715b91f5 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2090,7 +2090,7 @@ def test_table_join_unique_key(): }) result = t1.join(t2, "colA", join_type="full outer", right_suffix="_r") - assert result.combine_chunks() == pa.table({ + assert result.combine_chunks().sort_by("colA") == pa.table({ "colA": [1, 2, 6, 99], "col2": ["a", "b", "f", None], "col3": ["A", "B", None, "Z"] From f0c44c16a3f86064f1b6a2c997e5ff13908ba6cd Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 22 Mar 2022 12:35:52 +0100 Subject: [PATCH 16/24] lint --- python/pyarrow/tests/test_exec_plan.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 66af7a0285f4..cfaaa7735ec9 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -132,7 +132,8 @@ def test_table_join_collisions(): [300, 200, None, 100] ], names=["colA", "colB", "colVals", "colA", "colB", "colVals", "colUniq"]) - result = ep.tables_join("full outer", t1, "colA", t2, "colA", right_suffix="_r", deduplicate=False) + result = ep.tables_join("full outer", t1, "colA", + t2, "colA", right_suffix="_r", deduplicate=False) assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, None], "colB": [10, 20, 60, None], @@ -143,7 +144,8 @@ def test_table_join_collisions(): "colUniq": [300, 200, None, 100] }) - result = ep.tables_join("full outer", t1, "colA", t2, "colA", right_suffix="_r", deduplicate=True) + result = ep.tables_join("full outer", t1, "colA", + t2, "colA", right_suffix="_r", deduplicate=True) assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, 99], "colB": [10, 20, 60, None], @@ -151,4 +153,4 @@ def test_table_join_collisions(): "colB_r": [10, 20, None, 99], "colVals_r": ["A", "B", None, "Z"], "colUniq": [300, 200, None, 100] - }) \ No newline at end of file + }) From f5c06c4bdce921dcc091b97c8404c039337fa60e Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 22 Mar 2022 12:41:12 +0100 Subject: [PATCH 17/24] tweaks --- python/pyarrow/_exec_plan.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index d645f2ee9273..2ae5e22dc451 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -245,12 +245,12 @@ def tables_join(join_type, left_table not None, left_keys, tobytes(right_suffix or "") )) ) - left_columns_set = set(left_columns) - right_columns_set = set(right_columns) if join_type == "full outer": # In case of full outer joins, the join operation will output all columns # so that we can coalesce the keys and exclude duplicates in a subsequent projection. - right_table_index = len(left_columns) + left_columns_set = set(left_columns) + right_columns_set = set(right_columns) + right_table_index = len(left_columns) # Where the right table columns start. for idx, col in enumerate(left_columns + right_columns): if idx < len(left_keys): # Include keys only once and coalesce left+right table keys. From 7ffe222321a30d724edf49c198730d8cdfbf906e Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 22 Mar 2022 13:44:43 +0100 Subject: [PATCH 18/24] lint --- python/pyarrow/_exec_plan.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 2ae5e22dc451..f624c0eb2350 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -250,7 +250,8 @@ def tables_join(join_type, left_table not None, left_keys, # so that we can coalesce the keys and exclude duplicates in a subsequent projection. left_columns_set = set(left_columns) right_columns_set = set(right_columns) - right_table_index = len(left_columns) # Where the right table columns start. + # Where the right table columns start. + right_table_index = len(left_columns) for idx, col in enumerate(left_columns + right_columns): if idx < len(left_keys): # Include keys only once and coalesce left+right table keys. From 8160c6b8f289317d8ebf1710bec9a596cd936433 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 22 Mar 2022 16:26:28 +0100 Subject: [PATCH 19/24] expose the deduplicate option to Table.join too --- python/pyarrow/table.pxi | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 68a12723644f..79e227d9085a 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2601,7 +2601,8 @@ cdef class Table(_PandasConvertible): return table def join(self, right_table, keys, right_keys=None, join_type="left outer", - left_suffix=None, right_suffix=None, use_threads=True): + left_suffix=None, right_suffix=None, deduplicate=True, + use_threads=True): """ Perform a join between this table and another one. @@ -2630,6 +2631,9 @@ cdef class Table(_PandasConvertible): right_suffix : str, default None Which suffic to add to the left column names. This prevents confusion when the columns in left and right tables have colliding names. + deduplicate : bool, default True + If the duplicated keys should be omitted from one of the sides + in the join result. use_threads : bool, default True Whenever to use multithreading or not. @@ -2641,7 +2645,7 @@ cdef class Table(_PandasConvertible): right_keys = keys return _pc()._exec_plan.tables_join(join_type, self, keys, right_table, right_keys, left_suffix=left_suffix, right_suffix=right_suffix, - use_threads=use_threads, deduplicate=True) + use_threads=use_threads, deduplicate=deduplicate) def group_by(self, keys): """Declare a grouping over the columns of the table. From 9233b25299a3cc0b1cb3b2d65f98d1191b3bfcf6 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Tue, 22 Mar 2022 16:33:38 +0100 Subject: [PATCH 20/24] Update docstring --- python/pyarrow/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index d27b0f10ef21..0da61dcec0a7 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -592,22 +592,22 @@ def bottom_k_unstable(values, k, sort_keys=None, *, memory_pool=None): return call_function("select_k_unstable", [values], options, memory_pool) -def field(name): - """Reference a named column of the dataset. +def field(name_or_index): + """Reference a column of the dataset. Stores only the field's name. Type and other information is known only when the expression is bound to a dataset having an explicit scheme. Parameters ---------- - name : string - The name of the field the expression references to. + name_or_index : string or int + The name or index of the field the expression references to. Returns ------- field_expr : Expression """ - return Expression._field(name) + return Expression._field(name_or_index) def scalar(value): From e07d227407c565ef56c70e7c3fce33f61b5d2ddf Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 23 Mar 2022 14:27:16 +0100 Subject: [PATCH 21/24] Update python/pyarrow/_exec_plan.pyx Co-authored-by: Joris Van den Bossche --- python/pyarrow/_exec_plan.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index f624c0eb2350..086ed880a69a 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -110,7 +110,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads c_final_node_vec.push_back(c_node) # Create the output node - c_asyncexecbatchgen = make_shared[CAsyncExecBatchGenerator]() + c_async_exec_batch_gen = make_shared[CAsyncExecBatchGenerator]() c_sinkopts = make_shared[CSinkNodeOptions](c_asyncexecbatchgen.get()) GetResultValue( MakeExecNode(tobytes("sink"), &deref(c_exec_plan), From 3c3fa208ae6e11df633dfc8a5f3802b5f3d47ebf Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 23 Mar 2022 14:28:23 +0100 Subject: [PATCH 22/24] rename c_async_exec_batch_gen --- python/pyarrow/_exec_plan.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 086ed880a69a..b75a8ef4af9e 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -62,7 +62,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads shared_ptr[CTable] c_out_table shared_ptr[CSourceNodeOptions] c_sourceopts shared_ptr[CSinkNodeOptions] c_sinkopts - shared_ptr[CAsyncExecBatchGenerator] c_asyncexecbatchgen + shared_ptr[CAsyncExecBatchGenerator] c_async_exec_batch_gen shared_ptr[CRecordBatchReader] c_recordbatchreader vector[CDeclaration].iterator plan_iter @@ -111,7 +111,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads # Create the output node c_async_exec_batch_gen = make_shared[CAsyncExecBatchGenerator]() - c_sinkopts = make_shared[CSinkNodeOptions](c_asyncexecbatchgen.get()) + c_sinkopts = make_shared[CSinkNodeOptions](c_async_exec_batch_gen.get()) GetResultValue( MakeExecNode(tobytes("sink"), &deref(c_exec_plan), c_final_node_vec, deref(c_sinkopts)) @@ -119,7 +119,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads # Convert the asyncgenerator to a sync batch reader c_recordbatchreader = MakeGeneratorReader(c_node.output_schema(), - deref(c_asyncexecbatchgen), + deref(c_async_exec_batch_gen), deref(c_exec_context).memory_pool()) # Start execution of the ExecPlan From 8fd21e5fc506fafedd773ca2f8f4e2cc057c7bd4 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 23 Mar 2022 17:53:12 +0100 Subject: [PATCH 23/24] Fix bug in keys pairing logic --- python/pyarrow/_exec_plan.pyx | 30 ++++++++++++++++----- python/pyarrow/tests/test_exec_plan.py | 37 +++++++++++++++++++++----- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index b75a8ef4af9e..98168d15e120 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -187,14 +187,18 @@ def tables_join(join_type, left_table not None, left_keys, CJoinType c_join_type # Prepare left and right tables Keys to send them to the C++ function + left_keys_order = {} if isinstance(left_keys, str): left_keys = [left_keys] - for key in left_keys: + for idx, key in enumerate(left_keys): + left_keys_order[key] = idx c_left_keys.push_back(CFieldRef(tobytes(key))) + right_keys_order = {} if isinstance(right_keys, str): right_keys = [right_keys] - for key in right_keys: + for idx, key in enumerate(right_keys): + right_keys_order[key] = idx c_right_keys.push_back(CFieldRef(tobytes(key))) # By default expose all columns on both left and right table @@ -229,10 +233,17 @@ def tables_join(join_type, left_table not None, left_keys, raise ValueError("Unsupported join type") # Turn the columns to vectors of FieldRefs - for colname in left_columns: + # and set aside indices of keys. + left_column_keys_indices = {} + for idx, colname in enumerate(left_columns): c_left_columns.push_back(CFieldRef(tobytes(colname))) - for colname in right_columns: + if colname in left_keys: + left_column_keys_indices[colname] = idx + right_column_keys_indices = {} + for idx, colname in enumerate(right_columns): c_right_columns.push_back(CFieldRef(tobytes(colname))) + if colname in right_keys: + right_column_keys_indices[colname] = idx # Add the join node to the execplan if deduplicate: @@ -253,16 +264,21 @@ def tables_join(join_type, left_table not None, left_keys, # Where the right table columns start. right_table_index = len(left_columns) for idx, col in enumerate(left_columns + right_columns): - if idx < len(left_keys): + if idx < len(left_columns) and col in left_column_keys_indices: # Include keys only once and coalesce left+right table keys. c_projected_col_names.push_back(tobytes(col)) + # Get the index of the right key that is being paired + # with this left key. We do so by retrieving the name + # of the right key that is in the same position in the provided keys + # and then looking up the index for that name in the right table. + right_key_index = right_column_keys_indices[right_keys[left_keys_order[col]]] c_projections.push_back(Expression.unwrap( Expression._call("coalesce", [ Expression._field(idx), Expression._field( - right_table_index+idx) + right_table_index+right_key_index) ]) )) - elif right_table_index <= idx < right_table_index+len(right_keys): + elif idx >= right_table_index and col in right_column_keys_indices: # Do not include right table keys. As they would lead to duplicated keys. continue else: diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index cfaaa7735ec9..b864765d58d2 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -114,10 +114,10 @@ def test_table_join_collisions(): }) t2 = pa.table({ - "colA": [99, 2, 1], "colB": [99, 20, 10], "colVals": ["Z", "B", "A"], - "colUniq": [100, 200, 300] + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], }) result = ep.tables_join( @@ -126,11 +126,11 @@ def test_table_join_collisions(): [1, 2, 6, None], [10, 20, 60, None], ["a", "b", "f", None], - [1, 2, None, 99], [10, 20, None, 99], ["A", "B", None, "Z"], - [300, 200, None, 100] - ], names=["colA", "colB", "colVals", "colA", "colB", "colVals", "colUniq"]) + [300, 200, None, 100], + [1, 2, None, 99], + ], names=["colA", "colB", "colVals", "colB", "colVals", "colUniq", "colA"]) result = ep.tables_join("full outer", t1, "colA", t2, "colA", right_suffix="_r", deduplicate=False) @@ -138,10 +138,10 @@ def test_table_join_collisions(): "colA": [1, 2, 6, None], "colB": [10, 20, 60, None], "colVals": ["a", "b", "f", None], - "colA_r": [1, 2, None, 99], "colB_r": [10, 20, None, 99], "colVals_r": ["A", "B", None, "Z"], - "colUniq": [300, 200, None, 100] + "colUniq": [300, 200, None, 100], + "colA_r": [1, 2, None, 99], }) result = ep.tables_join("full outer", t1, "colA", @@ -154,3 +154,26 @@ def test_table_join_collisions(): "colVals_r": ["A", "B", None, "Z"], "colUniq": [300, 200, None, 100] }) + + +def test_table_join_keys_order(): + t1 = pa.table({ + "colB": [10, 20, 60], + "colA": [1, 2, 6], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colX": [99, 2, 1], + }) + + result = ep.tables_join("full outer", t1, "colA", t2, "colX", + left_suffix="_l", right_suffix="_r", + deduplicate=True) + assert result.combine_chunks() == pa.table({ + "colB": [10, 20, 60, None], + "colA": [1, 2, 6, 99], + "colVals_l": ["a", "b", "f", None], + "colVals_r": ["A", "B", None, "Z"], + }) From b43f3113bfaf4e2b58e758436c03f4c036478bc9 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 25 Mar 2022 12:26:08 +0100 Subject: [PATCH 24/24] Rename deduplicate to coalesce_keys --- python/pyarrow/_exec_plan.pyx | 6 +++--- python/pyarrow/table.pxi | 6 +++--- python/pyarrow/tests/test_exec_plan.py | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 98168d15e120..350409291526 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -142,7 +142,7 @@ cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads def tables_join(join_type, left_table not None, left_keys, right_table not None, right_keys, left_suffix=None, right_suffix=None, - use_threads=True, deduplicate=False): + use_threads=True, coalesce_keys=False): """ Perform join of two tables. @@ -168,7 +168,7 @@ def tables_join(join_type, left_table not None, left_keys, when the columns in left and right tables have colliding names. use_threads : bool, default True Whenever to use multithreading or not. - deduplicate : bool, default False + coalesce_keys : bool, default False If the duplicated keys should be omitted from one of the sides in the join result. @@ -246,7 +246,7 @@ def tables_join(join_type, left_table not None, left_keys, right_column_keys_indices[colname] = idx # Add the join node to the execplan - if deduplicate: + if coalesce_keys: c_decl_plan.push_back( CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( c_join_type, c_left_keys, c_right_keys, diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 79e227d9085a..af3d3a340ffe 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2601,7 +2601,7 @@ cdef class Table(_PandasConvertible): return table def join(self, right_table, keys, right_keys=None, join_type="left outer", - left_suffix=None, right_suffix=None, deduplicate=True, + left_suffix=None, right_suffix=None, coalesce_keys=True, use_threads=True): """ Perform a join between this table and another one. @@ -2631,7 +2631,7 @@ cdef class Table(_PandasConvertible): right_suffix : str, default None Which suffic to add to the left column names. This prevents confusion when the columns in left and right tables have colliding names. - deduplicate : bool, default True + coalesce_keys : bool, default True If the duplicated keys should be omitted from one of the sides in the join result. use_threads : bool, default True @@ -2645,7 +2645,7 @@ cdef class Table(_PandasConvertible): right_keys = keys return _pc()._exec_plan.tables_join(join_type, self, keys, right_table, right_keys, left_suffix=left_suffix, right_suffix=right_suffix, - use_threads=use_threads, deduplicate=deduplicate) + use_threads=use_threads, coalesce_keys=coalesce_keys) def group_by(self, keys): """Declare a grouping over the columns of the table. diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index b864765d58d2..8111660b027a 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -97,7 +97,7 @@ def test_joins(jointype, expected, use_threads): }) r = ep.tables_join(jointype, t1, "colA", t2, "colB", - use_threads=use_threads, deduplicate=True) + use_threads=use_threads, coalesce_keys=True) r = r.combine_chunks() if "right" in jointype: r = r.sort_by("colB") @@ -133,7 +133,7 @@ def test_table_join_collisions(): ], names=["colA", "colB", "colVals", "colB", "colVals", "colUniq", "colA"]) result = ep.tables_join("full outer", t1, "colA", - t2, "colA", right_suffix="_r", deduplicate=False) + t2, "colA", right_suffix="_r", coalesce_keys=False) assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, None], "colB": [10, 20, 60, None], @@ -145,7 +145,7 @@ def test_table_join_collisions(): }) result = ep.tables_join("full outer", t1, "colA", - t2, "colA", right_suffix="_r", deduplicate=True) + t2, "colA", right_suffix="_r", coalesce_keys=True) assert result.combine_chunks() == pa.table({ "colA": [1, 2, 6, 99], "colB": [10, 20, 60, None], @@ -170,7 +170,7 @@ def test_table_join_keys_order(): result = ep.tables_join("full outer", t1, "colA", t2, "colX", left_suffix="_l", right_suffix="_r", - deduplicate=True) + coalesce_keys=True) assert result.combine_chunks() == pa.table({ "colB": [10, 20, 60, None], "colA": [1, 2, 6, 99],