From 93f4359185e3f484eedc81ba929edd8f809c19b6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 9 Feb 2023 15:55:34 +0100 Subject: [PATCH 01/10] GH-33976: [Python] Initial bindings for acero Declaration and ExecNodeOptions classes --- python/CMakeLists.txt | 1 + python/pyarrow/_acero.pxd | 44 ++++++ python/pyarrow/_acero.pyx | 213 +++++++++++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 11 ++ python/pyarrow/tests/test_acero.py | 116 +++++++++++++++ python/setup.py | 1 + 6 files changed, 386 insertions(+) create mode 100644 python/pyarrow/_acero.pxd create mode 100644 python/pyarrow/_acero.pyx create mode 100644 python/pyarrow/tests/test_acero.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a2cf8901bac4..8f846348f352 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -525,6 +525,7 @@ set(CYTHON_EXTENSIONS _compute _csv _exec_plan + _acero _feather _fs _hdfsio diff --git a/python/pyarrow/_acero.pxd b/python/pyarrow/_acero.pxd new file mode 100644 index 000000000000..32133596663f --- /dev/null +++ b/python/pyarrow/_acero.pxd @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +from pyarrow.lib cimport * +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * + + +cdef class ExecNodeOptions(_Weakrefable): + cdef: + shared_ptr[CExecNodeOptions] wrapped + + cdef const CExecNodeOptions* get_options(self) except NULL + cdef void init(self, const shared_ptr[CExecNodeOptions]& sp) + cdef inline shared_ptr[CExecNodeOptions] unwrap(self) + + +cdef class Declaration(_Weakrefable): + + cdef: + CDeclaration decl + + cdef void init(self, const CDeclaration& c_decl) + + @staticmethod + cdef wrap(const CDeclaration& c_decl) + + cdef inline CDeclaration unwrap(self) nogil diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx new file mode 100644 index 000000000000..3cc75128ba89 --- /dev/null +++ b/python/pyarrow/_acero.pyx @@ -0,0 +1,213 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# --------------------------------------------------------------------- +# Low-level Acero bindings + +# cython: profile=False +# distutils: language = c++ +# cython: language_level = 3 + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_dataset cimport * +from pyarrow.lib cimport (Table, check_status, pyarrow_unwrap_table, pyarrow_wrap_table, + RecordBatchReader) +from pyarrow.lib import frombytes, tobytes +from pyarrow._compute cimport Expression + +# Initialize() # Initialise support for Datasets in ExecPlan + + +cdef class ExecNodeOptions(_Weakrefable): + __slots__ = () # avoid mistakingly creating attributes + + cdef const CExecNodeOptions* get_options(self) except NULL: + return self.wrapped.get() + + cdef void init(self, const shared_ptr[CExecNodeOptions]& sp): + self.wrapped = sp + + cdef inline shared_ptr[CExecNodeOptions] unwrap(self): + return self.wrapped + + # def __repr__(self): + # type_name = self.__class__.__name__ + # # Remove {} so we can use our own braces + # string_repr = frombytes(self.get_options().ToString())[1:-1] + # return f"{type_name}({string_repr})" + + # def __eq__(self, FunctionOptions other): + # return self.get_options().Equals(deref(other.get_options())) + + +cdef class _TableSourceNodeOptions(ExecNodeOptions): + + def _set_options(self, Table table): + cdef: + shared_ptr[CTable] c_table + + c_table = pyarrow_unwrap_table(table) + self.wrapped.reset( + new CTableSourceNodeOptions(c_table) + ) + + +class TableSourceNodeOptions(_TableSourceNodeOptions): + """ + A Source node which accepts a table. + + Parameters + ---------- + table : pyarrow.Table + The table which acts as the data source. + """ + def __init__(self, Table table): + self._set_options(table) + + +cdef class _FilterNodeOptions(ExecNodeOptions): + + def _set_options(self, Expression filter_expression not None): + self.wrapped.reset( + new CFilterNodeOptions(filter_expression.unwrap()) + ) + + +class FilterNodeOptions(_FilterNodeOptions): + """ + Make a node which excludes some rows from batches passed through it. + + The "filter" operation provides an option to define data filtering + criteria. It selects rows matching a given expression. Filters can + be written using pyarrow.compute.Expression. + + Parameters + ---------- + filter_expression : pyarrow.compute.Expression + + """ + def __init__(self, Expression filter_expression): + self._set_options(filter_expression) + + +cdef class _ProjectNodeOptions(ExecNodeOptions): + + def _set_options(self, expressions, names=None): + cdef: + Expression expr + vector[CExpression] c_expressions + vector[c_string] c_names + + for expr in expressions: + c_expressions.push_back(expr.unwrap()) + + if names is not None: + if len(names) != len(expressions): + raise ValueError("dd") + + for name in names: + c_names.push_back(tobytes(name)) + + self.wrapped.reset( + new CProjectNodeOptions(c_expressions, c_names) + ) + else: + self.wrapped.reset( + new CProjectNodeOptions(c_expressions) + ) + + +class ProjectNodeOptions(_ProjectNodeOptions): + """ + Make a node which executes expressions on input batches, + producing new batches. + + The "project" operation rearranges, deletes, transforms, and + creates columns. Each output column is computed by evaluating + an expression against the source record batch. + + Parameters + ---------- + expressions : list of pyarrow.compute.Expression + List of expressions to evaluate against the source batch. + names : list of str + List of names for each of the ouptut columns (same length as + `expressions`). If `names` is not provided, the string + representations of exprs will be used. + """ + def __init__(self, expressions, names=None): + self._set_options(expressions, names) + + +cdef class Declaration(_Weakrefable): + """ + + """ + cdef void init(self, const CDeclaration& c_decl): + self.decl = c_decl + + @staticmethod + cdef wrap(const CDeclaration& c_decl): + cdef Declaration self = Declaration.__new__(Declaration) + self.init(c_decl) + return self + + cdef inline CDeclaration unwrap(self) nogil: + return self.decl + + def __init__(self, factory_name, ExecNodeOptions options, inputs=None): + cdef: + c_string c_factory_name + CDeclaration c_decl + vector[CDeclaration.Input] c_inputs + + c_factory_name = tobytes(factory_name) + + if inputs is not None: + for ipt in inputs: + c_inputs.push_back( + CDeclaration.Input((ipt).unwrap()) + ) + + c_decl = CDeclaration(c_factory_name, c_inputs, options.unwrap()) + self.init(c_decl) + + @staticmethod + def from_sequence(decls): + cdef: + vector[CDeclaration] c_decls + CDeclaration c_decl + + for decl in decls: + c_decls.push_back(( decl).unwrap()) + + c_decl = CDeclaration.Sequence(c_decls) + return Declaration.wrap(c_decl) + + def __str__(self): + return frombytes(GetResultValue(DeclarationToString(self.decl))) + + def __repr__(self): + return "\n{0}".format(str(self)) + + def to_table(self, use_threads=True): + cdef: + shared_ptr[CTable] c_table + + c_table = GetResultValue(DeclarationToTable(self.decl, use_threads)) + return pyarrow_wrap_table(c_table) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 80a087740fa9..582042d4d67d 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2575,6 +2575,7 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil pass cdef cppclass CTableSourceNodeOptions "arrow::compute::TableSourceNodeOptions"(CExecNodeOptions): + CTableSourceNodeOptions(shared_ptr[CTable] table) CTableSourceNodeOptions(shared_ptr[CTable] table, int64_t max_batch_size) cdef cppclass CSinkNodeOptions "arrow::compute::SinkNodeOptions"(CExecNodeOptions): @@ -2658,6 +2659,16 @@ cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nog vector[CExecNode*] inputs, const CExecNodeOptions& options) + CResult[shared_ptr[CTable]] DeclarationToTable( + CDeclaration declaration, c_bool use_threads + ) + CResult[shared_ptr[CTable]] DeclarationToTable( + CDeclaration declaration, c_bool use_threads, + CMemoryPool* memory_pool, CFunctionRegistry* function_registry + ) + + CResult[c_string] DeclarationToString(const CDeclaration& declaration) + cdef extern from "arrow/extension_type.h" namespace "arrow": cdef cppclass CExtensionTypeRegistry" arrow::ExtensionTypeRegistry": diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py new file mode 100644 index 000000000000..59917b2e9428 --- /dev/null +++ b/python/pyarrow/tests/test_acero.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import pyarrow as pa +import pyarrow.compute as pc +from pyarrow.compute import field + +from pyarrow._acero import ( + TableSourceNodeOptions, + FilterNodeOptions, + ProjectNodeOptions, + Declaration +) + + +@pytest.fixture +def table_source(): + table = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) + table_opts = TableSourceNodeOptions(table) + table_source = Declaration("table_source", options=table_opts) + return table_source + + +def test_declaration(): + + table = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) + table_opts = TableSourceNodeOptions(table) + filter_opts = FilterNodeOptions(field('a') > 1) + + # using sequence + decl = Declaration.from_sequence([ + Declaration("table_source", options=table_opts), + Declaration("filter", options=filter_opts) + ]) + result = decl.to_table() + assert result.equals(table.slice(1, 2)) + + # using explicit inputs + table_source = Declaration("table_source", options=table_opts) + filtered = Declaration("filter", options=filter_opts, inputs=[table_source]) + result = filtered.to_table() + assert result.equals(table.slice(1, 2)) + + +def test_declaration_repr(table_source): + + assert "TableSourceNode" in str(table_source) + assert "TableSourceNode" in repr(table_source) + + +def test_table_source(): + with pytest.raises(TypeError): + TableSourceNodeOptions(pa.record_batch([pa.array([1, 2, 3])], ["a"])) + + table_source = TableSourceNodeOptions(None) + decl = Declaration("table_source", table_source) + with pytest.raises( + ValueError, match="TableSourceNode requires table which is not null" + ): + _ = decl.to_table() + + +def test_filter(table_source): + # referencing unknown field + decl = Declaration.from_sequence([ + table_source, + Declaration("filter", options=FilterNodeOptions(field("c") > 1)) + ]) + with pytest.raises(ValueError, match=r"No match for FieldRef.Name\(c\)"): + _ = decl.to_table() + + # requires a pyarrow Expression + with pytest.raises(TypeError): + FilterNodeOptions(pa.array([True, False, True])) + with pytest.raises(TypeError): + FilterNodeOptions(None) + + +def test_project(table_source): + # default name from expression + decl = Declaration.from_sequence([ + table_source, + Declaration("project", ProjectNodeOptions([pc.multiply(field("a"), 2)])) + ]) + result = decl.to_table() + assert result.schema.names == ["multiply(a, 2)"] + assert result[0].to_pylist() == [2, 4, 6] + + # provide name + decl = Declaration.from_sequence([ + table_source, + Declaration("project", ProjectNodeOptions([pc.multiply(field("a"), 2)], ["a2"])) + ]) + result = decl.to_table() + assert result.schema.names == ["a2"] + assert result["a2"].to_pylist() == [2, 4, 6] + + # input validation + with pytest.raises(ValueError): + ProjectNodeOptions([pc.multiply(field("a"), 2)], ["a2", "b2"]) diff --git a/python/setup.py b/python/setup.py index 4ee397ed4b3e..759813320668 100755 --- a/python/setup.py +++ b/python/setup.py @@ -213,6 +213,7 @@ def initialize_options(self): '_dataset_orc', '_dataset_parquet', '_exec_plan', + '_acero', '_feather', '_parquet', '_parquet_encryption', From e886024ee9f99eb262fb115c340dd44e07f68ce9 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 15 Feb 2023 10:20:37 +0100 Subject: [PATCH 02/10] add AggregateNodeOptions --- python/pyarrow/_acero.pyx | 64 +++++++++++++++++++++++++++- python/pyarrow/_compute.pxd | 2 + python/pyarrow/_compute.pyx | 63 ++++++++++++++------------- python/pyarrow/includes/libarrow.pxd | 3 ++ python/pyarrow/tests/test_acero.py | 47 ++++++++++++++++++++ 5 files changed, 149 insertions(+), 30 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 3cc75128ba89..8fdba6fffcb3 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -28,7 +28,8 @@ from pyarrow.includes.libarrow_dataset cimport * from pyarrow.lib cimport (Table, check_status, pyarrow_unwrap_table, pyarrow_wrap_table, RecordBatchReader) from pyarrow.lib import frombytes, tobytes -from pyarrow._compute cimport Expression +from pyarrow._compute cimport Expression, FunctionOptions, _ensure_field_ref +from pyarrow.compute import field # Initialize() # Initialise support for Datasets in ExecPlan @@ -154,6 +155,67 @@ class ProjectNodeOptions(_ProjectNodeOptions): self._set_options(expressions, names) +cdef class _AggregateNodeOptions(ExecNodeOptions): + + def _set_options(self, aggregates, keys=None): + cdef: + CAggregate c_aggr + vector[CAggregate] c_aggregations + vector[CFieldRef] c_keys + + for arg_names, func_name, opts, name in aggregates: + c_aggr.function = tobytes(func_name) + if opts is not None: + c_aggr.options = (opts).wrapped + else: + c_aggr.options = nullptr + for arg in arg_names: + c_aggr.target.push_back(_ensure_field_ref(arg)) + c_aggr.name = tobytes(name) + + c_aggregations.push_back(move(c_aggr)) + + if keys is None: + keys = [] + for name in keys: + c_keys.push_back(_ensure_field_ref(name)) + + self.wrapped.reset( + new CAggregateNodeOptions(c_aggregations, c_keys) + ) + + +class AggregateNodeOptions(_AggregateNodeOptions): + """ + Make a node which aggregates input batches, optionally grouped by keys. + + Acero supports two types of aggregates: "scalar" aggregates, + and "hash" aggregates. Scalar aggregates reduce an array or scalar + input to a single scalar output (e.g. computing the mean of a column). + Hash aggregates act like GROUP BY in SQL and first partition data + based on one or more key columns, then reduce the data in each partition. + The aggregate node supports both types of computation, and can compute + any number of aggregations at once. + + Parameters + ---------- + aggregates : list of tuples + Aggregations which will be applied to the targetted fields. + Specified as a list of tuples, where each tuple is one aggregation + specification and consists of: aggregation column name followed + by function name, aggregation function options object and the + output field name. + The column name can be a string, an empty list or a list of + column names, for unary, nullary and n-ary aggregation functions + respectively. + keys : list, optional + Keys by which aggregations will be grouped. + + """ + def __init__(self, aggregates, keys=None): + self._set_options(aggregates, keys) + + cdef class Declaration(_Weakrefable): """ diff --git a/python/pyarrow/_compute.pxd b/python/pyarrow/_compute.pxd index ee348e981697..da195f9d63d7 100644 --- a/python/pyarrow/_compute.pxd +++ b/python/pyarrow/_compute.pxd @@ -62,3 +62,5 @@ cdef class Expression(_Weakrefable): cdef CExpression _true + +cdef CFieldRef _ensure_field_ref(value) except * diff --git a/python/pyarrow/_compute.pyx b/python/pyarrow/_compute.pyx index 2f6d67c99114..e0c03edd0514 100644 --- a/python/pyarrow/_compute.pyx +++ b/python/pyarrow/_compute.pyx @@ -1395,38 +1395,43 @@ class MakeStructOptions(_MakeStructOptions): self._set_options(field_names, field_nullability, field_metadata) +cdef CFieldRef _ensure_field_ref(value) except *: + cdef: + CFieldRef field_ref + const CFieldRef* field_ref_ptr + + if isinstance(value, (list, tuple)): + value = Expression._nested_field(tuple(value)) + + if isinstance(value, Expression): + field_ref_ptr = (value).unwrap().field_ref() + if field_ref_ptr is NULL: + raise ValueError("Unable to get FieldRef from Expression") + field_ref = deref(field_ref_ptr) + elif isinstance(value, (bytes, str)): + if value.startswith(b'.' if isinstance(value, bytes) else '.'): + field_ref = GetResultValue( + CFieldRef.FromDotPath(tobytes(value))) + else: + field_ref = CFieldRef(tobytes(value)) + elif isinstance(value, int): + field_ref = CFieldRef( value) + else: + raise TypeError("Expected a field reference as a str or int, list of " + f"str or int, or Expression. Got {type(value)} instead.") + return field_ref + + cdef class _StructFieldOptions(FunctionOptions): def _set_options(self, indices): - cdef: - CFieldRef field_ref - const CFieldRef* field_ref_ptr - if isinstance(indices, (list, tuple)): - if len(indices): - indices = Expression._nested_field(tuple(indices)) - else: - # Allow empty indices; effecitively return same array - self.wrapped.reset( - new CStructFieldOptions(indices)) - return - - if isinstance(indices, Expression): - field_ref_ptr = (indices).unwrap().field_ref() - if field_ref_ptr is NULL: - raise ValueError("Unable to get CFieldRef from Expression") - field_ref = deref(field_ref_ptr) - elif isinstance(indices, (bytes, str)): - if indices.startswith(b'.' if isinstance(indices, bytes) else '.'): - field_ref = GetResultValue( - CFieldRef.FromDotPath(tobytes(indices))) - else: - field_ref = CFieldRef(tobytes(indices)) - elif isinstance(indices, int): - field_ref = CFieldRef( indices) - else: - raise TypeError("Expected List[str], List[int], List[bytes], " - "Expression, bytes, str, or int. " - f"Got: {type(indices)}") + if isinstance(indices, (list, tuple)) and not len(indices): + # Allow empty indices; effecitively return same array + self.wrapped.reset( + new CStructFieldOptions(indices)) + return + + cdef CFieldRef field_ref = _ensure_field_ref(indices) self.wrapped.reset(new CStructFieldOptions(field_ref)) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 40569d214d6a..87b487088477 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2594,6 +2594,9 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil CProjectNodeOptions(vector[CExpression] expressions, vector[c_string] names) + cdef cppclass CAggregateNodeOptions "arrow::compute::AggregateNodeOptions"(CExecNodeOptions): + CAggregateNodeOptions(vector[CAggregate] aggregates, vector[CFieldRef] names) + cdef cppclass COrderBySinkNodeOptions "arrow::compute::OrderBySinkNodeOptions"(CExecNodeOptions): COrderBySinkNodeOptions(vector[CSortOptions] options, CAsyncExecBatchGenerator generator) diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index 59917b2e9428..253aa95b4707 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -25,6 +25,7 @@ TableSourceNodeOptions, FilterNodeOptions, ProjectNodeOptions, + AggregateNodeOptions, Declaration ) @@ -114,3 +115,49 @@ def test_project(table_source): # input validation with pytest.raises(ValueError): ProjectNodeOptions([pc.multiply(field("a"), 2)], ["a2", "b2"]) + + +def test_aggregate_scalar(table_source): + decl = Declaration.from_sequence([ + table_source, + Declaration("aggregate", AggregateNodeOptions([("a", "sum", None, "a_sum")])) + ]) + result = decl.to_table() + assert result.schema.names == ["a_sum"] + assert result["a_sum"].to_pylist() == [6] + + +def test_aggregate_hash(): + table = pa.table({'a': [1, 2, None], 'b': ["foo", "bar", "foo"]}) + table_opts = TableSourceNodeOptions(table) + table_source = Declaration("table_source", options=table_opts) + + # default options + aggr_opts = AggregateNodeOptions( + [("a", "hash_count", None, "count(a)")], keys=["b"]) + decl = Declaration.from_sequence([ + table_source, Declaration("aggregate", aggr_opts) + ]) + result = decl.to_table() + expected = pa.table({"count(a)": [1, 1], "b": ["foo", "bar"]}) + assert result.equals(expected) + + # specify function options + aggr_opts = AggregateNodeOptions( + [("a", "hash_count", pc.CountOptions("all"), "count(a)")], keys=["b"] + ) + decl = Declaration.from_sequence([ + table_source, Declaration("aggregate", aggr_opts) + ]) + result = decl.to_table() + expected = pa.table({"count(a)": [2, 1], "b": ["foo", "bar"]}) + assert result.equals(expected) + + # wrong type of (aggregation) function + # TODO test with kernel that matches number of arguments (arity) -> avoid segfault + aggr_opts = AggregateNodeOptions([("a", "sum", None, "a_sum")], keys=["b"]) + decl = Declaration.from_sequence([ + table_source, Declaration("aggregate", aggr_opts) + ]) + with pytest.raises(ValueError): + _ = decl.to_table() From bf30ebc260edd73e0305282e2edf912d56f012c4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 15 Feb 2023 10:40:04 +0100 Subject: [PATCH 03/10] lint --- python/pyarrow/_acero.pyx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 8fdba6fffcb3..ad35a8ea40b8 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -77,6 +77,7 @@ class TableSourceNodeOptions(_TableSourceNodeOptions): table : pyarrow.Table The table which acts as the data source. """ + def __init__(self, Table table): self._set_options(table) @@ -102,6 +103,7 @@ class FilterNodeOptions(_FilterNodeOptions): filter_expression : pyarrow.compute.Expression """ + def __init__(self, Expression filter_expression): self._set_options(filter_expression) @@ -151,6 +153,7 @@ class ProjectNodeOptions(_ProjectNodeOptions): `expressions`). If `names` is not provided, the string representations of exprs will be used. """ + def __init__(self, expressions, names=None): self._set_options(expressions, names) @@ -181,8 +184,8 @@ cdef class _AggregateNodeOptions(ExecNodeOptions): c_keys.push_back(_ensure_field_ref(name)) self.wrapped.reset( - new CAggregateNodeOptions(c_aggregations, c_keys) - ) + new CAggregateNodeOptions(c_aggregations, c_keys) + ) class AggregateNodeOptions(_AggregateNodeOptions): @@ -212,6 +215,7 @@ class AggregateNodeOptions(_AggregateNodeOptions): Keys by which aggregations will be grouped. """ + def __init__(self, aggregates, keys=None): self._set_options(aggregates, keys) From 4a39e80179b81ceca0e66e300f5bb5a48fd07c16 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 15 Feb 2023 16:16:45 +0100 Subject: [PATCH 04/10] add hash join --- python/pyarrow/_acero.pyx | 127 ++++++++++++++++++++++++++++- python/pyarrow/tests/test_acero.py | 58 ++++++++++++- 2 files changed, 183 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index ad35a8ea40b8..b71758e1d80f 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -28,7 +28,7 @@ from pyarrow.includes.libarrow_dataset cimport * from pyarrow.lib cimport (Table, check_status, pyarrow_unwrap_table, pyarrow_wrap_table, RecordBatchReader) from pyarrow.lib import frombytes, tobytes -from pyarrow._compute cimport Expression, FunctionOptions, _ensure_field_ref +from pyarrow._compute cimport Expression, FunctionOptions, _ensure_field_ref, _true from pyarrow.compute import field # Initialize() # Initialise support for Datasets in ExecPlan @@ -220,6 +220,131 @@ class AggregateNodeOptions(_AggregateNodeOptions): self._set_options(aggregates, keys) +cdef class _HashJoinNodeOptions(ExecNodeOptions): + # _join_type_map = { + # "left semi": CJoinType_LEFT_SEMI, + # "right semi": CJoinType_RIGHT_SEMI, + # "left anti": CJoinType_LEFT_ANTI, + # "right anti": CJoinType_RIGHT_ANTI, + # "inner": CJoinType_INNER, + # "left outer": CJoinType_LEFT_OUTER, + # "right outer": CJoinType_RIGHT_OUTER, + # "full outer": CJoinType_FULL_OUTER, + # } + + def _set_options( + self, join_type, left_keys, right_keys, left_output=None, right_output=None, + output_suffix_for_left="", output_suffix_for_right="", + ): + cdef: + CJoinType c_join_type + vector[CFieldRef] c_left_keys + vector[CFieldRef] c_right_keys + vector[CFieldRef] c_left_output + vector[CFieldRef] c_right_output + + # join type + # try: + # c_join_type = self._join_type_map[join_type] + # except KeyError: + # raise ValueError("Unsupported join type") + if join_type == "left semi": + c_join_type = CJoinType_LEFT_SEMI + elif join_type == "right semi": + c_join_type = CJoinType_RIGHT_SEMI + elif join_type == "left anti": + c_join_type = CJoinType_LEFT_ANTI + elif join_type == "right anti": + c_join_type = CJoinType_RIGHT_ANTI + elif join_type == "inner": + c_join_type = CJoinType_INNER + elif join_type == "left outer": + c_join_type = CJoinType_LEFT_OUTER + elif join_type == "right outer": + c_join_type = CJoinType_RIGHT_OUTER + elif join_type == "full outer": + c_join_type = CJoinType_FULL_OUTER + else: + raise ValueError("Unsupported join type") + + # left/right keys + if isinstance(left_keys, str): + left_keys = [left_keys] + for key in left_keys: + c_left_keys.push_back(_ensure_field_ref(key)) + if isinstance(right_keys, str): + right_keys = [right_keys] + for key in right_keys: + c_right_keys.push_back(_ensure_field_ref(key)) + + # left/right output fields + if left_output is not None and right_output is not None: + for colname in left_output: + c_left_output.push_back(_ensure_field_ref(colname)) + for colname in right_output: + c_right_output.push_back(_ensure_field_ref(colname)) + + self.wrapped.reset( + new CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys, + c_left_output, c_right_output, + _true, + tobytes(output_suffix_for_left), + tobytes(output_suffix_for_right) + ) + ) + else: + self.wrapped.reset( + new CHashJoinNodeOptions( + c_join_type, c_left_keys, c_right_keys, + _true, + tobytes(output_suffix_for_left), + tobytes(output_suffix_for_right) + ) + ) + + +class HashJoinNodeOptions(_HashJoinNodeOptions): + """ + Make a node which implements join operation using hash join strategy. + + Parameters + ---------- + join_type : str + Type of join. One of "left semi", "right semi", "left anti", + "right anti", "inner", "left outer", "right outer", "full outer". + left_keys + Key fields from left input. + right_keys + key fields from right input. + left_output + Output fields passed from left input. If left and right output + fields are not specified, all valid fields from both left and + right input will be output + right_output + Output fields passed from right input. If left and right output + fields are not specified, all valid fields from both left and + right input will be output + output_suffix_for_left : str + Suffix added to names of output fields coming from left input + (used to distinguish, if necessary, between fields of the same + name in left and right input and can be left empty if there are + no name collisions). + output_suffix_for_right : str + Suffix added to names of output fields coming from right input, + see `output_suffix_for_left`. + """ + + def __init__( + self, join_type, left_keys, right_keys, left_output=None, right_output=None, + output_suffix_for_left="", output_suffix_for_right="" + ): + self._set_options( + join_type, left_keys, right_keys, left_output, right_output, + output_suffix_for_left, output_suffix_for_right + ) + + cdef class Declaration(_Weakrefable): """ diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index 253aa95b4707..704b5b201447 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -26,7 +26,8 @@ FilterNodeOptions, ProjectNodeOptions, AggregateNodeOptions, - Declaration + HashJoinNodeOptions, + Declaration, ) @@ -161,3 +162,58 @@ def test_aggregate_hash(): ]) with pytest.raises(ValueError): _ = decl.to_table() + + +def test_hash_join(): + left = pa.table({'key': [1, 2, 3], 'a': [4, 5, 6]}) + left_source = Declaration("table_source", options=TableSourceNodeOptions(left)) + right = pa.table({'key': [2, 3, 4], 'b': [4, 5, 6]}) + right_source = Declaration("table_source", options=TableSourceNodeOptions(right)) + + # inner join + join_opts = HashJoinNodeOptions("inner", left_keys="key", right_keys="key") + joined = Declaration( + "hashjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + expected = pa.table( + [[2, 3], [5, 6], [2, 3], [4, 5]], + names=["key", "a", "key", "b"]) + assert result.equals(expected) + + # left join + join_opts = HashJoinNodeOptions( + "left outer", left_keys="key", right_keys="key") + joined = Declaration( + "hashjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + expected = pa.table( + [[1, 2, 3], [4, 5, 6], [None, 2, 3], [None, 4, 5]], + names=["key", "a", "key", "b"] + ) + assert result.sort_by("a").equals(expected) + + # suffixes + join_opts = HashJoinNodeOptions( + "left outer", left_keys="key", right_keys="key", + output_suffix_for_left="_left", output_suffix_for_right="_right") + joined = Declaration( + "hashjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + expected = pa.table( + [[1, 2, 3], [4, 5, 6], [None, 2, 3], [None, 4, 5]], + names=["key_left", "a", "key_right", "b"] + ) + assert result.sort_by("a").equals(expected) + + # manually specifying output columns + join_opts = HashJoinNodeOptions( + "left outer", left_keys="key", right_keys="key", + left_output=["key", "a"], right_output=["b"]) + joined = Declaration( + "hashjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + expected = pa.table( + [[1, 2, 3], [4, 5, 6], [None, 4, 5]], + names=["key", "a", "b"] + ) + assert result.sort_by("a").equals(expected) From 0e93aeb76bc4f2a83f81672728acc9b2d1376280 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 15 Feb 2023 17:03:04 +0100 Subject: [PATCH 05/10] add docstrings --- python/pyarrow/_acero.pyx | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index b71758e1d80f..814797d30a18 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -347,7 +347,27 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): cdef class Declaration(_Weakrefable): """ + Helper class for declaring the nodes of an ExecPlan. + A Declaration represents an unconstructed ExecNode, and potentially + more since its inputs may also be Declarations or when constructed + with ``from_sequence``. + + Parameters + ---------- + factory_name : str + The ExecNode factory name, such as "table_source", "filter", + "project" etc. + options : ExecNodeOptions + Corresponding ExecNodeOptions subclass (matching the factory name). + inputs : list of Declaration, optional + Input nodes for this declaration. Optional if the node is a source + node, or when the declaration gets combined later with + `from_sequence`. + + Returns + ------- + Declaration """ cdef void init(self, const CDeclaration& c_decl): self.decl = c_decl @@ -380,6 +400,21 @@ cdef class Declaration(_Weakrefable): @staticmethod def from_sequence(decls): + """ + Convenience factory for the common case of a simple sequence of nodes. + + Each of the declarations will be appended to the inputs of the + subsequent declaration, and the final modified declaration will + be returned. + + Parameters + ---------- + decls : list of Declaration + + Returns + ------- + Declaration + """ cdef: vector[CDeclaration] c_decls CDeclaration c_decl @@ -397,6 +432,26 @@ cdef class Declaration(_Weakrefable): return "\n{0}".format(str(self)) def to_table(self, use_threads=True): + """ + Run the declaration and collect the results into a table. + + This method will implicitly add a sink node to the declaration + to collect results into a table. It will then create an ExecPlan + from the declaration, start the exec plan, block until the plan + has finished, and return the created table. + + Parameters + ---------- + use_threads : bool, default True + If set to False, then all CPU work will be done on the calling + thread. I/O tasks will still happen on the I/O executor + and may be multi-threaded (but should not use significant CPU + resources). + + Returns + ------- + pyarrow.Table + """ cdef: shared_ptr[CTable] c_table From a7fc3d044e86a4a50d1f2261d3f7727263302df4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 24 Feb 2023 14:59:47 +0100 Subject: [PATCH 06/10] address feedback (clean-up, update docs, expand tests for field refs) --- cpp/src/arrow/compute/exec/options.h | 3 +- docs/source/cpp/streaming_execution.rst | 12 ++-- python/pyarrow/_acero.pxd | 3 +- python/pyarrow/_acero.pyx | 78 ++++++++++++------------- python/pyarrow/tests/test_acero.py | 70 +++++++++++++++++++++- 5 files changed, 118 insertions(+), 48 deletions(-) diff --git a/cpp/src/arrow/compute/exec/options.h b/cpp/src/arrow/compute/exec/options.h index bd2bbcb8e64a..dbdf907a2db8 100644 --- a/cpp/src/arrow/compute/exec/options.h +++ b/cpp/src/arrow/compute/exec/options.h @@ -183,7 +183,8 @@ class ARROW_EXPORT FetchNodeOptions : public ExecNodeOptions { int64_t count; }; -/// \brief Make a node which executes expressions on input batches, producing new batches. +/// \brief Make a node which executes expressions on input batches, producing batches +/// of the same length with new columns. /// /// Each expression will be evaluated against each batch which is pushed to /// this node to produce a corresponding output column. diff --git a/docs/source/cpp/streaming_execution.rst b/docs/source/cpp/streaming_execution.rst index 0da3ad7ddcb0..740d580d3bcc 100644 --- a/docs/source/cpp/streaming_execution.rst +++ b/docs/source/cpp/streaming_execution.rst @@ -652,8 +652,9 @@ Example of using ``table_source`` ---------- ``filter`` operation, as the name suggests, provides an option to define data filtering -criteria. It selects rows matching a given expression. Filters can be written using -:class:`arrow::compute::Expression`. For example, if we wish to keep rows where the value +criteria. It selects rows where the given expression evaluates to true. Filters can bewritten using +:class:`arrow::compute::Expression`, and the expression should have a return type of boolean. +For example, if we wish to keep rows where the value of column ``b`` is greater than 3, then we can use the following expression. Filter example: @@ -672,8 +673,11 @@ Filter example: ``project`` operation rearranges, deletes, transforms, and creates columns. Each output column is computed by evaluating an expression -against the source record batch. This is exposed via -:class:`arrow::compute::ProjectNodeOptions` which requires, +against the source record batch. These must be scalar expressions +(expressions consisting of scalar literals, field references and scalar +functions, i.e. elementwise functions that return one value for each input +row independent of the value of all other rows). +This is exposed via :class:`arrow::compute::ProjectNodeOptions` which requires, an :class:`arrow::compute::Expression` and name for each of the output columns (if names are not provided, the string representations of exprs will be used). diff --git a/python/pyarrow/_acero.pxd b/python/pyarrow/_acero.pxd index 32133596663f..84d90b24dbb3 100644 --- a/python/pyarrow/_acero.pxd +++ b/python/pyarrow/_acero.pxd @@ -26,9 +26,8 @@ cdef class ExecNodeOptions(_Weakrefable): cdef: shared_ptr[CExecNodeOptions] wrapped - cdef const CExecNodeOptions* get_options(self) except NULL cdef void init(self, const shared_ptr[CExecNodeOptions]& sp) - cdef inline shared_ptr[CExecNodeOptions] unwrap(self) + cdef inline shared_ptr[CExecNodeOptions] unwrap(self) nogil cdef class Declaration(_Weakrefable): diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 814797d30a18..08cb8017b40e 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -37,24 +37,12 @@ from pyarrow.compute import field cdef class ExecNodeOptions(_Weakrefable): __slots__ = () # avoid mistakingly creating attributes - cdef const CExecNodeOptions* get_options(self) except NULL: - return self.wrapped.get() - cdef void init(self, const shared_ptr[CExecNodeOptions]& sp): self.wrapped = sp - cdef inline shared_ptr[CExecNodeOptions] unwrap(self): + cdef inline shared_ptr[CExecNodeOptions] unwrap(self) nogil: return self.wrapped - # def __repr__(self): - # type_name = self.__class__.__name__ - # # Remove {} so we can use our own braces - # string_repr = frombytes(self.get_options().ToString())[1:-1] - # return f"{type_name}({string_repr})" - - # def __eq__(self, FunctionOptions other): - # return self.get_options().Equals(deref(other.get_options())) - cdef class _TableSourceNodeOptions(ExecNodeOptions): @@ -95,13 +83,13 @@ class FilterNodeOptions(_FilterNodeOptions): Make a node which excludes some rows from batches passed through it. The "filter" operation provides an option to define data filtering - criteria. It selects rows matching a given expression. Filters can - be written using pyarrow.compute.Expression. + criteria. It selects rows where the given expression evaluates to true. + Filters can be written using pyarrow.compute.Expression, and the + expression must have a return type of boolean. Parameters ---------- filter_expression : pyarrow.compute.Expression - """ def __init__(self, Expression filter_expression): @@ -121,7 +109,9 @@ cdef class _ProjectNodeOptions(ExecNodeOptions): if names is not None: if len(names) != len(expressions): - raise ValueError("dd") + raise ValueError( + "The number of names should be equal to the number of expressions" + ) for name in names: c_names.push_back(tobytes(name)) @@ -138,17 +128,22 @@ cdef class _ProjectNodeOptions(ExecNodeOptions): class ProjectNodeOptions(_ProjectNodeOptions): """ Make a node which executes expressions on input batches, - producing new batches. + producing batches of the same length with new columns. The "project" operation rearranges, deletes, transforms, and creates columns. Each output column is computed by evaluating - an expression against the source record batch. + an expression against the source record batch. These must be + scalar expressions (expressions consisting of scalar literals, + field references and scalar functions, i.e. elementwise functions + that return one value for each input row independent of the value + of all other rows). Parameters ---------- expressions : list of pyarrow.compute.Expression - List of expressions to evaluate against the source batch. - names : list of str + List of expressions to evaluate against the source batch. This must + be scalar expressions. + names : list of str, optional List of names for each of the ouptut columns (same length as `expressions`). If `names` is not provided, the string representations of exprs will be used. @@ -172,6 +167,8 @@ cdef class _AggregateNodeOptions(ExecNodeOptions): c_aggr.options = (opts).wrapped else: c_aggr.options = nullptr + if not isinstance(arg_names, (list, tuple)): + arg_names = [arg_names] for arg in arg_names: c_aggr.target.push_back(_ensure_field_ref(arg)) c_aggr.name = tobytes(name) @@ -205,15 +202,16 @@ class AggregateNodeOptions(_AggregateNodeOptions): aggregates : list of tuples Aggregations which will be applied to the targetted fields. Specified as a list of tuples, where each tuple is one aggregation - specification and consists of: aggregation column name followed + specification and consists of: aggregation target column(s) followed by function name, aggregation function options object and the output field name. - The column name can be a string, an empty list or a list of - column names, for unary, nullary and n-ary aggregation functions - respectively. - keys : list, optional - Keys by which aggregations will be grouped. - + The target column(s) specification can be a single field reference, + an empty list or a list of fields unary, nullary and n-ary aggregation + functions respectively. Each field reference can be a string + column name or expression. + keys : list of field references, optional + Keys by which aggregations will be grouped. Each key can reference + a field using a string name or expression. """ def __init__(self, aggregates, keys=None): @@ -268,11 +266,11 @@ cdef class _HashJoinNodeOptions(ExecNodeOptions): raise ValueError("Unsupported join type") # left/right keys - if isinstance(left_keys, str): + if not isinstance(left_keys, (list, tuple)): left_keys = [left_keys] for key in left_keys: c_left_keys.push_back(_ensure_field_ref(key)) - if isinstance(right_keys, str): + if not isinstance(right_keys, (list, tuple)): right_keys = [right_keys] for key in right_keys: c_right_keys.push_back(_ensure_field_ref(key)) @@ -313,15 +311,16 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): join_type : str Type of join. One of "left semi", "right semi", "left anti", "right anti", "inner", "left outer", "right outer", "full outer". - left_keys - Key fields from left input. - right_keys - key fields from right input. - left_output + left_keys : str, Expression or list + Key fields from left input. Each key can be a string column name + or a field expression, or a list of such field references. + right_keys : str, Expression or list + Key fields from right input. See `left_keys` for details. + left_output : list, optional Output fields passed from left input. If left and right output fields are not specified, all valid fields from both left and right input will be output - right_output + right_output : list, optional Output fields passed from right input. If left and right output fields are not specified, all valid fields from both left and right input will be output @@ -332,7 +331,7 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): no name collisions). output_suffix_for_right : str Suffix added to names of output fields coming from right input, - see `output_suffix_for_left`. + see `output_suffix_for_left` for details. """ def __init__( @@ -431,7 +430,7 @@ cdef class Declaration(_Weakrefable): def __repr__(self): return "\n{0}".format(str(self)) - def to_table(self, use_threads=True): + def to_table(self, bint use_threads=True): """ Run the declaration and collect the results into a table. @@ -455,5 +454,6 @@ cdef class Declaration(_Weakrefable): cdef: shared_ptr[CTable] c_table - c_table = GetResultValue(DeclarationToTable(self.decl, use_threads)) + with nogil: + c_table = GetResultValue(DeclarationToTable(self.unwrap(), use_threads)) return pyarrow_wrap_table(c_table) diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index 704b5b201447..c0246d1ed421 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -117,6 +117,14 @@ def test_project(table_source): with pytest.raises(ValueError): ProjectNodeOptions([pc.multiply(field("a"), 2)], ["a2", "b2"]) + # no scalar expression + decl = Declaration.from_sequence([ + table_source, + Declaration("project", ProjectNodeOptions([pc.sum(field("a"))])) + ]) + with pytest.raises(ValueError, match="cannot Execute non-scalar expression"): + _ = decl.to_table() + def test_aggregate_scalar(table_source): decl = Declaration.from_sequence([ @@ -127,6 +135,47 @@ def test_aggregate_scalar(table_source): assert result.schema.names == ["a_sum"] assert result["a_sum"].to_pylist() == [6] + # with options class + table = pa.table({'a': [1, 2, None]}) + aggr_opts = AggregateNodeOptions( + [("a", "sum", pc.ScalarAggregateOptions(skip_nulls=False), "a_sum")] + ) + decl = Declaration.from_sequence([ + Declaration("table_source", TableSourceNodeOptions(table)), + Declaration("aggregate", aggr_opts), + ]) + result = decl.to_table() + assert result.schema.names == ["a_sum"] + assert result["a_sum"].to_pylist() == [None] + + # test various ways of specifying the target column + for target in ["a", field("a"), 0, field(0), ["a"], [field("a")], [0]]: + aggr_opts = AggregateNodeOptions([(target, "sum", None, "a_sum")]) + decl = Declaration.from_sequence( + [table_source, Declaration("aggregate", aggr_opts)] + ) + result = decl.to_table() + assert result.schema.names == ["a_sum"] + assert result["a_sum"].to_pylist() == [6] + + # proper error when specifying the wrong number of target columns + aggr_opts = AggregateNodeOptions([(["a", "b"], "sum", None, "a_sum")]) + decl = Declaration.from_sequence( + [table_source, Declaration("aggregate", aggr_opts)] + ) + with pytest.raises( + ValueError, match="Function 'sum' accepts 1 arguments but 2 passed" + ): + _ = decl.to_table() + + # proper error when using hash aggregation without keys + aggr_opts = AggregateNodeOptions([("a", "hash_sum", None, "a_sum")]) + decl = Declaration.from_sequence( + [table_source, Declaration("aggregate", aggr_opts)] + ) + with pytest.raises(ValueError, match="is a hash aggregate function"): + _ = decl.to_table() + def test_aggregate_hash(): table = pa.table({'a': [1, 2, None], 'b': ["foo", "bar", "foo"]}) @@ -151,7 +200,17 @@ def test_aggregate_hash(): table_source, Declaration("aggregate", aggr_opts) ]) result = decl.to_table() - expected = pa.table({"count(a)": [2, 1], "b": ["foo", "bar"]}) + expected_all = pa.table({"count(a)": [2, 1], "b": ["foo", "bar"]}) + assert result.equals(expected_all) + + # specify keys as field references + aggr_opts = AggregateNodeOptions( + [("a", "hash_count", None, "count(a)")], keys=[field("b")] + ) + decl = Declaration.from_sequence([ + table_source, Declaration("aggregate", aggr_opts) + ]) + result = decl.to_table() assert result.equals(expected) # wrong type of (aggregation) function @@ -180,6 +239,13 @@ def test_hash_join(): names=["key", "a", "key", "b"]) assert result.equals(expected) + for keys in [field("key"), ["key"], [field("key")]]: + join_opts = HashJoinNodeOptions("inner", left_keys=keys, right_keys=keys) + joined = Declaration( + "hashjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + assert result.equals(expected) + # left join join_opts = HashJoinNodeOptions( "left outer", left_keys="key", right_keys="key") @@ -208,7 +274,7 @@ def test_hash_join(): # manually specifying output columns join_opts = HashJoinNodeOptions( "left outer", left_keys="key", right_keys="key", - left_output=["key", "a"], right_output=["b"]) + left_output=["key", "a"], right_output=[field("b")]) joined = Declaration( "hashjoin", options=join_opts, inputs=[left_source, right_source]) result = joined.to_table() From 0d66a58676a81f40bfe2ed6bc98772ba17a655ba Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 24 Feb 2023 16:09:34 +0100 Subject: [PATCH 07/10] Add Declaration.to_reader --- python/pyarrow/_acero.pyx | 17 +++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 3 +++ python/pyarrow/tests/test_acero.py | 8 ++++++++ 3 files changed, 28 insertions(+) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 08cb8017b40e..e352329b0d09 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -457,3 +457,20 @@ cdef class Declaration(_Weakrefable): with nogil: c_table = GetResultValue(DeclarationToTable(self.unwrap(), use_threads)) return pyarrow_wrap_table(c_table) + + def to_reader(self, bint use_threads=True): + """Run the declaration and return results as a RecordBatchReader. + + For details about the parameters, see `to_table`. + + Returns + ------- + pyarrow.RecordBatchReader + """ + cdef: + RecordBatchReader reader + reader = RecordBatchReader.__new__(RecordBatchReader) + reader.reader.reset( + GetResultValue(DeclarationToReader(self.unwrap(), use_threads)).release() + ) + return reader diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 1e07190e53aa..fd381bb3de26 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2675,6 +2675,9 @@ cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nog CDeclaration declaration, c_bool use_threads, CMemoryPool* memory_pool, CFunctionRegistry* function_registry ) + CResult[unique_ptr[CRecordBatchReader]] DeclarationToReader( + CDeclaration declaration, c_bool use_threads + ) CResult[c_string] DeclarationToString(const CDeclaration& declaration) diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index c0246d1ed421..4cba55f7e421 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -66,6 +66,14 @@ def test_declaration_repr(table_source): assert "TableSourceNode" in repr(table_source) +def test_declaration_to_reader(table_source): + with table_source.to_reader() as reader: + assert reader.schema == pa.schema([("a", pa.int64()), ("b", pa.int64())]) + result = reader.read_all() + expected = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) + assert result.equals(expected) + + def test_table_source(): with pytest.raises(TypeError): TableSourceNodeOptions(pa.record_batch([pa.array([1, 2, 3])], ["a"])) From 50487fcfef215c224d90f6e40f3c662cefc0c410 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 3 Mar 2023 09:50:16 +0100 Subject: [PATCH 08/10] Update docs/source/cpp/streaming_execution.rst Co-authored-by: Weston Pace --- docs/source/cpp/streaming_execution.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/cpp/streaming_execution.rst b/docs/source/cpp/streaming_execution.rst index 740d580d3bcc..fc7a724959ca 100644 --- a/docs/source/cpp/streaming_execution.rst +++ b/docs/source/cpp/streaming_execution.rst @@ -652,7 +652,7 @@ Example of using ``table_source`` ---------- ``filter`` operation, as the name suggests, provides an option to define data filtering -criteria. It selects rows where the given expression evaluates to true. Filters can bewritten using +criteria. It selects rows where the given expression evaluates to true. Filters can be written using :class:`arrow::compute::Expression`, and the expression should have a return type of boolean. For example, if we wish to keep rows where the value of column ``b`` is greater than 3, then we can use the following expression. From af3b13d34a2b57d14b05608a90b03e6a0d8e62fe Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 3 Mar 2023 10:15:19 +0100 Subject: [PATCH 09/10] address feedback --- python/pyarrow/_acero.pyx | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index e352329b0d09..cbb374460171 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -60,6 +60,8 @@ class TableSourceNodeOptions(_TableSourceNodeOptions): """ A Source node which accepts a table. + This is the option class for the "table_source" node factory. + Parameters ---------- table : pyarrow.Table @@ -82,6 +84,8 @@ class FilterNodeOptions(_FilterNodeOptions): """ Make a node which excludes some rows from batches passed through it. + This is the option class for the "filter" node factory. + The "filter" operation provides an option to define data filtering criteria. It selects rows where the given expression evaluates to true. Filters can be written using pyarrow.compute.Expression, and the @@ -130,6 +134,8 @@ class ProjectNodeOptions(_ProjectNodeOptions): Make a node which executes expressions on input batches, producing batches of the same length with new columns. + This is the option class for the "project" node factory. + The "project" operation rearranges, deletes, transforms, and creates columns. Each output column is computed by evaluating an expression against the source record batch. These must be @@ -189,6 +195,8 @@ class AggregateNodeOptions(_AggregateNodeOptions): """ Make a node which aggregates input batches, optionally grouped by keys. + This is the option class for the "aggregate" node factory. + Acero supports two types of aggregates: "scalar" aggregates, and "hash" aggregates. Scalar aggregates reduce an array or scalar input to a single scalar output (e.g. computing the mean of a column). @@ -306,6 +314,8 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): """ Make a node which implements join operation using hash join strategy. + This is the option class for the "hashjoin" node factory. + Parameters ---------- join_type : str @@ -317,13 +327,15 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): right_keys : str, Expression or list Key fields from right input. See `left_keys` for details. left_output : list, optional - Output fields passed from left input. If left and right output - fields are not specified, all valid fields from both left and - right input will be output + List of output fields passed from left input. If left and right + output fields are not specified, all valid fields from both left and + right input will be output. Each field can be a string column name + or a field expression. right_output : list, optional - Output fields passed from right input. If left and right output - fields are not specified, all valid fields from both left and - right input will be output + List of output fields passed from right input. If left and right + output fields are not specified, all valid fields from both left and + right input will be output. Each field can be a string column name + or a field expression. output_suffix_for_left : str Suffix added to names of output fields coming from left input (used to distinguish, if necessary, between fields of the same @@ -352,17 +364,22 @@ cdef class Declaration(_Weakrefable): more since its inputs may also be Declarations or when constructed with ``from_sequence``. + The possible ExecNodes to use are registered with a name, + the "factory name", and need to be specified using this name, together + with its corresponding ExecNodeOptions subclass. + Parameters ---------- factory_name : str The ExecNode factory name, such as "table_source", "filter", - "project" etc. + "project" etc. See the ExecNodeOptions subclasses for the exact + factory names to use. options : ExecNodeOptions Corresponding ExecNodeOptions subclass (matching the factory name). inputs : list of Declaration, optional Input nodes for this declaration. Optional if the node is a source node, or when the declaration gets combined later with - `from_sequence`. + ``from_sequence``. Returns ------- From 8cf2450520d44df2101ebb865ce1e1adbf06c679 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 3 Mar 2023 10:16:00 +0100 Subject: [PATCH 10/10] clean-up --- python/pyarrow/_acero.pyx | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index cbb374460171..91f3ef6712f0 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -31,8 +31,6 @@ from pyarrow.lib import frombytes, tobytes from pyarrow._compute cimport Expression, FunctionOptions, _ensure_field_ref, _true from pyarrow.compute import field -# Initialize() # Initialise support for Datasets in ExecPlan - cdef class ExecNodeOptions(_Weakrefable): __slots__ = () # avoid mistakingly creating attributes @@ -227,16 +225,6 @@ class AggregateNodeOptions(_AggregateNodeOptions): cdef class _HashJoinNodeOptions(ExecNodeOptions): - # _join_type_map = { - # "left semi": CJoinType_LEFT_SEMI, - # "right semi": CJoinType_RIGHT_SEMI, - # "left anti": CJoinType_LEFT_ANTI, - # "right anti": CJoinType_RIGHT_ANTI, - # "inner": CJoinType_INNER, - # "left outer": CJoinType_LEFT_OUTER, - # "right outer": CJoinType_RIGHT_OUTER, - # "full outer": CJoinType_FULL_OUTER, - # } def _set_options( self, join_type, left_keys, right_keys, left_output=None, right_output=None, @@ -250,10 +238,6 @@ cdef class _HashJoinNodeOptions(ExecNodeOptions): vector[CFieldRef] c_right_output # join type - # try: - # c_join_type = self._join_type_map[join_type] - # except KeyError: - # raise ValueError("Unsupported join type") if join_type == "left semi": c_join_type = CJoinType_LEFT_SEMI elif join_type == "right semi":