diff --git a/docs/source/python/api.rst b/docs/source/python/api.rst index 0ce120664dc4..b4b260109127 100644 --- a/docs/source/python/api.rst +++ b/docs/source/python/api.rst @@ -27,10 +27,11 @@ API Reference api/datatypes api/arrays api/memory + api/tables api/compute + api/acero api/substrait api/files - api/tables api/ipc api/flight api/formats diff --git a/docs/source/python/api/acero.rst b/docs/source/python/api/acero.rst new file mode 100644 index 000000000000..a8107f3cc1d6 --- /dev/null +++ b/docs/source/python/api/acero.rst @@ -0,0 +1,52 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. _api.acero: +.. currentmodule:: pyarrow.acero + +Acero - Streaming Execution Engine +================================== + +.. warning:: + + Acero is experimental and a stable API is not yet guaranteed. + +Acero is a streaming query engine, which allows the computation to be expressed +as an "execution plan" (constructed using the :class:`Declaration` interface). +This enables to create a computation composed of ``pyarrow.compute`` functions +and to execute this efficiently in a batched manner. + +.. autosummary:: + :toctree: ../generated/ + + Declaration + ExecNodeOptions + TableSourceNodeOptions + ScanNodeOptions + FilterNodeOptions + ProjectNodeOptions + AggregateNodeOptions + OrderByNodeOptions + HashJoinNodeOptions + + +.. seealso:: + + :doc:`Acero C++ user guide <../../cpp/streaming_execution>` + + :ref:`api.substrait` + Alternative way to run Acero from a standardized Substrait plan. diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 522f4fd005c2..b799abb3e3a7 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -35,6 +35,11 @@ from pyarrow._compute cimport ( cdef class ExecNodeOptions(_Weakrefable): + """ + Base class for the node options. + + Use one of the subclasses to construct an options object. + """ __slots__ = () # avoid mistakingly creating attributes cdef void init(self, const shared_ptr[CExecNodeOptions]& sp): diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index d6fe91ba21ea..a4c365254ceb 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -27,7 +27,7 @@ from libcpp cimport bool import pyarrow as pa from pyarrow.lib cimport * -from pyarrow.lib import ArrowTypeError, frombytes, tobytes, _pc +from pyarrow.lib import ArrowTypeError, frombytes, tobytes, _pac from pyarrow.includes.libarrow_dataset cimport * from pyarrow._acero cimport ExecNodeOptions from pyarrow._compute cimport Expression, _bind @@ -808,8 +808,9 @@ cdef class Dataset(_Weakrefable): if isinstance(sorting, str): sorting = [(sorting, "ascending")] - res = _pc()._exec_plan._sort_source(self, output_type=InMemoryDataset, - sort_keys=sorting, **kwargs) + res = _pac()._sort_source( + self, output_type=InMemoryDataset, sort_keys=sorting, **kwargs + ) return res def join(self, right_dataset, keys, right_keys=None, join_type="left outer", @@ -855,10 +856,12 @@ cdef class Dataset(_Weakrefable): """ if right_keys is None: right_keys = keys - return _pc()._exec_plan._perform_join(join_type, self, keys, right_dataset, right_keys, - left_suffix=left_suffix, right_suffix=right_suffix, - use_threads=use_threads, coalesce_keys=coalesce_keys, - output_type=InMemoryDataset) + return _pac()._perform_join( + join_type, self, keys, right_dataset, right_keys, + left_suffix=left_suffix, right_suffix=right_suffix, + use_threads=use_threads, coalesce_keys=coalesce_keys, + output_type=InMemoryDataset + ) cdef class InMemoryDataset(Dataset): diff --git a/python/pyarrow/_exec_plan.py b/python/pyarrow/acero.py similarity index 92% rename from python/pyarrow/_exec_plan.py rename to python/pyarrow/acero.py index 6e49341a13cc..65ffc248d5eb 100644 --- a/python/pyarrow/_exec_plan.py +++ b/python/pyarrow/acero.py @@ -24,13 +24,24 @@ from pyarrow.lib import Table from pyarrow.compute import Expression -from pyarrow.dataset import Dataset, InMemoryDataset -from pyarrow._dataset import ScanNodeOptions -from pyarrow._acero import ( - Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, - ProjectNodeOptions, OrderByNodeOptions + +from pyarrow._acero import ( # noqa + Declaration, + ExecNodeOptions, + TableSourceNodeOptions, + FilterNodeOptions, + ProjectNodeOptions, + AggregateNodeOptions, + OrderByNodeOptions, + HashJoinNodeOptions, ) +try: + import pyarrow.dataset as ds + from pyarrow._dataset import ScanNodeOptions +except ImportError: + ds = None + def _dataset_to_decl(dataset, use_threads=True): decl = Declaration("scan", ScanNodeOptions(dataset, use_threads=use_threads)) @@ -86,9 +97,9 @@ def _perform_join(join_type, left_operand, left_keys, ------- result_table : Table or InMemoryDataset """ - if not isinstance(left_operand, (Table, Dataset)): + if not isinstance(left_operand, (Table, ds.Dataset)): raise TypeError(f"Expected Table or Dataset, got {type(left_operand)}") - if not isinstance(right_operand, (Table, Dataset)): + if not isinstance(right_operand, (Table, ds.Dataset)): raise TypeError(f"Expected Table or Dataset, got {type(right_operand)}") # Prepare left and right tables Keys to send them to the C++ function @@ -134,11 +145,11 @@ def _perform_join(join_type, left_operand, left_keys, right_column_keys_indices[colname] = idx # Add the join node to the execplan - if isinstance(left_operand, Dataset): + if isinstance(left_operand, ds.Dataset): left_source = _dataset_to_decl(left_operand, use_threads=use_threads) else: left_source = Declaration("table_source", TableSourceNodeOptions(left_operand)) - if isinstance(right_operand, Dataset): + if isinstance(right_operand, ds.Dataset): right_source = _dataset_to_decl(right_operand, use_threads=use_threads) else: right_source = Declaration( @@ -210,8 +221,8 @@ def _perform_join(join_type, left_operand, left_keys, if output_type == Table: return result_table - elif output_type == InMemoryDataset: - return InMemoryDataset(result_table) + elif output_type == ds.InMemoryDataset: + return ds.InMemoryDataset(result_table) else: raise TypeError("Unsupported output type") @@ -244,7 +255,7 @@ def _filter_table(table, expression): def _sort_source(table_or_dataset, sort_keys, output_type=Table, **kwargs): - if isinstance(table_or_dataset, Dataset): + if isinstance(table_or_dataset, ds.Dataset): data_source = _dataset_to_decl(table_or_dataset, use_threads=True) else: data_source = Declaration( @@ -258,9 +269,9 @@ def _sort_source(table_or_dataset, sort_keys, output_type=Table, **kwargs): if output_type == Table: return result_table - elif output_type == InMemoryDataset: + elif output_type == ds.InMemoryDataset: # Get rid of special dataset columns # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" - return InMemoryDataset(result_table.select(table_or_dataset.schema.names)) + return ds.InMemoryDataset(result_table.select(table_or_dataset.schema.names)) else: raise TypeError("Unsupported output type") diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 66039471131e..19d911dbb97b 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -120,21 +120,25 @@ UnionMode_SPARSE = _UnionMode_SPARSE UnionMode_DENSE = _UnionMode_DENSE __pc = None +__pac = None def _pc(): global __pc if __pc is None: import pyarrow.compute as pc - try: - from pyarrow import _exec_plan - pc._exec_plan = _exec_plan - except ImportError: - pass __pc = pc return __pc +def _pac(): + global __pac + if __pac is None: + import pyarrow.acero as pac + __pac = pac + return __pac + + def _gdb_test_session(): GdbTestSession() diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 9da7110a1d10..18ead9489c54 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -3145,7 +3145,7 @@ cdef class Table(_PandasConvertible): animals: [["Flamingo","Horse",null]] """ if isinstance(mask, _pc().Expression): - return _pc()._exec_plan._filter_table(self, mask) + return _pac()._filter_table(self, mask) else: return _pc().filter(self, mask, null_selection_behavior) @@ -5044,10 +5044,12 @@ cdef class Table(_PandasConvertible): """ if right_keys is None: right_keys = keys - return _pc()._exec_plan._perform_join(join_type, self, keys, right_table, right_keys, - left_suffix=left_suffix, right_suffix=right_suffix, - use_threads=use_threads, coalesce_keys=coalesce_keys, - output_type=Table) + return _pac()._perform_join( + join_type, self, keys, right_table, right_keys, + left_suffix=left_suffix, right_suffix=right_suffix, + use_threads=use_threads, coalesce_keys=coalesce_keys, + output_type=Table + ) def _reconstruct_table(arrays, schema): diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index 88d03f5dec6c..c0f0e850761c 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -21,7 +21,7 @@ import pyarrow.compute as pc from pyarrow.compute import field -from pyarrow._acero import ( +from pyarrow.acero import ( TableSourceNodeOptions, FilterNodeOptions, ProjectNodeOptions, @@ -33,7 +33,7 @@ try: import pyarrow.dataset as ds - from pyarrow._dataset import ScanNodeOptions + from pyarrow.acero import ScanNodeOptions except ImportError: ds = None diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index b57de8b3c28e..599b75330695 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -22,7 +22,7 @@ try: import pyarrow.dataset as ds - import pyarrow._exec_plan as ep + from pyarrow.acero import _perform_join, _filter_table except ImportError: pass @@ -41,13 +41,13 @@ def test_joins_corner_cases(): }) with pytest.raises(pa.ArrowInvalid): - ep._perform_join("left outer", t1, "", t2, "") + _perform_join("left outer", t1, "", t2, "") with pytest.raises(TypeError): - ep._perform_join("left outer", None, "colA", t2, "colB") + _perform_join("left outer", None, "colA", t2, "colB") with pytest.raises(ValueError): - ep._perform_join("super mario join", t1, "colA", t2, "colB") + _perform_join("super mario join", t1, "colA", t2, "colB") @pytest.mark.parametrize("jointype,expected", [ @@ -109,8 +109,8 @@ def test_joins(jointype, expected, use_threads, use_datasets): t1 = ds.dataset([t1]) t2 = ds.dataset([t2]) - r = ep._perform_join(jointype, t1, "colA", t2, "colB", - use_threads=use_threads, coalesce_keys=True) + r = _perform_join(jointype, t1, "colA", t2, "colB", + use_threads=use_threads, coalesce_keys=True) r = r.combine_chunks() if "right" in jointype: r = r.sort_by("colB") @@ -133,7 +133,7 @@ def test_table_join_collisions(): "colA": [99, 2, 1], }) - result = ep._perform_join( + result = _perform_join( "full outer", t1, ["colA", "colB"], t2, ["colA", "colB"]) result = result.combine_chunks() result = result.sort_by("colUniq") @@ -147,9 +147,9 @@ def test_table_join_collisions(): [99, 2, 1, None], ], names=["colA", "colB", "colVals", "colB", "colVals", "colUniq", "colA"]) - result = ep._perform_join("full outer", t1, "colA", - t2, "colA", right_suffix="_r", - coalesce_keys=False) + result = _perform_join("full outer", t1, "colA", + t2, "colA", right_suffix="_r", + coalesce_keys=False) result = result.combine_chunks() result = result.sort_by("colA") assert result == pa.table({ @@ -162,9 +162,9 @@ def test_table_join_collisions(): "colA_r": [1, 2, None, 99], }) - result = ep._perform_join("full outer", t1, "colA", - t2, "colA", right_suffix="_r", - coalesce_keys=True) + result = _perform_join("full outer", t1, "colA", + t2, "colA", right_suffix="_r", + coalesce_keys=True) result = result.combine_chunks() result = result.sort_by("colA") assert result == pa.table({ @@ -189,9 +189,9 @@ def test_table_join_keys_order(): "colX": [99, 2, 1], }) - result = ep._perform_join("full outer", t1, "colA", t2, "colX", - left_suffix="_l", right_suffix="_r", - coalesce_keys=True) + result = _perform_join("full outer", t1, "colA", t2, "colX", + left_suffix="_l", right_suffix="_r", + coalesce_keys=True) result = result.combine_chunks() result = result.sort_by("colA") assert result == pa.table({ @@ -209,10 +209,10 @@ def test_filter_table_errors(): }) with pytest.raises(pa.ArrowTypeError): - ep._filter_table(t, pc.divide(pc.field("a"), pc.scalar(2))) + _filter_table(t, pc.divide(pc.field("a"), pc.scalar(2))) with pytest.raises(pa.ArrowInvalid): - ep._filter_table(t, (pc.field("Z") <= pc.scalar(2))) + _filter_table(t, (pc.field("Z") <= pc.scalar(2))) def test_filter_table(): @@ -221,7 +221,7 @@ def test_filter_table(): "b": [10, 20, 30, 40, 50] }) - result = ep._filter_table( + result = _filter_table( t, (pc.field("a") <= pc.scalar(3)) & (pc.field("b") == pc.scalar(20)), ) assert result == pa.table({ @@ -229,7 +229,7 @@ def test_filter_table(): "b": [20] }) - result = ep._filter_table(t, pc.field("b") > pc.scalar(30)) + result = _filter_table(t, pc.field("b") > pc.scalar(30)) assert result == pa.table({ "a": [4, 5], "b": [40, 50] @@ -245,7 +245,7 @@ def test_filter_table_ordering(): # 20 seems to consistently cause errors when order is not preserved. # If the order problem is reintroduced this test will become flaky # which is still a signal that the order is not preserved. - r = ep._filter_table(table, pc.field('a') == 1) + r = _filter_table(table, pc.field('a') == 1) assert r["b"] == pa.chunked_array([["a"], ["b"]]) @@ -255,7 +255,7 @@ def test_complex_filter_table(): "b": [10, 20, 30, 40, 50, 60, 61] }) - result = ep._filter_table( + result = _filter_table( t, ((pc.bit_wise_and(pc.field("a"), pc.scalar(1)) == pc.scalar(0)) & (pc.multiply(pc.field("a"), pc.scalar(10)) == pc.field("b"))) ) @@ -289,18 +289,18 @@ def test_join_extension_array_column(): "colD": dict_array, }) - result = ep._perform_join( + result = _perform_join( "left outer", t1, ["colA"], t2, ["colA"]) assert result["colVals"] == pa.chunked_array(ext_array) - result = ep._perform_join( + result = _perform_join( "left outer", t1, ["colB"], t2, ["colC"]) assert result["colB"] == pa.chunked_array(ext_array) - result = ep._perform_join( + result = _perform_join( "left outer", t1, ["colA"], t3, ["colA"]) assert result["colVals"] == pa.chunked_array(ext_array) - result = ep._perform_join( + result = _perform_join( "left outer", t1, ["colB"], t3, ["colC"]) assert result["colB"] == pa.chunked_array(ext_array)