From 34aa3c401746fab99598890d23d5bf9797583a3c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 29 Mar 2023 17:18:14 +0200 Subject: [PATCH 1/5] GH-33976: [Python] Remove usage of TableGroupby helper in favor of pyarrow.acero.Declaration --- python/pyarrow/_compute.pyx | 32 ---------------------------- python/pyarrow/_exec_plan.py | 11 +++++++++- python/pyarrow/includes/libarrow.pxd | 5 ----- python/pyarrow/table.pxi | 11 +++------- 4 files changed, 13 insertions(+), 46 deletions(-) diff --git a/python/pyarrow/_compute.pyx b/python/pyarrow/_compute.pyx index dd980abad73c..a5db5be55145 100644 --- a/python/pyarrow/_compute.pyx +++ b/python/pyarrow/_compute.pyx @@ -2257,38 +2257,6 @@ class RankOptions(_RankOptions): self._set_options(sort_keys, null_placement, tiebreaker) -def _group_by(table, aggregates, keys): - cdef: - shared_ptr[CTable] c_table - vector[CAggregate] c_aggregates - vector[CFieldRef] c_keys - CAggregate c_aggr - - c_table = ( table).sp_table - - for aggr_arg_indices, aggr_func_name, aggr_opts, aggr_name in aggregates: - c_aggr.function = tobytes(aggr_func_name) - if aggr_opts is not None: - c_aggr.options = (aggr_opts).wrapped - else: - c_aggr.options = nullptr - for field_idx in aggr_arg_indices: - c_aggr.target.push_back(CFieldRef( field_idx)) - - c_aggr.name = tobytes(aggr_name) - c_aggregates.push_back(move(c_aggr)) - - for key_idx in keys: - c_keys.push_back(CFieldRef( key_idx)) - - with nogil: - sp_table = GetResultValue( - CTableGroupBy(c_table, c_aggregates, c_keys) - ) - - return pyarrow_wrap_table(sp_table) - - cdef class Expression(_Weakrefable): """ A logical expression to be evaluated against some input. diff --git a/python/pyarrow/_exec_plan.py b/python/pyarrow/_exec_plan.py index 6e49341a13cc..0cdbae9f1d1c 100644 --- a/python/pyarrow/_exec_plan.py +++ b/python/pyarrow/_exec_plan.py @@ -28,7 +28,7 @@ from pyarrow._dataset import ScanNodeOptions from pyarrow._acero import ( Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, - ProjectNodeOptions, OrderByNodeOptions + ProjectNodeOptions, OrderByNodeOptions, AggregateNodeOptions ) @@ -264,3 +264,12 @@ def _sort_source(table_or_dataset, sort_keys, output_type=Table, **kwargs): return InMemoryDataset(result_table.select(table_or_dataset.schema.names)) else: raise TypeError("Unsupported output type") + + +def _group_by(table, aggregates, keys): + + decl = Declaration.from_sequence([ + Declaration("table_source", TableSourceNodeOptions(table)), + Declaration("aggregate", AggregateNodeOptions(aggregates, keys=keys)) + ]) + return decl.to_table(use_threads=True) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index bf1907c354eb..4c9d396c2a31 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2545,11 +2545,6 @@ cdef extern from "arrow/compute/exec/groupby.h" namespace \ vector[CFieldRef] target c_string name - CResult[shared_ptr[CTable]] CTableGroupBy "arrow::compute::TableGroupBy"( - shared_ptr[CTable] table, - vector[CAggregate] aggregates, - vector[CFieldRef] keys) - cdef extern from * namespace "arrow::compute": # inlined from expression_internal.h to avoid diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 278025a04ede..60c6755ffe20 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5609,11 +5609,6 @@ list[tuple(str, str, FunctionOptions)] aggr_name = func_nohash else: aggr_name = "_".join(target) + "_" + func_nohash - # Calculate target indices by resolving field names - target_indices = [ - self._table.schema.get_field_index(f) for f in target] - group_by_aggrs.append((target_indices, func, opt, aggr_name)) - - key_indices = [ - self._table.schema.get_field_index(k) for k in self.keys] - return _pc()._group_by(self._table, group_by_aggrs, key_indices) + group_by_aggrs.append((target, func, opt, aggr_name)) + + return _pc()._exec_plan()._group_by(self._table, group_by_aggrs, self.keys) From 7168fb804024e55c31f7f96886de36ea4ab07786 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 29 Mar 2023 17:32:03 +0200 Subject: [PATCH 2/5] fixup --- python/pyarrow/compute.py | 1 - python/pyarrow/table.pxi | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index efc1e1d95628..b42e0f30a2a5 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -80,7 +80,6 @@ function_registry, get_function, list_functions, - _group_by, # Udf call_tabular_function, register_scalar_function, diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 60c6755ffe20..d5ee88c21897 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5611,4 +5611,4 @@ list[tuple(str, str, FunctionOptions)] aggr_name = "_".join(target) + "_" + func_nohash group_by_aggrs.append((target, func, opt, aggr_name)) - return _pc()._exec_plan()._group_by(self._table, group_by_aggrs, self.keys) + return _pc()._exec_plan._group_by(self._table, group_by_aggrs, self.keys) From 34cb33c2341f24322374ca56068ff705b0039b5c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 29 Mar 2023 18:08:47 +0200 Subject: [PATCH 3/5] fix import --- python/pyarrow/table.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 1fb36155a34b..50df4b29e468 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5613,4 +5613,4 @@ list[tuple(str, str, FunctionOptions)] aggr_name = "_".join(target) + "_" + func_nohash group_by_aggrs.append((target, func, opt, aggr_name)) - return _pc()._exec_plan._group_by(self._table, group_by_aggrs, self.keys) + return _pac()._group_by(self._table, group_by_aggrs, self.keys) From 3a71e6a4f6e4e700b552ac6d80e028f086556b26 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 4 Apr 2023 13:38:55 +0200 Subject: [PATCH 4/5] move CAggregate declaration to libarrow --- python/pyarrow/includes/libarrow.pxd | 6 ++++++ python/pyarrow/includes/libarrow_acero.pxd | 9 --------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 798f0e8395de..101a7157af2f 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2343,6 +2343,12 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: CIndexOptions(shared_ptr[CScalar] value) shared_ptr[CScalar] value + cdef cppclass CAggregate "arrow::compute::Aggregate": + c_string function + shared_ptr[CFunctionOptions] options + vector[CFieldRef] target + c_string name + cdef enum CMapLookupOccurrence \ "arrow::compute::MapLookupOptions::Occurrence": CMapLookupOccurrence_ALL "arrow::compute::MapLookupOptions::ALL" diff --git a/python/pyarrow/includes/libarrow_acero.pxd b/python/pyarrow/includes/libarrow_acero.pxd index b2b1739618b7..8ae00d4ae22f 100644 --- a/python/pyarrow/includes/libarrow_acero.pxd +++ b/python/pyarrow/includes/libarrow_acero.pxd @@ -21,15 +21,6 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * -cdef extern from "arrow/acero/groupby.h" namespace \ - "arrow::acero" nogil: - cdef cppclass CAggregate "arrow::compute::Aggregate": - c_string function - shared_ptr[CFunctionOptions] options - vector[CFieldRef] target - c_string name - - cdef extern from "arrow/acero/options.h" namespace "arrow::acero" nogil: cdef enum CJoinType "arrow::acero::JoinType": CJoinType_LEFT_SEMI "arrow::acero::JoinType::LEFT_SEMI" From e6eefac335c633eebe675dc141f9577ceff8aae2 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 23 May 2023 14:43:01 +0200 Subject: [PATCH 5/5] remove groupby.cc, rename test file --- cpp/src/arrow/acero/CMakeLists.txt | 3 +- cpp/src/arrow/acero/aggregate_benchmark.cc | 15 ++++- ...groupby_test.cc => aggregate_node_test.cc} | 14 +++- cpp/src/arrow/acero/groupby.cc | 61 ----------------- cpp/src/arrow/acero/groupby.h | 65 ------------------- 5 files changed, 28 insertions(+), 130 deletions(-) rename cpp/src/arrow/acero/{groupby_test.cc => aggregate_node_test.cc} (90%) delete mode 100644 cpp/src/arrow/acero/groupby.cc delete mode 100644 cpp/src/arrow/acero/groupby.h diff --git a/cpp/src/arrow/acero/CMakeLists.txt b/cpp/src/arrow/acero/CMakeLists.txt index 258dcb5580c3..b86b353f865d 100644 --- a/cpp/src/arrow/acero/CMakeLists.txt +++ b/cpp/src/arrow/acero/CMakeLists.txt @@ -28,7 +28,6 @@ macro(append_acero_avx2_src SRC) endmacro() set(ARROW_ACERO_SRCS - groupby.cc accumulation_queue.cc aggregate_node.cc asof_join_node.cc @@ -166,7 +165,7 @@ add_arrow_acero_test(pivot_longer_node_test SOURCES pivot_longer_node_test.cc add_arrow_acero_test(asof_join_node_test SOURCES asof_join_node_test.cc test_nodes.cc) add_arrow_acero_test(tpch_node_test SOURCES tpch_node_test.cc) add_arrow_acero_test(union_node_test SOURCES union_node_test.cc) -add_arrow_acero_test(groupby_test SOURCES groupby_test.cc) +add_arrow_acero_test(aggregate_node_test SOURCES aggregate_node_test.cc) add_arrow_acero_test(util_test SOURCES util_test.cc task_util_test.cc) add_arrow_acero_test(hash_aggregate_test SOURCES hash_aggregate_test.cc) diff --git a/cpp/src/arrow/acero/aggregate_benchmark.cc b/cpp/src/arrow/acero/aggregate_benchmark.cc index d65b52fe5208..4db7e443227d 100644 --- a/cpp/src/arrow/acero/aggregate_benchmark.cc +++ b/cpp/src/arrow/acero/aggregate_benchmark.cc @@ -19,7 +19,8 @@ #include -#include "arrow/acero/groupby.h" +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/options.h" #include "arrow/array/array_primitive.h" #include "arrow/compute/api.h" #include "arrow/table.h" @@ -347,6 +348,18 @@ std::shared_ptr RecordBatchFromArrays( return RecordBatch::Make(schema(std::move(fields)), length, std::move(all_arrays)); } +Result> BatchGroupBy( + std::shared_ptr batch, std::vector aggregates, + std::vector keys, bool use_threads = false, + MemoryPool* memory_pool = default_memory_pool()) { + ARROW_ASSIGN_OR_RAISE(std::shared_ptr
table, + Table::FromRecordBatches({std::move(batch)})); + Declaration plan = Declaration::Sequence( + {{"table_source", TableSourceNodeOptions(std::move(table))}, + {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); + return DeclarationToTable(std::move(plan), use_threads, memory_pool); +} + static void BenchmarkGroupBy(benchmark::State& state, std::vector aggregates, const std::vector>& arguments, const std::vector>& keys) { diff --git a/cpp/src/arrow/acero/groupby_test.cc b/cpp/src/arrow/acero/aggregate_node_test.cc similarity index 90% rename from cpp/src/arrow/acero/groupby_test.cc rename to cpp/src/arrow/acero/aggregate_node_test.cc index 7fe67a1f0d2d..5b6303887424 100644 --- a/cpp/src/arrow/acero/groupby_test.cc +++ b/cpp/src/arrow/acero/aggregate_node_test.cc @@ -14,13 +14,15 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include "arrow/acero/groupby.h" +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/options.h" #include #include #include +#include "arrow/result.h" #include "arrow/table.h" #include "arrow/testing/gtest_util.h" @@ -28,6 +30,16 @@ namespace arrow { namespace acero { +Result> TableGroupBy( + std::shared_ptr
table, std::vector aggregates, + std::vector keys, bool use_threads = false, + MemoryPool* memory_pool = default_memory_pool()) { + Declaration plan = Declaration::Sequence( + {{"table_source", TableSourceNodeOptions(std::move(table))}, + {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); + return DeclarationToTable(std::move(plan), use_threads, memory_pool); +} + TEST(GroupByConvenienceFunc, Basic) { std::shared_ptr in_schema = schema({field("key1", utf8()), field("key2", int32()), field("value", int32())}); diff --git a/cpp/src/arrow/acero/groupby.cc b/cpp/src/arrow/acero/groupby.cc deleted file mode 100644 index a0b220d65379..000000000000 --- a/cpp/src/arrow/acero/groupby.cc +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "arrow/acero/groupby.h" - -#include -#include -#include - -#include "arrow/compute/exec_internal.h" -#include "arrow/compute/registry.h" -#include "arrow/compute/row/grouper.h" -#include "arrow/record_batch.h" -#include "arrow/table.h" -#include "arrow/util/checked_cast.h" -#include "arrow/util/logging.h" -#include "arrow/util/string.h" -#include "arrow/util/task_group.h" - -namespace arrow { - -namespace acero { - -Result> TableGroupBy(std::shared_ptr
table, - std::vector aggregates, - std::vector keys, bool use_threads, - MemoryPool* memory_pool) { - Declaration plan = Declaration::Sequence( - {{"table_source", TableSourceNodeOptions(std::move(table))}, - {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); - return DeclarationToTable(std::move(plan), use_threads, memory_pool); -} - -Result> BatchGroupBy(std::shared_ptr batch, - std::vector aggregates, - std::vector keys, bool use_threads, - MemoryPool* memory_pool) { - ARROW_ASSIGN_OR_RAISE(std::shared_ptr
table, - Table::FromRecordBatches({std::move(batch)})); - Declaration plan = Declaration::Sequence( - {{"table_source", TableSourceNodeOptions(std::move(table))}, - {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); - return DeclarationToTable(std::move(plan), use_threads, memory_pool); -} - -} // namespace acero -} // namespace arrow diff --git a/cpp/src/arrow/acero/groupby.h b/cpp/src/arrow/acero/groupby.h deleted file mode 100644 index c24990ad69d7..000000000000 --- a/cpp/src/arrow/acero/groupby.h +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "arrow/acero/exec_plan.h" -#include "arrow/acero/options.h" -#include "arrow/acero/visibility.h" -#include "arrow/compute/api_aggregate.h" -#include "arrow/compute/exec.h" -#include "arrow/compute/kernel.h" -#include "arrow/datum.h" -#include "arrow/result.h" - -namespace arrow { -namespace acero { - -/// Convenience function to perform a group-by on a table -/// -/// The result will be calculated using an exec plan with an aggregate node. -/// -/// If there are no arguments/aggregates then the returned table will have one row -/// for each unique combination of keys -/// -/// Note: If there are many groups the output table may have multiple chunks. -/// -/// If there are no keys then the aggregates will be applied to the full table. -/// The output table in this scenario is guaranteed to have exactly 1 row. -/// -/// \return a table that will have one column for each aggregate, named after they -/// aggregate function, and one column for each key -ARROW_ACERO_EXPORT -Result> TableGroupBy( - std::shared_ptr
table, std::vector aggregates, - std::vector keys, bool use_threads = false, - MemoryPool* memory_pool = default_memory_pool()); - -/// Convenience function to perform a group-by on a record batch -/// -/// \see GroupByTable -ARROW_ACERO_EXPORT -Result> BatchGroupBy( - std::shared_ptr record_batch, std::vector aggregates, - std::vector keys, bool use_threads = false, - MemoryPool* memory_pool = default_memory_pool()); - -} // namespace acero -} // namespace arrow