diff --git a/cpp/src/arrow/acero/CMakeLists.txt b/cpp/src/arrow/acero/CMakeLists.txt index 258dcb5580c3..b86b353f865d 100644 --- a/cpp/src/arrow/acero/CMakeLists.txt +++ b/cpp/src/arrow/acero/CMakeLists.txt @@ -28,7 +28,6 @@ macro(append_acero_avx2_src SRC) endmacro() set(ARROW_ACERO_SRCS - groupby.cc accumulation_queue.cc aggregate_node.cc asof_join_node.cc @@ -166,7 +165,7 @@ add_arrow_acero_test(pivot_longer_node_test SOURCES pivot_longer_node_test.cc add_arrow_acero_test(asof_join_node_test SOURCES asof_join_node_test.cc test_nodes.cc) add_arrow_acero_test(tpch_node_test SOURCES tpch_node_test.cc) add_arrow_acero_test(union_node_test SOURCES union_node_test.cc) -add_arrow_acero_test(groupby_test SOURCES groupby_test.cc) +add_arrow_acero_test(aggregate_node_test SOURCES aggregate_node_test.cc) add_arrow_acero_test(util_test SOURCES util_test.cc task_util_test.cc) add_arrow_acero_test(hash_aggregate_test SOURCES hash_aggregate_test.cc) diff --git a/cpp/src/arrow/acero/aggregate_benchmark.cc b/cpp/src/arrow/acero/aggregate_benchmark.cc index d65b52fe5208..4db7e443227d 100644 --- a/cpp/src/arrow/acero/aggregate_benchmark.cc +++ b/cpp/src/arrow/acero/aggregate_benchmark.cc @@ -19,7 +19,8 @@ #include -#include "arrow/acero/groupby.h" +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/options.h" #include "arrow/array/array_primitive.h" #include "arrow/compute/api.h" #include "arrow/table.h" @@ -347,6 +348,18 @@ std::shared_ptr RecordBatchFromArrays( return RecordBatch::Make(schema(std::move(fields)), length, std::move(all_arrays)); } +Result> BatchGroupBy( + std::shared_ptr batch, std::vector aggregates, + std::vector keys, bool use_threads = false, + MemoryPool* memory_pool = default_memory_pool()) { + ARROW_ASSIGN_OR_RAISE(std::shared_ptr table, + Table::FromRecordBatches({std::move(batch)})); + Declaration plan = Declaration::Sequence( + {{"table_source", TableSourceNodeOptions(std::move(table))}, + {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); + return DeclarationToTable(std::move(plan), use_threads, memory_pool); +} + static void BenchmarkGroupBy(benchmark::State& state, std::vector aggregates, const std::vector>& arguments, const std::vector>& keys) { diff --git a/cpp/src/arrow/acero/groupby_test.cc b/cpp/src/arrow/acero/aggregate_node_test.cc similarity index 90% rename from cpp/src/arrow/acero/groupby_test.cc rename to cpp/src/arrow/acero/aggregate_node_test.cc index 7fe67a1f0d2d..5b6303887424 100644 --- a/cpp/src/arrow/acero/groupby_test.cc +++ b/cpp/src/arrow/acero/aggregate_node_test.cc @@ -14,13 +14,15 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include "arrow/acero/groupby.h" +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/options.h" #include #include #include +#include "arrow/result.h" #include "arrow/table.h" #include "arrow/testing/gtest_util.h" @@ -28,6 +30,16 @@ namespace arrow { namespace acero { +Result> TableGroupBy( + std::shared_ptr
table, std::vector aggregates, + std::vector keys, bool use_threads = false, + MemoryPool* memory_pool = default_memory_pool()) { + Declaration plan = Declaration::Sequence( + {{"table_source", TableSourceNodeOptions(std::move(table))}, + {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); + return DeclarationToTable(std::move(plan), use_threads, memory_pool); +} + TEST(GroupByConvenienceFunc, Basic) { std::shared_ptr in_schema = schema({field("key1", utf8()), field("key2", int32()), field("value", int32())}); diff --git a/cpp/src/arrow/acero/groupby.cc b/cpp/src/arrow/acero/groupby.cc deleted file mode 100644 index a0b220d65379..000000000000 --- a/cpp/src/arrow/acero/groupby.cc +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "arrow/acero/groupby.h" - -#include -#include -#include - -#include "arrow/compute/exec_internal.h" -#include "arrow/compute/registry.h" -#include "arrow/compute/row/grouper.h" -#include "arrow/record_batch.h" -#include "arrow/table.h" -#include "arrow/util/checked_cast.h" -#include "arrow/util/logging.h" -#include "arrow/util/string.h" -#include "arrow/util/task_group.h" - -namespace arrow { - -namespace acero { - -Result> TableGroupBy(std::shared_ptr
table, - std::vector aggregates, - std::vector keys, bool use_threads, - MemoryPool* memory_pool) { - Declaration plan = Declaration::Sequence( - {{"table_source", TableSourceNodeOptions(std::move(table))}, - {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); - return DeclarationToTable(std::move(plan), use_threads, memory_pool); -} - -Result> BatchGroupBy(std::shared_ptr batch, - std::vector aggregates, - std::vector keys, bool use_threads, - MemoryPool* memory_pool) { - ARROW_ASSIGN_OR_RAISE(std::shared_ptr
table, - Table::FromRecordBatches({std::move(batch)})); - Declaration plan = Declaration::Sequence( - {{"table_source", TableSourceNodeOptions(std::move(table))}, - {"aggregate", AggregateNodeOptions(std::move(aggregates), std::move(keys))}}); - return DeclarationToTable(std::move(plan), use_threads, memory_pool); -} - -} // namespace acero -} // namespace arrow diff --git a/cpp/src/arrow/acero/groupby.h b/cpp/src/arrow/acero/groupby.h deleted file mode 100644 index c24990ad69d7..000000000000 --- a/cpp/src/arrow/acero/groupby.h +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "arrow/acero/exec_plan.h" -#include "arrow/acero/options.h" -#include "arrow/acero/visibility.h" -#include "arrow/compute/api_aggregate.h" -#include "arrow/compute/exec.h" -#include "arrow/compute/kernel.h" -#include "arrow/datum.h" -#include "arrow/result.h" - -namespace arrow { -namespace acero { - -/// Convenience function to perform a group-by on a table -/// -/// The result will be calculated using an exec plan with an aggregate node. -/// -/// If there are no arguments/aggregates then the returned table will have one row -/// for each unique combination of keys -/// -/// Note: If there are many groups the output table may have multiple chunks. -/// -/// If there are no keys then the aggregates will be applied to the full table. -/// The output table in this scenario is guaranteed to have exactly 1 row. -/// -/// \return a table that will have one column for each aggregate, named after they -/// aggregate function, and one column for each key -ARROW_ACERO_EXPORT -Result> TableGroupBy( - std::shared_ptr
table, std::vector aggregates, - std::vector keys, bool use_threads = false, - MemoryPool* memory_pool = default_memory_pool()); - -/// Convenience function to perform a group-by on a record batch -/// -/// \see GroupByTable -ARROW_ACERO_EXPORT -Result> BatchGroupBy( - std::shared_ptr record_batch, std::vector aggregates, - std::vector keys, bool use_threads = false, - MemoryPool* memory_pool = default_memory_pool()); - -} // namespace acero -} // namespace arrow diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index af39ce361fc8..bb3196c86ef0 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -527,35 +527,3 @@ cdef class Declaration(_Weakrefable): GetResultValue(DeclarationToReader(self.unwrap(), use_threads)).release() ) return reader - - -def _group_by(table, aggregates, keys): - cdef: - shared_ptr[CTable] c_table - vector[CAggregate] c_aggregates - vector[CFieldRef] c_keys - CAggregate c_aggr - - c_table = (
table).sp_table - - for aggr_arg_indices, aggr_func_name, aggr_opts, aggr_name in aggregates: - c_aggr.function = tobytes(aggr_func_name) - if aggr_opts is not None: - c_aggr.options = (aggr_opts).wrapped - else: - c_aggr.options = nullptr - for field_idx in aggr_arg_indices: - c_aggr.target.push_back(CFieldRef( field_idx)) - - c_aggr.name = tobytes(aggr_name) - c_aggregates.push_back(move(c_aggr)) - - for key_idx in keys: - c_keys.push_back(CFieldRef( key_idx)) - - with nogil: - sp_table = GetResultValue( - CTableGroupBy(c_table, c_aggregates, c_keys) - ) - - return pyarrow_wrap_table(sp_table) diff --git a/python/pyarrow/acero.py b/python/pyarrow/acero.py index 04c396a26feb..0150148b5047 100644 --- a/python/pyarrow/acero.py +++ b/python/pyarrow/acero.py @@ -26,7 +26,6 @@ from pyarrow.compute import Expression, field from pyarrow._acero import ( # noqa - _group_by, Declaration, ExecNodeOptions, TableSourceNodeOptions, @@ -292,3 +291,12 @@ def _sort_source(table_or_dataset, sort_keys, output_type=Table, **kwargs): return ds.InMemoryDataset(result_table) else: raise TypeError("Unsupported output type") + + +def _group_by(table, aggregates, keys): + + decl = Declaration.from_sequence([ + Declaration("table_source", TableSourceNodeOptions(table)), + Declaration("aggregate", AggregateNodeOptions(aggregates, keys=keys)) + ]) + return decl.to_table(use_threads=True) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8fc531dff00b..3190877ea099 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2346,6 +2346,12 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: CIndexOptions(shared_ptr[CScalar] value) shared_ptr[CScalar] value + cdef cppclass CAggregate "arrow::compute::Aggregate": + c_string function + shared_ptr[CFunctionOptions] options + vector[CFieldRef] target + c_string name + cdef enum CMapLookupOccurrence \ "arrow::compute::MapLookupOptions::Occurrence": CMapLookupOccurrence_ALL "arrow::compute::MapLookupOptions::ALL" diff --git a/python/pyarrow/includes/libarrow_acero.pxd b/python/pyarrow/includes/libarrow_acero.pxd index 920668cdd024..bb1e3646c91b 100644 --- a/python/pyarrow/includes/libarrow_acero.pxd +++ b/python/pyarrow/includes/libarrow_acero.pxd @@ -21,20 +21,6 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * -cdef extern from "arrow/acero/groupby.h" namespace \ - "arrow::acero" nogil: - cdef cppclass CAggregate "arrow::compute::Aggregate": - c_string function - shared_ptr[CFunctionOptions] options - vector[CFieldRef] target - c_string name - - CResult[shared_ptr[CTable]] CTableGroupBy "arrow::acero::TableGroupBy"( - shared_ptr[CTable] table, - vector[CAggregate] aggregates, - vector[CFieldRef] keys) - - cdef extern from "arrow/acero/options.h" namespace "arrow::acero" nogil: cdef enum CJoinType "arrow::acero::JoinType": CJoinType_LEFT_SEMI "arrow::acero::JoinType::LEFT_SEMI" diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index e051a53f701b..5f1ee0020158 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5430,11 +5430,6 @@ list[tuple(str, str, FunctionOptions)] aggr_name = func_nohash else: aggr_name = "_".join(target) + "_" + func_nohash - # Calculate target indices by resolving field names - target_indices = [ - self._table.schema.get_field_index(f) for f in target] - group_by_aggrs.append((target_indices, func, opt, aggr_name)) - - key_indices = [ - self._table.schema.get_field_index(k) for k in self.keys] - return _pac()._group_by(self._table, group_by_aggrs, key_indices) + group_by_aggrs.append((target, func, opt, aggr_name)) + + return _pac()._group_by(self._table, group_by_aggrs, self.keys)