From 5bf011ea8a531c9a632912a79d265deedf10e769 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 27 Feb 2023 11:15:18 +0100 Subject: [PATCH 1/5] refactor exec_plan --- python/pyarrow/_exec_plan.pyx | 224 ++++++++++--------------- python/pyarrow/table.pxi | 3 +- python/pyarrow/tests/test_exec_plan.py | 25 +-- 3 files changed, 89 insertions(+), 163 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 9ef268779482..f29067e580e5 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -33,6 +33,8 @@ from pyarrow._compute cimport Expression, _true, _SortOptions from pyarrow._dataset cimport Dataset, Scanner from pyarrow._dataset import InMemoryDataset +from pyarrow._acero import Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, ProjectNodeOptions + Initialize() # Initialise support for Datasets in ExecPlan @@ -231,163 +233,121 @@ def _perform_join(join_type, left_operand not None, left_keys, ------- result_table : Table or InMemoryDataset """ - cdef: - vector[CFieldRef] c_left_keys - vector[CFieldRef] c_right_keys - vector[CFieldRef] c_left_columns - vector[CFieldRef] c_right_columns - vector[CDeclaration] c_decl_plan - vector[CExpression] c_projections - vector[c_string] c_projected_col_names - CJoinType c_join_type - # Prepare left and right tables Keys to send them to the C++ function left_keys_order = {} - if isinstance(left_keys, str): + if not isinstance(left_keys, (tuple, list)): left_keys = [left_keys] for idx, key in enumerate(left_keys): left_keys_order[key] = idx - c_left_keys.push_back(CFieldRef(tobytes(key))) right_keys_order = {} - if isinstance(right_keys, str): + if not isinstance(right_keys, (list, tuple)): right_keys = [right_keys] for idx, key in enumerate(right_keys): right_keys_order[key] = idx - c_right_keys.push_back(CFieldRef(tobytes(key))) # By default expose all columns on both left and right table - if isinstance(left_operand, Table): - left_columns = left_operand.column_names - elif isinstance(left_operand, Dataset): - left_columns = left_operand.schema.names - else: - raise TypeError("Unsupported left join member type") - - if isinstance(right_operand, Table): - right_columns = right_operand.column_names - elif isinstance(right_operand, Dataset): - right_columns = right_operand.schema.names - else: - raise TypeError("Unsupported right join member type") + left_columns = left_operand.schema.names + right_columns = right_operand.schema.names # Pick the join type - if join_type == "left semi": - c_join_type = CJoinType_LEFT_SEMI - right_columns = [] - elif join_type == "right semi": - c_join_type = CJoinType_RIGHT_SEMI - left_columns = [] - elif join_type == "left anti": - c_join_type = CJoinType_LEFT_ANTI + if join_type == "left semi" or join_type == "left anti": right_columns = [] - elif join_type == "right anti": - c_join_type = CJoinType_RIGHT_ANTI + elif join_type == "right semi" or join_type == "right anti": left_columns = [] - elif join_type == "inner": - c_join_type = CJoinType_INNER - right_columns = [ - col for col in right_columns if col not in right_keys_order - ] - elif join_type == "left outer": - c_join_type = CJoinType_LEFT_OUTER + elif join_type == "inner" or join_type == "left outer": right_columns = [ col for col in right_columns if col not in right_keys_order ] elif join_type == "right outer": - c_join_type = CJoinType_RIGHT_OUTER left_columns = [ col for col in left_columns if col not in left_keys_order ] - elif join_type == "full outer": - c_join_type = CJoinType_FULL_OUTER - else: - raise ValueError("Unsupported join type") # Turn the columns to vectors of FieldRefs # and set aside indices of keys. left_column_keys_indices = {} for idx, colname in enumerate(left_columns): - c_left_columns.push_back(CFieldRef(tobytes(colname))) if colname in left_keys: left_column_keys_indices[colname] = idx right_column_keys_indices = {} for idx, colname in enumerate(right_columns): - c_right_columns.push_back(CFieldRef(tobytes(colname))) if colname in right_keys: right_column_keys_indices[colname] = idx # Add the join node to the execplan - if coalesce_keys: - c_decl_plan.push_back( - CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( - c_join_type, c_left_keys, c_right_keys, - c_left_columns, c_right_columns, - _true, - tobytes(left_suffix or ""), - tobytes(right_suffix or "") - )) - ) - if join_type == "full outer": - # In case of full outer joins, the join operation will output all columns - # so that we can coalesce the keys and exclude duplicates in a subsequent projection. - left_columns_set = set(left_columns) - right_columns_set = set(right_columns) - # Where the right table columns start. - right_operand_index = len(left_columns) - for idx, col in enumerate(left_columns + right_columns): - if idx < len(left_columns) and col in left_column_keys_indices: - # Include keys only once and coalesce left+right table keys. - c_projected_col_names.push_back(tobytes(col)) - # Get the index of the right key that is being paired - # with this left key. We do so by retrieving the name - # of the right key that is in the same position in the provided keys - # and then looking up the index for that name in the right table. - right_key_index = right_column_keys_indices[right_keys[left_keys_order[col]]] - c_projections.push_back(Expression.unwrap( - Expression._call("coalesce", [ - Expression._field(idx), Expression._field( - right_operand_index+right_key_index) - ]) - )) - elif idx >= right_operand_index and col in right_column_keys_indices: - # Do not include right table keys. As they would lead to duplicated keys. - continue - else: - # For all the other columns incude them as they are. - # Just recompute the suffixes that the join produced as the projection - # would lose them otherwise. - if left_suffix and idx < right_operand_index and col in right_columns_set: - col += left_suffix - if right_suffix and idx >= right_operand_index and col in left_columns_set: - col += right_suffix - c_projected_col_names.push_back(tobytes(col)) - c_projections.push_back( - Expression.unwrap(Expression._field(idx))) - c_decl_plan.push_back( - CDeclaration(tobytes("project"), CProjectNodeOptions( - c_projections, c_projected_col_names)) - ) - else: - c_decl_plan.push_back( - CDeclaration(tobytes("hashjoin"), CHashJoinNodeOptions( - c_join_type, c_left_keys, c_right_keys, - _true, - tobytes(left_suffix or ""), - tobytes(right_suffix or "") - )) + # TEMP + if isinstance(left_operand, Dataset): + left_operand = left_operand.to_table() + if isinstance(right_operand, Dataset): + right_operand = right_operand.to_table() + + left_source = Declaration("table_source", TableSourceNodeOptions(left_operand)) + right_source = Declaration("table_source", TableSourceNodeOptions(right_operand)) + + join_opts = HashJoinNodeOptions( + join_type, left_keys, right_keys, left_columns, right_columns, + output_suffix_for_left=left_suffix or "", + output_suffix_for_right=right_suffix or "", + ) + decl = Declaration("hashjoin", options=join_opts, inputs=[left_source, right_source]) + + if coalesce_keys and join_type == "full outer": + # In case of full outer joins, the join operation will output all columns + # so that we can coalesce the keys and exclude duplicates in a subsequent projection. + left_columns_set = set(left_columns) + right_columns_set = set(right_columns) + # Where the right table columns start. + right_operand_index = len(left_columns) + projected_col_names = [] + projections = [] + for idx, col in enumerate(left_columns + right_columns): + if idx < len(left_columns) and col in left_column_keys_indices: + # Include keys only once and coalesce left+right table keys. + projected_col_names.append(col) + # Get the index of the right key that is being paired + # with this left key. We do so by retrieving the name + # of the right key that is in the same position in the provided keys + # and then looking up the index for that name in the right table. + right_key_index = right_column_keys_indices[right_keys[left_keys_order[col]]] + projections.append( + Expression._call("coalesce", [ + Expression._field(idx), Expression._field( + right_operand_index+right_key_index) + ]) + ) + elif idx >= right_operand_index and col in right_column_keys_indices: + # Do not include right table keys. As they would lead to duplicated keys. + continue + else: + # For all the other columns incude them as they are. + # Just recompute the suffixes that the join produced as the projection + # would lose them otherwise. + if left_suffix and idx < right_operand_index and col in right_columns_set: + col += left_suffix + if right_suffix and idx >= right_operand_index and col in left_columns_set: + col += right_suffix + projected_col_names.append(col) + projections.append( + Expression._field(idx) + ) + projection = Declaration( + "project", ProjectNodeOptions(projections, projected_col_names) ) + decl = Declaration.from_sequence([decl, projection]) - result_table = execplan([left_operand, right_operand], - plan=c_decl_plan, - output_type=output_type, - use_threads=use_threads) + result_table = decl.to_table(use_threads=use_threads) - return result_table + if output_type == Table: + return result_table + elif output_type == InMemoryDataset: + return InMemoryDataset(result_table) + else: + raise TypeError("Unsupported output type") -def _filter_table(table, expression, output_type=Table): - """Filter rows of a table or dataset based on the provided expression. +def _filter_table(table, expression): + """Filter rows of a table based on the provided expression. The result will be an output table with only the rows matching the provided expression. @@ -398,34 +358,18 @@ def _filter_table(table, expression, output_type=Table): Table or Dataset that should be filtered. expression : Expression The expression on which rows should be filtered. - output_type: Table or InMemoryDataset - The output type for the filtered result. Returns ------- - result_table : Table or InMemoryDataset + Table """ - cdef: - vector[CDeclaration] c_decl_plan - Expression expr = expression - - c_decl_plan.push_back( - CDeclaration(tobytes("filter"), CFilterNodeOptions( - expr.unwrap() - )) - ) - - r = execplan([table], plan=c_decl_plan, - output_type=Table, use_threads=False) - - if output_type == Table: - return r - elif output_type == InMemoryDataset: - # Get rid of special dataset columns - # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" - return InMemoryDataset(r.select(table.schema.names)) - else: - raise TypeError("Unsupported output type") + decl = Declaration.from_sequence([ + Declaration("table_source", options=TableSourceNodeOptions(table)), + Declaration("filter", options=FilterNodeOptions(expression)) + ]) + # TODO use_threads is set to False because this doesn't yet support + # preserving the order of the in-memory table's batches + return decl.to_table(use_threads=False) def _sort_source(table_or_dataset, sort_options, output_type=Table): diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5af381e60841..9da7110a1d10 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -3145,8 +3145,7 @@ cdef class Table(_PandasConvertible): animals: [["Flamingo","Horse",null]] """ if isinstance(mask, _pc().Expression): - return _pc()._exec_plan._filter_table(self, mask, - output_type=Table) + return _pc()._exec_plan._filter_table(self, mask) else: return _pc().filter(self, mask, null_selection_behavior) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 7875dff55755..b57de8b3c28e 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -209,44 +209,27 @@ def test_filter_table_errors(): }) with pytest.raises(pa.ArrowTypeError): - ep._filter_table( - t, pc.divide(pc.field("a"), pc.scalar(2)), - output_type=pa.Table - ) + ep._filter_table(t, pc.divide(pc.field("a"), pc.scalar(2))) with pytest.raises(pa.ArrowInvalid): - ep._filter_table( - t, (pc.field("Z") <= pc.scalar(2)), - output_type=pa.Table - ) + ep._filter_table(t, (pc.field("Z") <= pc.scalar(2))) -@pytest.mark.parametrize("use_datasets", [False, True]) -def test_filter_table(use_datasets): +def test_filter_table(): t = pa.table({ "a": [1, 2, 3, 4, 5], "b": [10, 20, 30, 40, 50] }) - if use_datasets: - t = ds.dataset([t]) result = ep._filter_table( t, (pc.field("a") <= pc.scalar(3)) & (pc.field("b") == pc.scalar(20)), - output_type=pa.Table if not use_datasets else ds.InMemoryDataset ) - if use_datasets: - result = result.to_table() assert result == pa.table({ "a": [2], "b": [20] }) - result = ep._filter_table( - t, pc.field("b") > pc.scalar(30), - output_type=pa.Table if not use_datasets else ds.InMemoryDataset - ) - if use_datasets: - result = result.to_table() + result = ep._filter_table(t, pc.field("b") > pc.scalar(30)) assert result == pa.table({ "a": [4, 5], "b": [40, 50] From e2d8c75e5df830bf35584d4aa041e6ce69e13fc4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 14 Mar 2023 09:40:28 +0100 Subject: [PATCH 2/5] use ScanNodeOptions for dataset input --- python/pyarrow/_exec_plan.pyx | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index f29067e580e5..0ae61392bcce 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -29,15 +29,29 @@ from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_dataset cimport * from pyarrow.lib cimport Table, check_status, pyarrow_unwrap_table, pyarrow_wrap_table from pyarrow.lib import tobytes -from pyarrow._compute cimport Expression, _true, _SortOptions +from pyarrow._compute cimport Expression, _SortOptions from pyarrow._dataset cimport Dataset, Scanner -from pyarrow._dataset import InMemoryDataset +from pyarrow._dataset import InMemoryDataset, ScanNodeOptions from pyarrow._acero import Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, ProjectNodeOptions Initialize() # Initialise support for Datasets in ExecPlan +def _dataset_to_decl(Dataset dataset, use_threads=True): + decl = Declaration("scan", ScanNodeOptions(dataset, use_threads=use_threads)) + + filter_expr = dataset._scan_options.get("filter") + if filter_expr is not None: + # Filters applied in CScanNodeOptions are "best effort" for the scan node itself, + # so we always need to inject an additional Filter node to apply them for real. + decl = Declaration.from_sequence( + [decl, Declaration("filter", FilterNodeOptions(filter_expr))] + ) + + return decl + + cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads=True, _SortOptions sort_options=None): """ @@ -276,14 +290,14 @@ def _perform_join(join_type, left_operand not None, left_keys, right_column_keys_indices[colname] = idx # Add the join node to the execplan - # TEMP if isinstance(left_operand, Dataset): - left_operand = left_operand.to_table() + left_source = _dataset_to_decl(left_operand, use_threads=use_threads) + else: + left_source = Declaration("table_source", TableSourceNodeOptions(left_operand)) if isinstance(right_operand, Dataset): - right_operand = right_operand.to_table() - - left_source = Declaration("table_source", TableSourceNodeOptions(left_operand)) - right_source = Declaration("table_source", TableSourceNodeOptions(right_operand)) + right_source = _dataset_to_decl(right_operand, use_threads=use_threads) + else: + right_source = Declaration("table_source", TableSourceNodeOptions(right_operand)) join_opts = HashJoinNodeOptions( join_type, left_keys, right_keys, left_columns, right_columns, From b2e85efc7318658bafd8b30baf28ab897a177718 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 22 Mar 2023 14:48:53 +0100 Subject: [PATCH 3/5] use OrderByNodeOptions for sort + remove now redundant execplan --- python/pyarrow/_dataset.pyx | 4 +- python/pyarrow/_exec_plan.pyx | 203 ++++------------------------------ 2 files changed, 24 insertions(+), 183 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 67b82c8c874a..d6fe91ba21ea 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -809,9 +809,7 @@ cdef class Dataset(_Weakrefable): sorting = [(sorting, "ascending")] res = _pc()._exec_plan._sort_source(self, output_type=InMemoryDataset, - sort_options=_pc().SortOptions( - sort_keys=sorting, **kwargs - )) + sort_keys=sorting, **kwargs) return res def join(self, right_dataset, keys, right_keys=None, join_type="left outer", diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 0ae61392bcce..bc170550ba75 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -22,23 +22,17 @@ # distutils: language = c++ # cython: language_level = 3 -from cython.operator cimport dereference as deref, preincrement as inc +from pyarrow.lib import Table +from pyarrow.compute import Expression +from pyarrow.dataset import Dataset, InMemoryDataset +from pyarrow._dataset import ScanNodeOptions +from pyarrow._acero import ( + Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, + ProjectNodeOptions, OrderByNodeOptions +) -from pyarrow.includes.common cimport * -from pyarrow.includes.libarrow cimport * -from pyarrow.includes.libarrow_dataset cimport * -from pyarrow.lib cimport Table, check_status, pyarrow_unwrap_table, pyarrow_wrap_table -from pyarrow.lib import tobytes -from pyarrow._compute cimport Expression, _SortOptions -from pyarrow._dataset cimport Dataset, Scanner -from pyarrow._dataset import InMemoryDataset, ScanNodeOptions -from pyarrow._acero import Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, ProjectNodeOptions - -Initialize() # Initialise support for Datasets in ExecPlan - - -def _dataset_to_decl(Dataset dataset, use_threads=True): +def _dataset_to_decl(dataset, use_threads=True): decl = Declaration("scan", ScanNodeOptions(dataset, use_threads=use_threads)) filter_expr = dataset._scan_options.get("filter") @@ -52,161 +46,6 @@ def _dataset_to_decl(Dataset dataset, use_threads=True): return decl -cdef execplan(inputs, output_type, vector[CDeclaration] plan, c_bool use_threads=True, - _SortOptions sort_options=None): - """ - Internal Function to create an ExecPlan and run it. - - Parameters - ---------- - inputs : list of Table or Dataset - The sources from which the ExecPlan should fetch data. - In most cases this is only one, unless the first node of the - plan is able to get data from multiple different sources. - output_type : Table or InMemoryDataset - In which format the output should be provided. - plan : vector[CDeclaration] - The nodes of the plan that should be applied to the sources - to produce the output. - use_threads : bool, default True - Whether to use multithreading or not. - """ - cdef: - CExecutor *c_executor - shared_ptr[CExecContext] c_exec_context - shared_ptr[CExecPlan] c_exec_plan - CDeclaration current_decl - vector[CDeclaration] c_decls - vector[CExecNode*] _empty - vector[CExecNode*] c_final_node_vec - CExecNode *c_node - shared_ptr[CTable] c_in_table - shared_ptr[CTable] c_out_table - shared_ptr[CTableSourceNodeOptions] c_tablesourceopts - shared_ptr[CScanNodeOptions] c_scanopts - shared_ptr[CExecNodeOptions] c_input_node_opts - shared_ptr[CSinkNodeOptions] c_sinkopts - shared_ptr[COrderBySinkNodeOptions] c_orderbysinkopts - shared_ptr[CAsyncExecBatchGenerator] c_async_exec_batch_gen - shared_ptr[CRecordBatchReader] c_recordbatchreader - vector[CDeclaration].iterator plan_iter - vector[CDeclaration.Input] no_c_inputs - CStatus c_plan_status - - if use_threads: - c_executor = GetCpuThreadPool() - else: - c_executor = NULL - - # TODO(weston): This is deprecated. Once ordering is better supported - # in the exec plan we can remove all references to ExecPlan and use the - # DeclarationToXyz methods - c_exec_context = make_shared[CExecContext]( - c_default_memory_pool(), c_executor) - c_exec_plan = GetResultValue(CExecPlan.Make(c_exec_context.get())) - - plan_iter = plan.begin() - - # Create source nodes for each input - for ipt in inputs: - if isinstance(ipt, Table): - c_in_table = pyarrow_unwrap_table(ipt) - c_tablesourceopts = make_shared[CTableSourceNodeOptions]( - c_in_table) - c_input_node_opts = static_pointer_cast[CExecNodeOptions, CTableSourceNodeOptions]( - c_tablesourceopts) - - current_decl = CDeclaration( - tobytes("table_source"), no_c_inputs, c_input_node_opts) - elif isinstance(ipt, Dataset): - c_in_dataset = (ipt).unwrap() - c_scanopts = make_shared[CScanNodeOptions]( - c_in_dataset, Scanner._make_scan_options(ipt, {"use_threads": use_threads})) - c_input_node_opts = static_pointer_cast[CExecNodeOptions, CScanNodeOptions]( - c_scanopts) - - # Filters applied in CScanNodeOptions are "best effort" for the scan node itself, - # so we always need to inject an additional Filter node to apply them for real. - current_decl = CDeclaration( - tobytes("filter"), - no_c_inputs, - static_pointer_cast[CExecNodeOptions, CFilterNodeOptions]( - make_shared[CFilterNodeOptions]( - deref(deref(c_scanopts).scan_options).filter - ) - ) - ) - current_decl.inputs.push_back( - CDeclaration.Input( - CDeclaration(tobytes("scan"), no_c_inputs, c_input_node_opts)) - ) - else: - raise TypeError("Unsupported type") - - if plan_iter != plan.end(): - # Flag the source as the input of the first plan node. - deref(plan_iter).inputs.push_back(CDeclaration.Input(current_decl)) - else: - # Empty plan, make the source the first plan node. - c_decls.push_back(current_decl) - - # Add Here additional nodes - while plan_iter != plan.end(): - c_decls.push_back(deref(plan_iter)) - inc(plan_iter) - - # Add all CDeclarations to the plan - c_node = GetResultValue( - CDeclaration.Sequence(c_decls).AddToPlan(&deref(c_exec_plan)) - ) - c_final_node_vec.push_back(c_node) - - # Create the output node - c_async_exec_batch_gen = make_shared[CAsyncExecBatchGenerator]() - - if sort_options is None: - c_sinkopts = make_shared[CSinkNodeOptions]( - c_async_exec_batch_gen.get()) - GetResultValue( - MakeExecNode(tobytes("sink"), &deref(c_exec_plan), - c_final_node_vec, deref(c_sinkopts)) - ) - else: - c_orderbysinkopts = make_shared[COrderBySinkNodeOptions]( - deref((sort_options.unwrap().get())), - c_async_exec_batch_gen.get() - ) - GetResultValue( - MakeExecNode(tobytes("order_by_sink"), &deref(c_exec_plan), - c_final_node_vec, deref(c_orderbysinkopts)) - ) - - # Convert the asyncgenerator to a sync batch reader - c_recordbatchreader = MakeGeneratorReader(c_node.output_schema(), - deref(c_async_exec_batch_gen), - deref(c_exec_context).memory_pool()) - - # Start execution of the ExecPlan - deref(c_exec_plan).Validate() - deref(c_exec_plan).StartProducing() - - # Convert output to the expected one. - c_out_table = GetResultValue( - CTable.FromRecordBatchReader(c_recordbatchreader.get())) - if output_type == Table: - output = pyarrow_wrap_table(c_out_table) - elif output_type == InMemoryDataset: - output = InMemoryDataset(pyarrow_wrap_table(c_out_table)) - else: - raise TypeError("Unsupported output type") - - with nogil: - c_plan_status = deref(c_exec_plan).finished().status() - check_status(c_plan_status) - - return output - - def _perform_join(join_type, left_operand not None, left_keys, right_operand not None, right_keys, left_suffix=None, right_suffix=None, @@ -386,21 +225,25 @@ def _filter_table(table, expression): return decl.to_table(use_threads=False) -def _sort_source(table_or_dataset, sort_options, output_type=Table): - cdef: - vector[CDeclaration] c_empty_decl_plan +def _sort_source(table_or_dataset, sort_keys, output_type=Table, **kwargs): + + if isinstance(table_or_dataset, Dataset): + data_source = _dataset_to_decl(table_or_dataset, use_threads=True) + else: + data_source = Declaration( + "table_source", TableSourceNodeOptions(table_or_dataset) + ) - r = execplan([table_or_dataset], - plan=c_empty_decl_plan, - output_type=Table, - use_threads=True, - sort_options=sort_options) + order_by = Declaration("order_by", OrderByNodeOptions(sort_keys, **kwargs)) + + decl = Declaration.from_sequence([data_source, order_by]) + result_table = decl.to_table(use_threads=True) if output_type == Table: - return r + return result_table elif output_type == InMemoryDataset: # Get rid of special dataset columns # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" - return InMemoryDataset(r.select(table_or_dataset.schema.names)) + return InMemoryDataset(result_table.select(table_or_dataset.schema.names)) else: raise TypeError("Unsupported output type") From 474103ed13e84ae65b691b8640d88cb0948d8a81 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 22 Mar 2023 15:00:06 +0100 Subject: [PATCH 4/5] convert _exec_plan.pyx cython helpers to python file --- python/CMakeLists.txt | 2 - .../pyarrow/{_exec_plan.pyx => _exec_plan.py} | 39 +++++++++++++------ python/setup.py | 1 - 3 files changed, 28 insertions(+), 14 deletions(-) rename python/pyarrow/{_exec_plan.pyx => _exec_plan.py} (89%) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 580857d50c78..b5fc432242de 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -520,7 +520,6 @@ set(CYTHON_EXTENSIONS lib _compute _csv - _exec_plan _acero _feather _fs @@ -781,7 +780,6 @@ endif() if(PYARROW_BUILD_DATASET) target_link_libraries(_dataset PRIVATE ${DATASET_LINK_LIBS}) - target_link_libraries(_exec_plan PRIVATE ${DATASET_LINK_LIBS}) if(PYARROW_BUILD_ORC) target_link_libraries(_dataset_orc PRIVATE ${DATASET_LINK_LIBS}) endif() diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.py similarity index 89% rename from python/pyarrow/_exec_plan.pyx rename to python/pyarrow/_exec_plan.py index bc170550ba75..6e49341a13cc 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.py @@ -27,7 +27,7 @@ from pyarrow.dataset import Dataset, InMemoryDataset from pyarrow._dataset import ScanNodeOptions from pyarrow._acero import ( - Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, + Declaration, TableSourceNodeOptions, FilterNodeOptions, HashJoinNodeOptions, ProjectNodeOptions, OrderByNodeOptions ) @@ -37,7 +37,7 @@ def _dataset_to_decl(dataset, use_threads=True): filter_expr = dataset._scan_options.get("filter") if filter_expr is not None: - # Filters applied in CScanNodeOptions are "best effort" for the scan node itself, + # Filters applied in CScanNodeOptions are "best effort" for the scan node itself # so we always need to inject an additional Filter node to apply them for real. decl = Declaration.from_sequence( [decl, Declaration("filter", FilterNodeOptions(filter_expr))] @@ -46,8 +46,8 @@ def _dataset_to_decl(dataset, use_threads=True): return decl -def _perform_join(join_type, left_operand not None, left_keys, - right_operand not None, right_keys, +def _perform_join(join_type, left_operand, left_keys, + right_operand, right_keys, left_suffix=None, right_suffix=None, use_threads=True, coalesce_keys=False, output_type=Table): @@ -86,6 +86,11 @@ def _perform_join(join_type, left_operand not None, left_keys, ------- result_table : Table or InMemoryDataset """ + if not isinstance(left_operand, (Table, Dataset)): + raise TypeError(f"Expected Table or Dataset, got {type(left_operand)}") + if not isinstance(right_operand, (Table, Dataset)): + raise TypeError(f"Expected Table or Dataset, got {type(right_operand)}") + # Prepare left and right tables Keys to send them to the C++ function left_keys_order = {} if not isinstance(left_keys, (tuple, list)): @@ -136,18 +141,23 @@ def _perform_join(join_type, left_operand not None, left_keys, if isinstance(right_operand, Dataset): right_source = _dataset_to_decl(right_operand, use_threads=use_threads) else: - right_source = Declaration("table_source", TableSourceNodeOptions(right_operand)) + right_source = Declaration( + "table_source", TableSourceNodeOptions(right_operand) + ) join_opts = HashJoinNodeOptions( join_type, left_keys, right_keys, left_columns, right_columns, output_suffix_for_left=left_suffix or "", output_suffix_for_right=right_suffix or "", ) - decl = Declaration("hashjoin", options=join_opts, inputs=[left_source, right_source]) + decl = Declaration( + "hashjoin", options=join_opts, inputs=[left_source, right_source] + ) if coalesce_keys and join_type == "full outer": # In case of full outer joins, the join operation will output all columns - # so that we can coalesce the keys and exclude duplicates in a subsequent projection. + # so that we can coalesce the keys and exclude duplicates in a subsequent + # projection. left_columns_set = set(left_columns) right_columns_set = set(right_columns) # Where the right table columns start. @@ -162,7 +172,8 @@ def _perform_join(join_type, left_operand not None, left_keys, # with this left key. We do so by retrieving the name # of the right key that is in the same position in the provided keys # and then looking up the index for that name in the right table. - right_key_index = right_column_keys_indices[right_keys[left_keys_order[col]]] + right_key_index = right_column_keys_indices[ + right_keys[left_keys_order[col]]] projections.append( Expression._call("coalesce", [ Expression._field(idx), Expression._field( @@ -170,15 +181,21 @@ def _perform_join(join_type, left_operand not None, left_keys, ]) ) elif idx >= right_operand_index and col in right_column_keys_indices: - # Do not include right table keys. As they would lead to duplicated keys. + # Do not include right table keys. As they would lead to duplicated keys continue else: # For all the other columns incude them as they are. # Just recompute the suffixes that the join produced as the projection # would lose them otherwise. - if left_suffix and idx < right_operand_index and col in right_columns_set: + if ( + left_suffix and idx < right_operand_index + and col in right_columns_set + ): col += left_suffix - if right_suffix and idx >= right_operand_index and col in left_columns_set: + if ( + right_suffix and idx >= right_operand_index + and col in left_columns_set + ): col += right_suffix projected_col_names.append(col) projections.append( diff --git a/python/setup.py b/python/setup.py index 8d3b56cc5157..c2de0c71c9dc 100755 --- a/python/setup.py +++ b/python/setup.py @@ -202,7 +202,6 @@ def initialize_options(self): '_dataset', '_dataset_orc', '_dataset_parquet', - '_exec_plan', '_acero', '_feather', '_parquet', From 40551c8efe474b09b3f08c3da69de512160ee959 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 28 Mar 2023 10:10:20 +0200 Subject: [PATCH 5/5] fix linting --- python/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.cfg b/python/setup.cfg index 9b36d54fb55b..618fa475cd90 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -32,7 +32,7 @@ filterwarnings = faulthandler_timeout = 300 [pep8] -ignore = E211,E225,E226,E227,E402,W504 +ignore = E211,E225,E226,E227,E402,W503,W504 max_line_length = 88 [flake8]