From 078b9a907d56f268059fc6b16f82c086e38d4aee Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 17 Feb 2023 12:25:11 +0000 Subject: [PATCH 01/27] Add `join_asof` binding --- python/pyarrow/_dataset.pyx | 46 +++++++++++ python/pyarrow/_exec_plan.pyx | 78 ++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 7 ++ python/pyarrow/table.pxi | 113 +++++++++++++++++++++++++++ 4 files changed, 244 insertions(+) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 8bb1a0a1cb78..d606787cbb02 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -857,6 +857,52 @@ cdef class Dataset(_Weakrefable): use_threads=use_threads, coalesce_keys=coalesce_keys, output_type=InMemoryDataset) + def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): + """ + Perform a join between this dataset and another one. + + Result of the join will be a new dataset, where further + operations can be applied. + + Parameters + ---------- + right_dataset : dataset + The dataset to join to the current one, acting as the right dataset + in the join operation. + keys : str or list[str] + The columns from current dataset that should be used as keys + of the join operation left side. + right_keys : str or list[str], default None + The columns from the right_dataset that should be used as keys + on the join operation right side. + When ``None`` use the same key names as the left dataset. + join_type : str, default "left outer" + The kind of join that should be performed, one of + ("left semi", "right semi", "left anti", "right anti", + "inner", "left outer", "right outer", "full outer") + left_suffix : str, default None + Which suffix to add to right column names. This prevents confusion + when the columns in left and right datasets have colliding names. + right_suffix : str, default None + Which suffic to add to the left column names. This prevents confusion + when the columns in left and right datasets have colliding names. + coalesce_keys : bool, default True + If the duplicated keys should be omitted from one of the sides + in the join result. + use_threads : bool, default True + Whenever to use multithreading or not. + + Returns + ------- + InMemoryDataset + """ + if right_on is None: + right_on = on + if right_by is None: + right_by = by + return _pc()._exec_plan._perform_join_asof(self, on, by, right_table, right_on, right_by, + tolerance, output_type=InMemoryDataset) + cdef class InMemoryDataset(Dataset): """ diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index c01e6cba6a18..14c7e23b3c4c 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -390,6 +390,84 @@ def _perform_join(join_type, left_operand not None, left_keys, return result_table +def _perform_join_asof(left_operand not None, left_on, left_by, + right_operand not None, right_on, right_by, + tolerance, output_type=Table): + """ + Perform asof join of two tables or datasets. + + The result will be an output table with the result of the join operation + + Parameters + ---------- + left_operand : Table or Dataset + The left operand for the join operation. + left_on : str + The left key (or keys) on which the join operation should be performed. + left_by: str or list[str] + The left key (or keys) on which the join operation should be performed. + right_operand : Table or Dataset + The right operand for the join operation. + right_on : str or list[str] + The right key (or keys) on which the join operation should be performed. + right_by: str or list[str] + The right key (or keys) on which the join operation should be performed. + tolerance : int + The tolerance to use for the asof join. The tolerance is interpreted in + the same units as the "on" key. + output_type: Table or InMemoryDataset + The output type for the exec plan result. + + Returns + ------- + result_table : Table or InMemoryDataset + """ + cdef: + vector[CFieldRef] c_left_by + vector[CFieldRef] c_right_by + CAsofJoinKeys c_left_keys + CAsofJoinKeys c_right_keys + vector[CAsofJoinKeys] c_input_keys + vector[CDeclaration] c_decl_plan + + # Prepare left AsofJoinNodeOption::Keys + if isinstance(left_by, str): + left_by = [left_by] + for key in left_by: + c_left_by.push_back(CFieldRef(tobytes(key))) + + c_left_keys.on_key = CFieldRef(tobytes(left_on)) + c_left_keys.by_key = c_left_by + + c_input_keys.push_back(c_left_keys) + + # Prepare right AsofJoinNodeOption::Keys + right_by_order = {} + if isinstance(right_by, str): + right_by = [right_by] + for key in right_by: + c_right_by.push_back(CFieldRef(tobytes(key))) + + c_right_keys.on_key = CFieldRef(tobytes(right_on)) + c_right_keys.by_key = c_right_by + + c_input_keys.push_back(c_right_keys) + + c_decl_plan.push_back( + CDeclaration( + tobytes("asofjoin"), + CAsofJoinNodeOptions(c_input_keys, tolerance) + ) + ) + + result_table = execplan([left_operand, right_operand], + plan=c_decl_plan, + output_type=output_type, + use_threads=False) + + return result_table + + def _filter_table(table, expression, output_type=Table): """Filter rows of a table or dataset based on the provided expression. diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 295d89d351b9..beb1f60eacfc 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -2621,6 +2621,13 @@ cdef extern from "arrow/compute/exec/options.h" namespace "arrow::compute" nogil c_string output_suffix_for_left, c_string output_suffix_for_right) + cdef struct CAsofJoinKeys "arrow::compute::AsofJoinNodeOptions::Keys": + CFieldRef on_key + vector[CFieldRef] by_key + + cdef cppclass CAsofJoinNodeOptions "arrow::compute::AsofJoinNodeOptions"(CExecNodeOptions): + CAsofJoinNodeOptions(vector[CAsofJoinKeys] keys, int64_t tolerance) + cdef extern from "arrow/compute/exec/exec_plan.h" namespace "arrow::compute" nogil: cdef cppclass CDeclaration "arrow::compute::Declaration": diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index e400605e5662..8bb33e3806e8 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5026,6 +5026,119 @@ cdef class Table(_PandasConvertible): use_threads=use_threads, coalesce_keys=coalesce_keys, output_type=Table) + def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): + """ + Perform a join between this table and another one. + + Result of the join will be a new Table, where further + operations can be applied. + + Parameters + ---------- + right_table : Table + The table to join to the current one, acting as the right table + in the join operation. + keys : str or list[str] + The columns from current table that should be used as keys + of the join operation left side. + right_keys : str or list[str], default None + The columns from the right_table that should be used as keys + on the join operation right side. + When ``None`` use the same key names as the left table. + join_type : str, default "left outer" + The kind of join that should be performed, one of + ("left semi", "right semi", "left anti", "right anti", + "inner", "left outer", "right outer", "full outer") + left_suffix : str, default None + Which suffix to add to left column names. This prevents confusion + when the columns in left and right tables have colliding names. + right_suffix : str, default None + Which suffix to add to the right column names. This prevents confusion + when the columns in left and right tables have colliding names. + coalesce_keys : bool, default True + If the duplicated keys should be omitted from one of the sides + in the join result. + use_threads : bool, default True + Whether to use multithreading or not. + + Returns + ------- + Table + + Examples + -------- + >>> import pandas as pd + >>> import pyarrow as pa + >>> df1 = pd.DataFrame({'id': [1, 2, 3], + ... 'year': [2020, 2022, 2019]}) + >>> df2 = pd.DataFrame({'id': [3, 4], + ... 'n_legs': [5, 100], + ... 'animal': ["Brittle stars", "Centipede"]}) + >>> t1 = pa.Table.from_pandas(df1) + >>> t2 = pa.Table.from_pandas(df2) + + Left outer join: + + >>> t1.join(t2, 'id').combine_chunks().sort_by('year') + pyarrow.Table + id: int64 + year: int64 + n_legs: int64 + animal: string + ---- + id: [[3,1,2]] + year: [[2019,2020,2022]] + n_legs: [[5,null,null]] + animal: [["Brittle stars",null,null]] + + Full outer join: + + >>> t1.join(t2, 'id', join_type="full outer").combine_chunks().sort_by('year') + pyarrow.Table + id: int64 + year: int64 + n_legs: int64 + animal: string + ---- + id: [[3,1,2,4]] + year: [[2019,2020,2022,null]] + n_legs: [[5,null,null,100]] + animal: [["Brittle stars",null,null,"Centipede"]] + + Right outer join: + + >>> t1.join(t2, 'id', join_type="right outer").combine_chunks().sort_by('year') + pyarrow.Table + year: int64 + id: int64 + n_legs: int64 + animal: string + ---- + year: [[2019,null]] + id: [[3,4]] + n_legs: [[5,100]] + animal: [["Brittle stars","Centipede"]] + + Right anti join + + >>> t1.join(t2, 'id', join_type="right anti") + pyarrow.Table + id: int64 + n_legs: int64 + animal: string + ---- + id: [[4]] + n_legs: [[100]] + animal: [["Centipede"]] + """ + if right_on is None: + right_on = on + if right_by is None: + right_by = by + return _pc()._exec_plan._perform_join_asof(self, on, by, right_table, right_on, right_by, + tolerance, output_type=Table) + + def _reconstruct_table(arrays, schema): """ From 2aa990bab875f75382fa2652f61c896ebdc48df8 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 17 Feb 2023 12:25:27 +0000 Subject: [PATCH 02/27] Add test for `join_asof` binding --- python/pyarrow/tests/test_exec_plan.py | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 7875dff55755..e0d3f90179d0 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -119,6 +119,59 @@ def test_joins(jointype, expected, use_threads, use_datasets): assert r == expected +@pytest.mark.parametrize("tolerance,expected", [ + ( + 1, + { + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, None, None], + }, + ), + ( + 3, + { + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, 3., None], + }, + ), + ( + -5, + { + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [None, None, 1., None, None], + }, + ), +]) +@pytest.mark.parametrize("use_datasets", [False, True]) +def test_join_asof(tolerance, expected, use_datasets): + # Allocate table here instead of using parametrize + # this prevents having arrow allocated memory forever around. + expected = pa.table(expected) + + t1 = pa.Table.from_pydict({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [2, 9, 15], + "col3": ["a", "b", "g"], + "colC": [1., 3., 5.] + }) + + if use_datasets: + t1 = ds.dataset([t1]) + t2 = ds.dataset([t2]) + + r = ep._perform_join_asof(t1, "colA", "col2", t2, "colB", "col3", tolerance) + r = r.combine_chunks() + r = r.sort_by("colA") + assert r == expected + + def test_table_join_collisions(): t1 = pa.table({ "colA": [1, 2, 6], From dad04165df6fc2fab3ddba29fe65639be02278f7 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 09:34:58 +0000 Subject: [PATCH 03/27] Handle removal of special Dataset columns --- python/pyarrow/_exec_plan.pyx | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 14c7e23b3c4c..ee64a31e24ee 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -453,6 +453,26 @@ def _perform_join_asof(left_operand not None, left_on, left_by, c_input_keys.push_back(c_right_keys) + # By default expose all columns on both left and right table + if isinstance(left_operand, Table): + left_columns = left_operand.column_names + elif isinstance(left_operand, Dataset): + left_columns = left_operand.schema.names + else: + raise TypeError("Unsupported left join member type") + + if isinstance(right_operand, Table): + right_columns = right_operand.column_names + elif isinstance(right_operand, Dataset): + right_columns = right_operand.schema.names + else: + raise TypeError("Unsupported right join member type") + + # AsofJoin does not return on or by columns for right_operand. + right_columns = [ + col for col in right_columns if col not in [right_on] + right_by + ] + c_decl_plan.push_back( CDeclaration( tobytes("asofjoin"), @@ -465,7 +485,10 @@ def _perform_join_asof(left_operand not None, left_on, left_by, output_type=output_type, use_threads=False) - return result_table + # If the output_type is InMemoryDataset we need to get rid of the + # special dataset columns + # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" + return result_table.select(left_columns + right_columns) def _filter_table(table, expression, output_type=Table): From fd33b8cebef8d42ae64ffdfae0ea9026a5ee7f6f Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 09:57:19 +0000 Subject: [PATCH 04/27] Explicitly don't handle column collisions --- python/pyarrow/_exec_plan.pyx | 7 ++ python/pyarrow/tests/test_exec_plan.py | 127 ++++++++++++++----------- 2 files changed, 81 insertions(+), 53 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index ee64a31e24ee..34676539e203 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -473,6 +473,13 @@ def _perform_join_asof(left_operand not None, left_on, left_by, col for col in right_columns if col not in [right_on] + right_by ] + for col in right_columns: + if col in left_columns: + raise ValueError( + "{} present in both tables. AsofJoin does not support " + "column collisions.".format(col), + ) + c_decl_plan.push_back( CDeclaration( tobytes("asofjoin"), diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index e0d3f90179d0..aa6e6ac473cb 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -119,59 +119,6 @@ def test_joins(jointype, expected, use_threads, use_datasets): assert r == expected -@pytest.mark.parametrize("tolerance,expected", [ - ( - 1, - { - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"], - "colC": [1., None, None, None, None], - }, - ), - ( - 3, - { - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"], - "colC": [1., None, None, 3., None], - }, - ), - ( - -5, - { - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"], - "colC": [None, None, 1., None, None], - }, - ), -]) -@pytest.mark.parametrize("use_datasets", [False, True]) -def test_join_asof(tolerance, expected, use_datasets): - # Allocate table here instead of using parametrize - # this prevents having arrow allocated memory forever around. - expected = pa.table(expected) - - t1 = pa.Table.from_pydict({ - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"] - }) - - t2 = pa.Table.from_pydict({ - "colB": [2, 9, 15], - "col3": ["a", "b", "g"], - "colC": [1., 3., 5.] - }) - - if use_datasets: - t1 = ds.dataset([t1]) - t2 = ds.dataset([t2]) - - r = ep._perform_join_asof(t1, "colA", "col2", t2, "colB", "col3", tolerance) - r = r.combine_chunks() - r = r.sort_by("colA") - assert r == expected - - def test_table_join_collisions(): t1 = pa.table({ "colA": [1, 2, 6], @@ -374,3 +321,77 @@ def test_join_extension_array_column(): result = ep._perform_join( "left outer", t1, ["colB"], t3, ["colC"]) assert result["colB"] == pa.chunked_array(ext_array) + + +@pytest.mark.parametrize("tolerance,expected", [ + ( + 1, + { + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, None, None], + }, + ), + ( + 3, + { + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, 3., None], + }, + ), + ( + -5, + { + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [None, None, 1., None, None], + }, + ), +]) +@pytest.mark.parametrize("use_datasets", [False, True]) +def test_join_asof(tolerance, expected, use_datasets): + # Allocate table here instead of using parametrize + # this prevents having arrow allocated memory forever around. + expected = pa.table(expected) + + t1 = pa.Table.from_pydict({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [2, 9, 15], + "col3": ["a", "b", "g"], + "colC": [1., 3., 5.] + }) + + if use_datasets: + t1 = ds.dataset([t1]) + t2 = ds.dataset([t2]) + + r = ep._perform_join_asof(t1, "colA", "col2", t2, "colB", "col3", tolerance) + r = r.combine_chunks() + r = r.sort_by("colA") + assert r == expected + + +def test_table_asof_join_collisions(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + msg = "colVals present in both tables. AsofJoin does not support column collisions." + with pytest.raises(ValueError, match=msg): + ep._perform_join_asof(t1, "on", ["colA", "colB"], t2, "on", ["colA", "colB"], 1) From 09bdd0497e1a0a6d32550a05c2beea2da579c128 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 10:48:12 +0000 Subject: [PATCH 05/27] Fix linting --- python/pyarrow/_exec_plan.pyx | 4 ++-- python/pyarrow/table.pxi | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 34676539e203..91cfba49406b 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -391,8 +391,8 @@ def _perform_join(join_type, left_operand not None, left_keys, def _perform_join_asof(left_operand not None, left_on, left_by, - right_operand not None, right_on, right_by, - tolerance, output_type=Table): + right_operand not None, right_on, right_by, + tolerance, output_type=Table): """ Perform asof join of two tables or datasets. diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 8bb33e3806e8..4e62e11b4f6a 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5139,7 +5139,6 @@ cdef class Table(_PandasConvertible): tolerance, output_type=Table) - def _reconstruct_table(arrays, schema): """ Internal: reconstruct pa.Table from pickled components. From 27e5862870b3d575e365df4b9f6515b788758489 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 11:07:48 +0000 Subject: [PATCH 06/27] Fix the docs --- python/pyarrow/_dataset.pyx | 45 +++++++++--------- python/pyarrow/table.pxi | 94 +++++++++++-------------------------- 2 files changed, 50 insertions(+), 89 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index d606787cbb02..be346c153e39 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -857,9 +857,9 @@ cdef class Dataset(_Weakrefable): use_threads=use_threads, coalesce_keys=coalesce_keys, output_type=InMemoryDataset) - def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): + def join_asof(self, right_dataset, on, by, tolerance, right_on=None, right_by=None): """ - Perform a join between this dataset and another one. + Perform an asof join between this dataset and another one. Result of the join will be a new dataset, where further operations can be applied. @@ -869,28 +869,29 @@ cdef class Dataset(_Weakrefable): right_dataset : dataset The dataset to join to the current one, acting as the right dataset in the join operation. - keys : str or list[str] - The columns from current dataset that should be used as keys + on : str + The column from current dataset that should be used as the on key of the join operation left side. - right_keys : str or list[str], default None - The columns from the right_dataset that should be used as keys + by : str or list[str] + The columns from current dataset that should be used as the by keys + of the join operation left side. + tolerance : int + The tolerance for inexact "on" key matching. A right row is considered + a match with the left row `right.on - left.on <= tolerance`. The + `tolerance` may be: + - negative, in which case a past-as-of-join occurs; + - or positive, in which case a future-as-of-join occurs; + - or zero, in which case an exact-as-of-join occurs. + + The tolerance is interpreted in the same units as the "on" key. + right_on : str or list[str], default None + The columns from the right_dataset that should be used as the on key + on the join operation right side. + When ``None`` use the same key name as the left dataset. + right_by : str or list[str], default None + The columns from the right_dataset that should be used as by keys on the join operation right side. When ``None`` use the same key names as the left dataset. - join_type : str, default "left outer" - The kind of join that should be performed, one of - ("left semi", "right semi", "left anti", "right anti", - "inner", "left outer", "right outer", "full outer") - left_suffix : str, default None - Which suffix to add to right column names. This prevents confusion - when the columns in left and right datasets have colliding names. - right_suffix : str, default None - Which suffic to add to the left column names. This prevents confusion - when the columns in left and right datasets have colliding names. - coalesce_keys : bool, default True - If the duplicated keys should be omitted from one of the sides - in the join result. - use_threads : bool, default True - Whenever to use multithreading or not. Returns ------- @@ -900,7 +901,7 @@ cdef class Dataset(_Weakrefable): right_on = on if right_by is None: right_by = by - return _pc()._exec_plan._perform_join_asof(self, on, by, right_table, right_on, right_by, + return _pc()._exec_plan._perform_join_asof(self, on, by, right_dataset, right_on, right_by, tolerance, output_type=InMemoryDataset) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 4e62e11b4f6a..ac0932b2cfa9 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5028,9 +5028,9 @@ cdef class Table(_PandasConvertible): def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): """ - Perform a join between this table and another one. + Perform an asof join between this table and another one. - Result of the join will be a new Table, where further + Result of the join will be a new dataset, where further operations can be applied. Parameters @@ -5038,48 +5038,48 @@ cdef class Table(_PandasConvertible): right_table : Table The table to join to the current one, acting as the right table in the join operation. - keys : str or list[str] - The columns from current table that should be used as keys + on : str + The column from current table that should be used as the on key of the join operation left side. - right_keys : str or list[str], default None - The columns from the right_table that should be used as keys + by : str or list[str] + The columns from current table that should be used as the by keys + of the join operation left side. + tolerance : int + The tolerance for inexact "on" key matching. A right row is considered + a match with the left row ``right.on - left.on <= tolerance``. The + ``tolerance`` may be: + - negative, in which case a past-as-of-join occurs; + - or positive, in which case a future-as-of-join occurs; + - or zero, in which case an exact-as-of-join occurs. + + The tolerance is interpreted in the same units as the "on" key. + right_on : str or list[str], default None + The columns from the right_table that should be used as the on key + on the join operation right side. + When ``None`` use the same key name as the left table. + right_by : str or list[str], default None + The columns from the right_table that should be used as by keys on the join operation right side. When ``None`` use the same key names as the left table. - join_type : str, default "left outer" - The kind of join that should be performed, one of - ("left semi", "right semi", "left anti", "right anti", - "inner", "left outer", "right outer", "full outer") - left_suffix : str, default None - Which suffix to add to left column names. This prevents confusion - when the columns in left and right tables have colliding names. - right_suffix : str, default None - Which suffix to add to the right column names. This prevents confusion - when the columns in left and right tables have colliding names. - coalesce_keys : bool, default True - If the duplicated keys should be omitted from one of the sides - in the join result. - use_threads : bool, default True - Whether to use multithreading or not. Returns ------- Table - Examples + Example -------- >>> import pandas as pd >>> import pyarrow as pa >>> df1 = pd.DataFrame({'id': [1, 2, 3], ... 'year': [2020, 2022, 2019]}) >>> df2 = pd.DataFrame({'id': [3, 4], + ... 'year': [2020, 2021], ... 'n_legs': [5, 100], ... 'animal': ["Brittle stars", "Centipede"]}) - >>> t1 = pa.Table.from_pandas(df1) - >>> t2 = pa.Table.from_pandas(df2) + >>> t1 = pa.Table.from_pandas(df1).sort_by('year') + >>> t2 = pa.Table.from_pandas(df2).sort_by('year') - Left outer join: - - >>> t1.join(t2, 'id').combine_chunks().sort_by('year') + >>> t1.join_asof(t2, 'year', 'id', 1).combine_chunks().sort_by('year') pyarrow.Table id: int64 year: int64 @@ -5090,46 +5090,6 @@ cdef class Table(_PandasConvertible): year: [[2019,2020,2022]] n_legs: [[5,null,null]] animal: [["Brittle stars",null,null]] - - Full outer join: - - >>> t1.join(t2, 'id', join_type="full outer").combine_chunks().sort_by('year') - pyarrow.Table - id: int64 - year: int64 - n_legs: int64 - animal: string - ---- - id: [[3,1,2,4]] - year: [[2019,2020,2022,null]] - n_legs: [[5,null,null,100]] - animal: [["Brittle stars",null,null,"Centipede"]] - - Right outer join: - - >>> t1.join(t2, 'id', join_type="right outer").combine_chunks().sort_by('year') - pyarrow.Table - year: int64 - id: int64 - n_legs: int64 - animal: string - ---- - year: [[2019,null]] - id: [[3,4]] - n_legs: [[5,100]] - animal: [["Brittle stars","Centipede"]] - - Right anti join - - >>> t1.join(t2, 'id', join_type="right anti") - pyarrow.Table - id: int64 - n_legs: int64 - animal: string - ---- - id: [[4]] - n_legs: [[100]] - animal: [["Centipede"]] """ if right_on is None: right_on = on From 312378eb43171e0b15a3511344bf5591fec0e94f Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 16:15:50 +0000 Subject: [PATCH 07/27] Add more tests --- python/pyarrow/tests/test_dataset.py | 51 ++++++++++++++++++++++++++ python/pyarrow/tests/test_exec_plan.py | 2 +- python/pyarrow/tests/test_table.py | 43 ++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index ebacd0235916..384de5cea951 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4839,6 +4839,57 @@ def test_dataset_join_collisions(tempdir): ], names=["colA", "colB", "colVals", "colB_r", "colVals_r"]) +@pytest.mark.dataset +def test_dataset_join_asof(tempdir): + t1 = pa.Table.from_pydict({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"] + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.Table.from_pydict({ + "colB": [2, 9, 15], + "col3": ["a", "b", "g"], + "colC": [1., 3., 5.] + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + result = ds1.join_asof(ds2, "colA", "col2", 1, "colB", "col3") + assert result.to_table().sort_by("colA") == pa.table({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, None, None], + }) + + +@pytest.mark.dataset +def test_dataset_join_asof_collisions(tempdir): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": ["a", "b", "f"] + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + msg = "colVals present in both tables. AsofJoin does not support column collisions." + with pytest.raises(ValueError, match=msg): + ds1.join_asof(ds2, "on", ["colA", "colB"], 1, "on", ["colA", "colB"]) + + @pytest.mark.parametrize('dstype', [ "fs", "mem" ]) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index aa6e6ac473cb..5d99d25f9e82 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -376,7 +376,7 @@ def test_join_asof(tolerance, expected, use_datasets): assert r == expected -def test_table_asof_join_collisions(): +def test_table_join_asof_collisions(): t1 = pa.table({ "colA": [1, 2, 6], "colB": [10, 20, 60], diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 1a0eaa33772e..16efd6e9b631 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2297,6 +2297,49 @@ def test_table_join_many_columns(): }) +@pytest.mark.dataset +def test_table_join_asof(): + t1 = pa.Table.from_pydict({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [2, 9, 15], + "col3": ["a", "b", "g"], + "colC": [1., 3., 5.] + }) + + r = t1.join_asof(t2, "colA", "col2", 1, "colB", "col3") + assert r.combine_chunks() == pa.table({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, None, None], + }) + + +@pytest.mark.dataset +def test_table_join_asof_collisions(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + msg = "colVals present in both tables. AsofJoin does not support column collisions." + with pytest.raises(ValueError, match=msg): + t1.join_asof(t2, "on", ["colA", "colB"], 1, "on", ["colA", "colB"]) + + def test_table_cast_invalid(): # Casting a nullable field to non-nullable should be invalid! table = pa.table({'a': [None, 1], 'b': [None, True]}) From 8303310b1eeda0d8eaa035e36f031ead84382ef6 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 16:16:00 +0000 Subject: [PATCH 08/27] Fix bug with datasets --- python/pyarrow/_exec_plan.pyx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 91cfba49406b..0e2a879d8e24 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -489,13 +489,17 @@ def _perform_join_asof(left_operand not None, left_on, left_by, result_table = execplan([left_operand, right_operand], plan=c_decl_plan, - output_type=output_type, + output_type=Table, use_threads=False) - # If the output_type is InMemoryDataset we need to get rid of the - # special dataset columns - # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" - return result_table.select(left_columns + right_columns) + if output_type == Table: + return result_table + elif output_type == InMemoryDataset: + # Get rid of special dataset columns + # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" + return InMemoryDataset(result_table.select(left_columns + right_columns)) + else: + raise TypeError("Unsupported output type") def _filter_table(table, expression, output_type=Table): From 46a1d2afa99fafac9a723d76de891f73c8fbb775 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 20 Feb 2023 17:10:32 +0000 Subject: [PATCH 09/27] Make sure tests pass --- python/pyarrow/_exec_plan.pyx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index 0e2a879d8e24..d16beae3543f 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -492,12 +492,14 @@ def _perform_join_asof(left_operand not None, left_on, left_by, output_type=Table, use_threads=False) + # Get rid of special dataset columns + # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" + result_table = result_table.select(left_columns + right_columns) + if output_type == Table: return result_table elif output_type == InMemoryDataset: - # Get rid of special dataset columns - # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" - return InMemoryDataset(result_table.select(left_columns + right_columns)) + return InMemoryDataset(result_table) else: raise TypeError("Unsupported output type") From d39926770fbd9dc857d80956fc15b2e2bbde64bb Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Sun, 5 Mar 2023 22:18:14 +0000 Subject: [PATCH 10/27] Add `AsofJoinNodeOptions` class --- python/pyarrow/_acero.pyx | 72 ++++++++++++++++++++++++++++++ python/pyarrow/tests/test_acero.py | 35 +++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 91f3ef6712f0..f6dbfc6288c1 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -340,6 +340,78 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): ) +cdef class _AsofJoinNodeOptions(ExecNodeOptions): + + def _set_options(self, left_on, left_by, right_on, right_by, tolerance): + cdef: + vector[CFieldRef] c_left_by + vector[CFieldRef] c_right_by + CAsofJoinKeys c_left_keys + CAsofJoinKeys c_right_keys + vector[CAsofJoinKeys] c_input_keys + + # Prepare left AsofJoinNodeOption::Keys + if not isinstance(left_by, (list, tuple)): + left_by = [left_by] + for key in left_by: + c_left_by.push_back(_ensure_field_ref(key)) + + c_left_keys.on_key = _ensure_field_ref(left_on) + c_left_keys.by_key = c_left_by + + c_input_keys.push_back(c_left_keys) + + # Prepare right AsofJoinNodeOption::Keys + if not isinstance(right_by, (list, tuple)): + right_by = [right_by] + for key in right_by: + c_right_by.push_back(_ensure_field_ref(key)) + + c_right_keys.on_key = _ensure_field_ref(right_on) + c_right_keys.by_key = c_right_by + + c_input_keys.push_back(c_right_keys) + + self.wrapped.reset( + new CAsofJoinNodeOptions( + c_input_keys, + tolerance, + ) + ) + + +class AsofJoinNodeOptions(_AsofJoinNodeOptions): + """ + Make a node which implements 'as of join' operation. + + This is the option class for the "asofjoin" node factory. + + Parameters + ---------- + left_on : str, Expression + The left key on which the join operation should be performed. + Can be a string column name or a field expression. + left_by: str, Expression or list[str] + The left keys on which the join operation should be performed. + Each key can be a string column name or a field expression, + or a list of such field references. + right_operand : Table or Dataset + The right operand for the join operation. + right_on : str, Expression + The right key on which the join operation should be performed. + See `left_on` for details. + right_by: str, Expression or list[str] + The right keys on which the join operation should be performed. + See `left_by` for details. + tolerance : int + The tolerance to use for the asof join. The tolerance is interpreted in + the same units as the "on" key. + """ + + def __init__(self, left_on, left_by, right_on, right_by, tolerance): + self._set_options(left_on, left_by, right_on, right_by, tolerance) + + cdef class Declaration(_Weakrefable): """ Helper class for declaring the nodes of an ExecPlan. diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index 4cba55f7e421..af294189a785 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -27,6 +27,7 @@ ProjectNodeOptions, AggregateNodeOptions, HashJoinNodeOptions, + AsofJoinNodeOptions, Declaration, ) @@ -291,3 +292,37 @@ def test_hash_join(): names=["key", "a", "b"] ) assert result.sort_by("a").equals(expected) + + +def test_asof_join(): + left = pa.table({'key': [1, 2, 3], 'ts': [1, 1, 1], 'a': [4, 5, 6]}) + left_source = Declaration("table_source", options=TableSourceNodeOptions(left)) + right = pa.table({'key': [2, 3, 4], 'ts': [2, 5, 2], 'b': [4, 5, 6]}) + right_source = Declaration("table_source", options=TableSourceNodeOptions(right)) + + # asof join + join_opts = AsofJoinNodeOptions( + left_on="ts", left_by=["key"], + right_on="ts", right_by=["key"], + tolerance=1, + ) + joined = Declaration( + "asofjoin", options=join_opts, inputs=[left_source, right_source] + ) + result = joined.to_table() + expected = pa.table( + [[1, 2, 3], [1, 1, 1], [4, 5, 6], [None, 4, None]], + names=["key", "ts", "a", "b"]) + assert result == expected + + for by in [field("key"), ["key"], [field("key")]]: + for on in [field("ts"), "ts"]: + join_opts = AsofJoinNodeOptions( + left_on=on, left_by=by, + right_on=on, right_by=by, + tolerance=1, + ) + joined = Declaration( + "asofjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + assert result == expected From 367a17d22cebfabc893b3176846b776177797364 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Sun, 5 Mar 2023 22:22:47 +0000 Subject: [PATCH 11/27] Use `AsofJoinNodeOptions` in `_perform_join_asof` --- python/pyarrow/_exec_plan.pyx | 56 ++++++++++------------------------- 1 file changed, 15 insertions(+), 41 deletions(-) diff --git a/python/pyarrow/_exec_plan.pyx b/python/pyarrow/_exec_plan.pyx index d16beae3543f..739d03991461 100644 --- a/python/pyarrow/_exec_plan.pyx +++ b/python/pyarrow/_exec_plan.pyx @@ -34,6 +34,8 @@ from pyarrow._compute cimport Expression, _true, _SortOptions from pyarrow._dataset cimport Dataset, Scanner from pyarrow._dataset import InMemoryDataset +from pyarrow._acero import Declaration, TableSourceNodeOptions, AsofJoinNodeOptions + Initialize() # Initialise support for Datasets in ExecPlan @@ -422,36 +424,10 @@ def _perform_join_asof(left_operand not None, left_on, left_by, ------- result_table : Table or InMemoryDataset """ - cdef: - vector[CFieldRef] c_left_by - vector[CFieldRef] c_right_by - CAsofJoinKeys c_left_keys - CAsofJoinKeys c_right_keys - vector[CAsofJoinKeys] c_input_keys - vector[CDeclaration] c_decl_plan - - # Prepare left AsofJoinNodeOption::Keys - if isinstance(left_by, str): + if not isinstance(left_by, (list, tuple)): left_by = [left_by] - for key in left_by: - c_left_by.push_back(CFieldRef(tobytes(key))) - - c_left_keys.on_key = CFieldRef(tobytes(left_on)) - c_left_keys.by_key = c_left_by - - c_input_keys.push_back(c_left_keys) - - # Prepare right AsofJoinNodeOption::Keys - right_by_order = {} - if isinstance(right_by, str): + if not isinstance(right_by, (list, tuple)): right_by = [right_by] - for key in right_by: - c_right_by.push_back(CFieldRef(tobytes(key))) - - c_right_keys.on_key = CFieldRef(tobytes(right_on)) - c_right_keys.by_key = c_right_by - - c_input_keys.push_back(c_right_keys) # By default expose all columns on both left and right table if isinstance(left_operand, Table): @@ -480,21 +456,19 @@ def _perform_join_asof(left_operand not None, left_on, left_by, "column collisions.".format(col), ) - c_decl_plan.push_back( - CDeclaration( - tobytes("asofjoin"), - CAsofJoinNodeOptions(c_input_keys, tolerance) - ) - ) + # TEMP + if isinstance(left_operand, Dataset): + left_operand = left_operand.to_table() + if isinstance(right_operand, Dataset): + right_operand = right_operand.to_table() - result_table = execplan([left_operand, right_operand], - plan=c_decl_plan, - output_type=Table, - use_threads=False) + left_source = Declaration("table_source", TableSourceNodeOptions(left_operand)) + right_source = Declaration("table_source", TableSourceNodeOptions(right_operand)) - # Get rid of special dataset columns - # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" - result_table = result_table.select(left_columns + right_columns) + join_opts = AsofJoinNodeOptions(left_on, left_by, right_on, right_by, tolerance) + decl = Declaration("asofjoin", options=join_opts, + inputs=[left_source, right_source]) + result_table = decl.to_table(use_threads=False) if output_type == Table: return result_table From 28ea0728fd9bf570306a74926fcc584c6f54acb7 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Sun, 5 Mar 2023 22:28:55 +0000 Subject: [PATCH 12/27] Swap to kwargs --- python/pyarrow/table.pxi | 4 +++- python/pyarrow/tests/test_dataset.py | 10 ++++++++-- python/pyarrow/tests/test_table.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index ac0932b2cfa9..4f0dea104d7b 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5079,7 +5079,9 @@ cdef class Table(_PandasConvertible): >>> t1 = pa.Table.from_pandas(df1).sort_by('year') >>> t2 = pa.Table.from_pandas(df2).sort_by('year') - >>> t1.join_asof(t2, 'year', 'id', 1).combine_chunks().sort_by('year') + >>> t1.join_asof( + ... t2, on='year', by='id', tolerance=1 + ... ).combine_chunks().sort_by('year') pyarrow.Table id: int64 year: int64 diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 384de5cea951..ca382a2014e1 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4856,7 +4856,10 @@ def test_dataset_join_asof(tempdir): ds.write_dataset(t2, tempdir / "t2", format="ipc") ds2 = ds.dataset(tempdir / "t2", format="ipc") - result = ds1.join_asof(ds2, "colA", "col2", 1, "colB", "col3") + result = ds1.join_asof( + ds2, on="colA", by="col2", tolerance=1, + right_on="colB", right_by="col3", + ) assert result.to_table().sort_by("colA") == pa.table({ "colA": [1, 1, 5, 6, 7], "col2": ["a", "b", "a", "b", "f"], @@ -4887,7 +4890,10 @@ def test_dataset_join_asof_collisions(tempdir): msg = "colVals present in both tables. AsofJoin does not support column collisions." with pytest.raises(ValueError, match=msg): - ds1.join_asof(ds2, "on", ["colA", "colB"], 1, "on", ["colA", "colB"]) + ds1.join_asof( + ds2, on="on", by=["colA", "colB"], tolerance=1, + right_on="on", right_by=["colA", "colB"], + ) @pytest.mark.parametrize('dstype', [ diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 16efd6e9b631..c46c51a8db69 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2310,7 +2310,10 @@ def test_table_join_asof(): "colC": [1., 3., 5.] }) - r = t1.join_asof(t2, "colA", "col2", 1, "colB", "col3") + r = t1.join_asof( + t2, on="colA", by="col2", tolerance=1, + right_on="colB", right_by="col3", + ) assert r.combine_chunks() == pa.table({ "colA": [1, 1, 5, 6, 7], "col2": ["a", "b", "a", "b", "f"], @@ -2337,7 +2340,10 @@ def test_table_join_asof_collisions(): msg = "colVals present in both tables. AsofJoin does not support column collisions." with pytest.raises(ValueError, match=msg): - t1.join_asof(t2, "on", ["colA", "colB"], 1, "on", ["colA", "colB"]) + t1.join_asof( + t2, on="on", by=["colA", "colB"], tolerance=1, + right_on="on", right_by=["colA", "colB"], + ) def test_table_cast_invalid(): From 0a517f3cfcfa218db576d4874a9ba79d617dc98c Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Sun, 5 Mar 2023 22:30:06 +0000 Subject: [PATCH 13/27] Fix restructuredtext formatting --- python/pyarrow/_dataset.pyx | 7 ++++--- python/pyarrow/table.pxi | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index be346c153e39..e55ecde6e301 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -879,9 +879,10 @@ cdef class Dataset(_Weakrefable): The tolerance for inexact "on" key matching. A right row is considered a match with the left row `right.on - left.on <= tolerance`. The `tolerance` may be: - - negative, in which case a past-as-of-join occurs; - - or positive, in which case a future-as-of-join occurs; - - or zero, in which case an exact-as-of-join occurs. + + - negative, in which case a past-as-of-join occurs; + - or positive, in which case a future-as-of-join occurs; + - or zero, in which case an exact-as-of-join occurs. The tolerance is interpreted in the same units as the "on" key. right_on : str or list[str], default None diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 4f0dea104d7b..f2d0c4443c60 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5048,9 +5048,10 @@ cdef class Table(_PandasConvertible): The tolerance for inexact "on" key matching. A right row is considered a match with the left row ``right.on - left.on <= tolerance``. The ``tolerance`` may be: - - negative, in which case a past-as-of-join occurs; - - or positive, in which case a future-as-of-join occurs; - - or zero, in which case an exact-as-of-join occurs. + + - negative, in which case a past-as-of-join occurs; + - or positive, in which case a future-as-of-join occurs; + - or zero, in which case an exact-as-of-join occurs. The tolerance is interpreted in the same units as the "on" key. right_on : str or list[str], default None From 984b159710a8c39d51b12d54b582f58d0f8a1dc7 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Sun, 5 Mar 2023 23:18:05 +0000 Subject: [PATCH 14/27] Add test cases --- python/pyarrow/tests/test_dataset.py | 54 ++++++++++++++++++++++++++++ python/pyarrow/tests/test_table.py | 46 ++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index ca382a2014e1..f055fa2e04f4 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4867,6 +4867,60 @@ def test_dataset_join_asof(tempdir): }) +@pytest.mark.dataset +def test_dataset_join_asof_multiple_by(tempdir): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + result = ds1.join_asof( + ds2, on="on", by=["colA", "colB"], tolerance=1 + ) + assert result.to_table().sort_by("colA") == pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": [None, "B", None], + }) + + +@pytest.mark.dataset +def test_dataset_join_asof_empty_by(tempdir): + t1 = pa.table({ + "on": [1, 2, 3], + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "on": [2, 3, 4], + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + result = ds1.join_asof( + ds2, on="on", by=[], tolerance=1 + ) + assert result.to_table() == pa.table({ + "on": [1, 2, 3], + "colVals": ["Z", "Z", "B"], + }) + + @pytest.mark.dataset def test_dataset_join_asof_collisions(tempdir): t1 = pa.table({ diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index c46c51a8db69..d50e14d9ed4e 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2321,6 +2321,52 @@ def test_table_join_asof(): }) +@pytest.mark.dataset +def test_table_join_asof_multiple_by(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + result = t1.join_asof( + t2, on="on", by=["colA", "colB"], tolerance=1 + ) + assert result.sort_by("colA") == pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": [None, "B", None], + }) + + +@pytest.mark.dataset +def test_table_join_asof_empty_by(): + t1 = pa.table({ + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "on": [2, 3, 4], + }) + + result = t1.join_asof( + t2, on="on", by=[], tolerance=1 + ) + assert result == pa.table({ + "on": [1, 2, 3], + "colVals": ["Z", "Z", "B"], + }) + + @pytest.mark.dataset def test_table_join_asof_collisions(): t1 = pa.table({ From 6f8db25fec78546e19c91e405a3f1c14a0ddcd64 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 8 Sep 2023 13:04:31 +0100 Subject: [PATCH 15/27] Fix some typos in doc strings --- python/pyarrow/_acero.pyx | 2 -- python/pyarrow/_dataset.pyx | 2 +- python/pyarrow/table.pxi | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index b411cff1fdd2..266876935e06 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -447,8 +447,6 @@ class AsofJoinNodeOptions(_AsofJoinNodeOptions): The left keys on which the join operation should be performed. Each key can be a string column name or a field expression, or a list of such field references. - right_operand : Table or Dataset - The right operand for the join operation. right_on : str, Expression The right key on which the join operation should be performed. See `left_on` for details. diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index e00769d9dffd..82565133cb02 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -878,7 +878,7 @@ cdef class Dataset(_Weakrefable): """ Perform an asof join between this dataset and another one. - Result of the join will be a new dataset, where further + Result of the join will be a new Dataset, where further operations can be applied. Parameters diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 8616dcbd7490..72603228b88e 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4757,7 +4757,7 @@ cdef class Table(_Tabular): """ Perform an asof join between this table and another one. - Result of the join will be a new dataset, where further + Result of the join will be a new Table, where further operations can be applied. Parameters From 102f965f83ba702997b74dd72626b78970cfb91a Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:30:50 +0100 Subject: [PATCH 16/27] Add additional test cases --- python/pyarrow/tests/test_table.py | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index eca1d2393ec9..6a3d264d8f57 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2425,6 +2425,71 @@ def test_table_join_asof_collisions(): ) +@pytest.mark.dataset +def test_table_join_asof_by_length_mismatch(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + msg = "inconsistent size of by-key across inputs" + with pytest.raises(pa.lib.ArrowInvalid, match=msg): + t1.join_asof( + t2, on="on", by=["colA", "colB"], tolerance=1, + right_on="on", right_by=["colA"], + ) + + +def test_table_join_asof_by_type_mismatch(): + t1 = pa.table({ + "colA": [1, 2, 6], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99., 2., 1.], + "on": [2, 3, 4], + }) + + msg = "Expected by-key type int64 but got double for field colA in input 1" + with pytest.raises(pa.lib.ArrowInvalid, match=msg): + t1.join_asof( + t2, on="on", by=["colA"], tolerance=1, + right_on="on", right_by=["colA"], + ) + + +def test_table_join_asof_on_type_mismatch(): + t1 = pa.table({ + "colA": [1, 2, 6], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2., 3., 4.], + }) + + msg = "Expected on-key type int64 but got double for field on in input 1" + with pytest.raises(pa.lib.ArrowInvalid, match=msg): + t1.join_asof( + t2, on="on", by=["colA"], tolerance=1, + right_on="on", right_by=["colA"], + ) + + def test_table_cast_invalid(): # Casting a nullable field to non-nullable should be invalid! table = pa.table({'a': [None, 1], 'b': [None, True]}) From f645e65bbe96cf95b156fe26c42d86857080949d Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:31:03 +0100 Subject: [PATCH 17/27] Fix typo in C++ error message --- cpp/src/arrow/acero/asof_join_node.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc index 23c07b8acb95..46d31983aaae 100644 --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc @@ -1545,7 +1545,7 @@ class AsofJoinNode : public ExecNode { if (by_key_type[k] == NULLPTR) { by_key_type[k] = by_field[k]->type().get(); } else if (*by_key_type[k] != *by_field[k]->type()) { - return Status::Invalid("Expected on-key type ", *by_key_type[k], " but got ", + return Status::Invalid("Expected by-key type ", *by_key_type[k], " but got ", *by_field[k]->type(), " for field ", by_field[k]->name(), " in input ", j); } From d66a2e8089d422cfdf9bd605c628cda45e24142f Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:40:22 +0100 Subject: [PATCH 18/27] Add more detail about 'on' to doc strings --- python/pyarrow/_acero.pyx | 8 ++++++++ python/pyarrow/_dataset.pyx | 10 +++++++++- python/pyarrow/table.pxi | 12 ++++++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 266876935e06..0de0ff748304 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -443,6 +443,14 @@ class AsofJoinNodeOptions(_AsofJoinNodeOptions): left_on : str, Expression The left key on which the join operation should be performed. Can be a string column name or a field expression. + + An inexact match is used on the “on” key.i.e., a row is considered a + match iff left_on - tolerance <= right_on <= left_on. + + The input dataset must be sorted by the “on” key. Must be a single + field of a common type. + + Currently, the “on” key must be an integer, date, or timestamp type. left_by: str, Expression or list[str] The left keys on which the join operation should be performed. Each key can be a string column name or a field expression, diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 82565133cb02..ca9ecb9e53f4 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -889,8 +889,16 @@ cdef class Dataset(_Weakrefable): on : str The column from current dataset that should be used as the on key of the join operation left side. + + An inexact match is used on the “on” key.i.e., a row is considered a + match iff left_on - tolerance <= right_on <= left_on. + + The input table must be sorted by the “on” key. Must be a single + field of a common type. + + Currently, the “on” key must be an integer, date, or timestamp type. by : str or list[str] - The columns from current dataset that should be used as the by keys + The columns from current dataset that should be used as the keys of the join operation left side. tolerance : int The tolerance for inexact "on" key matching. A right row is considered diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 72603228b88e..92ed5b85d69b 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4768,8 +4768,16 @@ cdef class Table(_Tabular): on : str The column from current table that should be used as the on key of the join operation left side. + + An inexact match is used on the “on” key.i.e., a row is considered a + match iff left_on - tolerance <= right_on <= left_on. + + The input dataset must be sorted by the “on” key. Must be a single + field of a common type. + + Currently, the “on” key must be an integer, date, or timestamp type. by : str or list[str] - The columns from current table that should be used as the by keys + The columns from current table that should be used as the keys of the join operation left side. tolerance : int The tolerance for inexact "on" key matching. A right row is considered @@ -4786,7 +4794,7 @@ cdef class Table(_Tabular): on the join operation right side. When ``None`` use the same key name as the left table. right_by : str or list[str], default None - The columns from the right_table that should be used as by keys + The columns from the right_table that should be used as keys on the join operation right side. When ``None`` use the same key names as the left table. From acb082e8da4647c13a60484f654d7ca1425f7f6e Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 23 Oct 2023 20:30:55 +0100 Subject: [PATCH 19/27] Improve docstrings --- python/pyarrow/_dataset.pyx | 3 ++- python/pyarrow/table.pxi | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index b5413c446c6d..72951e3cf0f5 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -899,7 +899,8 @@ cdef class Dataset(_Weakrefable): Currently, the “on” key must be an integer, date, or timestamp type. by : str or list[str] The columns from current dataset that should be used as the keys - of the join operation left side. + of the join operation left side. The join operation is then done + only for the matches in these columns. tolerance : int The tolerance for inexact "on" key matching. A right row is considered a match with the left row `right.on - left.on <= tolerance`. The diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 92ed5b85d69b..5fbaac3d6ba9 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4778,7 +4778,8 @@ cdef class Table(_Tabular): Currently, the “on” key must be an integer, date, or timestamp type. by : str or list[str] The columns from current table that should be used as the keys - of the join operation left side. + of the join operation left side. The join operation is then done + only for the matches in these columns. tolerance : int The tolerance for inexact "on" key matching. A right row is considered a match with the left row ``right.on - left.on <= tolerance``. The From ae7e9e0fe33df1707f246f3b26899dcd679d8601 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 23 Oct 2023 20:39:18 +0100 Subject: [PATCH 20/27] Enable `use_threads` --- python/pyarrow/acero.py | 3 +-- python/pyarrow/table.pxi | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/acero.py b/python/pyarrow/acero.py index 3df69d460872..a1a1ecf0e56c 100644 --- a/python/pyarrow/acero.py +++ b/python/pyarrow/acero.py @@ -330,8 +330,7 @@ def _perform_join_asof(left_operand, left_on, left_by, "asofjoin", options=join_opts, inputs=[left_source, right_source] ) - # Last time I checked the asof join node does not support using threads. - result_table = decl.to_table(use_threads=False) + result_table = decl.to_table(use_threads=use_threads) if output_type == Table: return result_table diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 32047ca4ad05..63685d3c426d 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4937,6 +4937,7 @@ cdef class Table(_Tabular): return _pac()._perform_join_asof(self, on, by, right_table, right_on, right_by, tolerance, output_type=Table) + def __arrow_c_stream__(self, requested_schema=None): """ Export the table as an Arrow C stream PyCapsule. From 7f7b877bd573e4e1087d5ee3ff3bbee4c2bee242 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:01:42 +0000 Subject: [PATCH 21/27] Apply docstring suggestions from code review Co-authored-by: Joris Van den Bossche --- python/pyarrow/_acero.pyx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 313edcac460a..9e8cbd65be22 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -444,21 +444,22 @@ class AsofJoinNodeOptions(_AsofJoinNodeOptions): The left key on which the join operation should be performed. Can be a string column name or a field expression. - An inexact match is used on the “on” key.i.e., a row is considered a - match iff left_on - tolerance <= right_on <= left_on. + An inexact match is used on the "on" key, i.e. a row is considered a + match if and only if left_on - tolerance <= right_on <= left_on. - The input dataset must be sorted by the “on” key. Must be a single + The input dataset must be sorted by the "on" key. Must be a single field of a common type. - Currently, the “on” key must be an integer, date, or timestamp type. - left_by: str, Expression or list[str] + Currently, the "on" key must be an integer, date, or timestamp type. + left_by: str, Expression or list The left keys on which the join operation should be performed. + Exact equality is used for each field of the "by" keys. Each key can be a string column name or a field expression, or a list of such field references. right_on : str, Expression The right key on which the join operation should be performed. See `left_on` for details. - right_by: str, Expression or list[str] + right_by: str, Expression or list The right keys on which the join operation should be performed. See `left_by` for details. tolerance : int From 5fcce5f689d4ff42af13baec2816f8e49345e075 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:04:08 +0000 Subject: [PATCH 22/27] Remove tests from `test_exec_plan.py` --- python/pyarrow/tests/test_exec_plan.py | 77 -------------------------- 1 file changed, 77 deletions(-) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index 430463ee1905..c8e93ed7884c 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -323,83 +323,6 @@ def test_join_extension_array_column(): assert result["colB"] == pa.chunked_array(ext_array) -@pytest.mark.parametrize("tolerance,expected", [ - ( - 1, - { - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"], - "colC": [1., None, None, None, None], - }, - ), - ( - 3, - { - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"], - "colC": [1., None, None, 3., None], - }, - ), - ( - -5, - { - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"], - "colC": [None, None, 1., None, None], - }, - ), -]) -@pytest.mark.parametrize("use_datasets", [False, True]) -def test_join_asof(tolerance, expected, use_datasets): - # Allocate table here instead of using parametrize - # this prevents having arrow allocated memory forever around. - expected = pa.table(expected) - - t1 = pa.Table.from_pydict({ - "colA": [1, 1, 5, 6, 7], - "col2": ["a", "b", "a", "b", "f"] - }) - - t2 = pa.Table.from_pydict({ - "colB": [2, 9, 15], - "col3": ["a", "b", "g"], - "colC": [1., 3., 5.] - }) - - if use_datasets: - t1 = ds.dataset([t1]) - t2 = ds.dataset([t2]) - - r = _perform_join_asof(t1, "colA", "col2", t2, "colB", "col3", tolerance) - r = r.combine_chunks() - r = r.sort_by("colA") - assert r == expected - - -def test_table_join_asof_collisions(): - t1 = pa.table({ - "colA": [1, 2, 6], - "colB": [10, 20, 60], - "on": [1, 2, 3], - "colVals": ["a", "b", "f"] - }) - - t2 = pa.table({ - "colB": [99, 20, 10], - "colVals": ["Z", "B", "A"], - "colUniq": [100, 200, 300], - "colA": [99, 2, 1], - "on": [2, 3, 4], - }) - - msg = ( - "Columns {'colVals'} present in both tables. " - "AsofJoin does not support column collisions." - ) - with pytest.raises(ValueError, match=msg): - _perform_join_asof(t1, "on", ["colA", "colB"], t2, "on", ["colA", "colB"], 1) - - def test_group_by_ordering(): # GH-36709 - preserve ordering in groupby by setting use_threads=False table1 = pa.table({'a': [1, 2, 3, 4], 'b': ['a'] * 4}) From 7dd4e218e7c4b6ee0f6e4de9994174a8727539f3 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:20:54 +0000 Subject: [PATCH 23/27] More docstring formatting changes --- python/pyarrow/_dataset.pyx | 6 +++--- python/pyarrow/table.pxi | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 06d9c594bc3d..d8bc68e15e6b 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -896,13 +896,13 @@ cdef class Dataset(_Weakrefable): The column from current dataset that should be used as the on key of the join operation left side. - An inexact match is used on the “on” key.i.e., a row is considered a + An inexact match is used on the "on” key.i.e., a row is considered a match iff left_on - tolerance <= right_on <= left_on. - The input table must be sorted by the “on” key. Must be a single + The input table must be sorted by the "on” key. Must be a single field of a common type. - Currently, the “on” key must be an integer, date, or timestamp type. + Currently, the "on” key must be an integer, date, or timestamp type. by : str or list[str] The columns from current dataset that should be used as the keys of the join operation left side. The join operation is then done diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index a27dfd0b3dbf..04bd608a4066 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4875,13 +4875,13 @@ cdef class Table(_Tabular): The column from current table that should be used as the on key of the join operation left side. - An inexact match is used on the “on” key.i.e., a row is considered a + An inexact match is used on the "on” key.i.e., a row is considered a match iff left_on - tolerance <= right_on <= left_on. - The input dataset must be sorted by the “on” key. Must be a single + The input dataset must be sorted by the "on” key. Must be a single field of a common type. - Currently, the “on” key must be an integer, date, or timestamp type. + Currently, the "on” key must be an integer, date, or timestamp type. by : str or list[str] The columns from current table that should be used as the keys of the join operation left side. The join operation is then done From e6da4df5cae7464439ae51c92ca1f43012a331a1 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:23:28 +0000 Subject: [PATCH 24/27] Elaborate on `join_asof` description These descriptions are mostly lifted from the Pandas `merge_asof` docs. https://pandas.pydata.org/docs/reference/api/pandas.merge_asof.html#pandas.merge_asof --- python/pyarrow/_dataset.pyx | 6 ++++++ python/pyarrow/table.pxi | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index d8bc68e15e6b..d586bdbab162 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -884,6 +884,12 @@ cdef class Dataset(_Weakrefable): """ Perform an asof join between this dataset and another one. + This is similar to a left-join except that we match on nearest key rather + than equal keys. Both datasets must be sorted by the key. This type of join + is most useful for time series data that are not perfectly aligned. + + Optionally match on equivalent keys with "by" before searching with "on". + Result of the join will be a new Dataset, where further operations can be applied. diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 04bd608a4066..daefa80d381c 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4863,6 +4863,12 @@ cdef class Table(_Tabular): """ Perform an asof join between this table and another one. + This is similar to a left-join except that we match on nearest key rather + than equal keys. Both tables must be sorted by the key. This type of join + is most useful for time series data that are not perfectly aligned. + + Optionally match on equivalent keys with "by" before searching with "on". + Result of the join will be a new Table, where further operations can be applied. From 0553f1dcb1c556729839e36b59e7fc84de2c5faa Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:35:39 +0000 Subject: [PATCH 25/27] Make `join_as` example usage more complex This example now shows the behavior of duplicate values in a `by` predicate as well as how `tolerance` works. --- python/pyarrow/table.pxi | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index daefa80d381c..2cb1f8b2a97c 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4919,8 +4919,8 @@ cdef class Table(_Tabular): -------- >>> import pandas as pd >>> import pyarrow as pa - >>> df1 = pd.DataFrame({'id': [1, 2, 3], - ... 'year': [2020, 2022, 2019]}) + >>> df1 = pd.DataFrame({'id': [1, 2, 3, 3, 3], + ... 'year': [2020, 2022, 2021, 2022, 2023]}) >>> df2 = pd.DataFrame({'id': [3, 4], ... 'year': [2020, 2021], ... 'n_legs': [5, 100], @@ -4929,7 +4929,7 @@ cdef class Table(_Tabular): >>> t2 = pa.Table.from_pandas(df2).sort_by('year') >>> t1.join_asof( - ... t2, on='year', by='id', tolerance=1 + ... t2, on='year', by='id', tolerance=-2 ... ).combine_chunks().sort_by('year') pyarrow.Table id: int64 @@ -4937,10 +4937,10 @@ cdef class Table(_Tabular): n_legs: int64 animal: string ---- - id: [[3,1,2]] - year: [[2019,2020,2022]] - n_legs: [[5,null,null]] - animal: [["Brittle stars",null,null]] + id: [[1,3,2,3,3]] + year: [[2020,2021,2022,2022,2023]] + n_legs: [[null,5,null,5,null]] + animal: [[null,"Brittle stars",null,"Brittle stars",null]] """ if right_on is None: right_on = on From df46f9eac50d4b05aaaa1500574beb493a9de9cf Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:39:49 +0000 Subject: [PATCH 26/27] Remove unused import --- python/pyarrow/tests/test_exec_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_exec_plan.py b/python/pyarrow/tests/test_exec_plan.py index c8e93ed7884c..d85a2c215248 100644 --- a/python/pyarrow/tests/test_exec_plan.py +++ b/python/pyarrow/tests/test_exec_plan.py @@ -26,7 +26,7 @@ pass try: - from pyarrow.acero import _perform_join, _perform_join_asof, _filter_table + from pyarrow.acero import _perform_join, _filter_table except ImportError: pass From f30de79565a6c2a6f8f41a139ac47fda8c03062d Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 15 Mar 2024 16:18:56 +0100 Subject: [PATCH 27/27] clean-up docstrings --- python/pyarrow/_dataset.pyx | 10 +++++----- python/pyarrow/table.pxi | 31 +++++++++++++------------------ 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index cdb95e73b235..3583a3213ccb 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -899,16 +899,16 @@ cdef class Dataset(_Weakrefable): The dataset to join to the current one, acting as the right dataset in the join operation. on : str - The column from current dataset that should be used as the on key + The column from current dataset that should be used as the "on" key of the join operation left side. - An inexact match is used on the "on” key.i.e., a row is considered a - match iff left_on - tolerance <= right_on <= left_on. + An inexact match is used on the "on" key, i.e. a row is considered a + match if and only if left_on - tolerance <= right_on <= left_on. - The input table must be sorted by the "on” key. Must be a single + The input table must be sorted by the "on" key. Must be a single field of a common type. - Currently, the "on” key must be an integer, date, or timestamp type. + Currently, the "on" key must be an integer, date, or timestamp type. by : str or list[str] The columns from current dataset that should be used as the keys of the join operation left side. The join operation is then done diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index c9ff156d4d14..f73f15bb1d47 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5094,16 +5094,16 @@ cdef class Table(_Tabular): The table to join to the current one, acting as the right table in the join operation. on : str - The column from current table that should be used as the on key + The column from current table that should be used as the "on" key of the join operation left side. - An inexact match is used on the "on” key.i.e., a row is considered a - match iff left_on - tolerance <= right_on <= left_on. + An inexact match is used on the "on" key, i.e. a row is considered a + match if and only if left_on - tolerance <= right_on <= left_on. - The input dataset must be sorted by the "on” key. Must be a single + The input dataset must be sorted by the "on" key. Must be a single field of a common type. - Currently, the "on” key must be an integer, date, or timestamp type. + Currently, the "on" key must be an integer, date, or timestamp type. by : str or list[str] The columns from current table that should be used as the keys of the join operation left side. The join operation is then done @@ -5133,20 +5133,15 @@ cdef class Table(_Tabular): Example -------- - >>> import pandas as pd >>> import pyarrow as pa - >>> df1 = pd.DataFrame({'id': [1, 2, 3, 3, 3], - ... 'year': [2020, 2022, 2021, 2022, 2023]}) - >>> df2 = pd.DataFrame({'id': [3, 4], - ... 'year': [2020, 2021], - ... 'n_legs': [5, 100], - ... 'animal': ["Brittle stars", "Centipede"]}) - >>> t1 = pa.Table.from_pandas(df1).sort_by('year') - >>> t2 = pa.Table.from_pandas(df2).sort_by('year') - - >>> t1.join_asof( - ... t2, on='year', by='id', tolerance=-2 - ... ).combine_chunks().sort_by('year') + >>> t1 = pa.table({'id': [1, 3, 2, 3, 3], + ... 'year': [2020, 2021, 2022, 2022, 2023]}) + >>> t2 = pa.table({'id': [3, 4], + ... 'year': [2020, 2021], + ... 'n_legs': [5, 100], + ... 'animal': ["Brittle stars", "Centipede"]}) + + >>> t1.join_asof(t2, on='year', by='id', tolerance=-2) pyarrow.Table id: int64 year: int64