diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc index e96d5ad44a9e..cf0d475c1d77 100644 --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc @@ -1242,7 +1242,7 @@ class AsofJoinNode : public ExecNode { if (by_key_type[k] == NULLPTR) { by_key_type[k] = by_field[k]->type().get(); } else if (*by_key_type[k] != *by_field[k]->type()) { - return Status::Invalid("Expected on-key type ", *by_key_type[k], " but got ", + return Status::Invalid("Expected by-key type ", *by_key_type[k], " but got ", *by_field[k]->type(), " for field ", by_field[k]->name(), " in input ", j); } diff --git a/python/pyarrow/_acero.pyx b/python/pyarrow/_acero.pyx index 1c9b2f75c39f..9e8cbd65be22 100644 --- a/python/pyarrow/_acero.pyx +++ b/python/pyarrow/_acero.pyx @@ -392,6 +392,85 @@ class HashJoinNodeOptions(_HashJoinNodeOptions): ) +cdef class _AsofJoinNodeOptions(ExecNodeOptions): + + def _set_options(self, left_on, left_by, right_on, right_by, tolerance): + cdef: + vector[CFieldRef] c_left_by + vector[CFieldRef] c_right_by + CAsofJoinKeys c_left_keys + CAsofJoinKeys c_right_keys + vector[CAsofJoinKeys] c_input_keys + + # Prepare left AsofJoinNodeOption::Keys + if not isinstance(left_by, (list, tuple)): + left_by = [left_by] + for key in left_by: + c_left_by.push_back(_ensure_field_ref(key)) + + c_left_keys.on_key = _ensure_field_ref(left_on) + c_left_keys.by_key = c_left_by + + c_input_keys.push_back(c_left_keys) + + # Prepare right AsofJoinNodeOption::Keys + if not isinstance(right_by, (list, tuple)): + right_by = [right_by] + for key in right_by: + c_right_by.push_back(_ensure_field_ref(key)) + + c_right_keys.on_key = _ensure_field_ref(right_on) + c_right_keys.by_key = c_right_by + + c_input_keys.push_back(c_right_keys) + + self.wrapped.reset( + new CAsofJoinNodeOptions( + c_input_keys, + tolerance, + ) + ) + + +class AsofJoinNodeOptions(_AsofJoinNodeOptions): + """ + Make a node which implements 'as of join' operation. + + This is the option class for the "asofjoin" node factory. + + Parameters + ---------- + left_on : str, Expression + The left key on which the join operation should be performed. + Can be a string column name or a field expression. + + An inexact match is used on the "on" key, i.e. a row is considered a + match if and only if left_on - tolerance <= right_on <= left_on. + + The input dataset must be sorted by the "on" key. Must be a single + field of a common type. + + Currently, the "on" key must be an integer, date, or timestamp type. + left_by: str, Expression or list + The left keys on which the join operation should be performed. + Exact equality is used for each field of the "by" keys. + Each key can be a string column name or a field expression, + or a list of such field references. + right_on : str, Expression + The right key on which the join operation should be performed. + See `left_on` for details. + right_by: str, Expression or list + The right keys on which the join operation should be performed. + See `left_by` for details. + tolerance : int + The tolerance to use for the asof join. The tolerance is interpreted in + the same units as the "on" key. + """ + + def __init__(self, left_on, left_by, right_on, right_by, tolerance): + self._set_options(left_on, left_by, right_on, right_by, tolerance) + + cdef class Declaration(_Weakrefable): """ Helper class for declaring the nodes of an ExecPlan. diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 8b9e62d62887..3583a3213ccb 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -880,6 +880,70 @@ cdef class Dataset(_Weakrefable): output_type=InMemoryDataset ) + def join_asof(self, right_dataset, on, by, tolerance, right_on=None, right_by=None): + """ + Perform an asof join between this dataset and another one. + + This is similar to a left-join except that we match on nearest key rather + than equal keys. Both datasets must be sorted by the key. This type of join + is most useful for time series data that are not perfectly aligned. + + Optionally match on equivalent keys with "by" before searching with "on". + + Result of the join will be a new Dataset, where further + operations can be applied. + + Parameters + ---------- + right_dataset : dataset + The dataset to join to the current one, acting as the right dataset + in the join operation. + on : str + The column from current dataset that should be used as the "on" key + of the join operation left side. + + An inexact match is used on the "on" key, i.e. a row is considered a + match if and only if left_on - tolerance <= right_on <= left_on. + + The input table must be sorted by the "on" key. Must be a single + field of a common type. + + Currently, the "on" key must be an integer, date, or timestamp type. + by : str or list[str] + The columns from current dataset that should be used as the keys + of the join operation left side. The join operation is then done + only for the matches in these columns. + tolerance : int + The tolerance for inexact "on" key matching. A right row is considered + a match with the left row `right.on - left.on <= tolerance`. The + `tolerance` may be: + + - negative, in which case a past-as-of-join occurs; + - or positive, in which case a future-as-of-join occurs; + - or zero, in which case an exact-as-of-join occurs. + + The tolerance is interpreted in the same units as the "on" key. + right_on : str or list[str], default None + The columns from the right_dataset that should be used as the on key + on the join operation right side. + When ``None`` use the same key name as the left dataset. + right_by : str or list[str], default None + The columns from the right_dataset that should be used as by keys + on the join operation right side. + When ``None`` use the same key names as the left dataset. + + Returns + ------- + InMemoryDataset + """ + if right_on is None: + right_on = on + if right_by is None: + right_by = by + return _pac()._perform_join_asof(self, on, by, + right_dataset, right_on, right_by, + tolerance, output_type=InMemoryDataset) + cdef class InMemoryDataset(Dataset): """ diff --git a/python/pyarrow/acero.py b/python/pyarrow/acero.py index a5583c9e657d..619e1fce393a 100644 --- a/python/pyarrow/acero.py +++ b/python/pyarrow/acero.py @@ -35,6 +35,7 @@ AggregateNodeOptions, OrderByNodeOptions, HashJoinNodeOptions, + AsofJoinNodeOptions, ) except ImportError as exc: raise ImportError( @@ -253,6 +254,92 @@ def _perform_join(join_type, left_operand, left_keys, raise TypeError("Unsupported output type") +def _perform_join_asof(left_operand, left_on, left_by, + right_operand, right_on, right_by, + tolerance, use_threads=True, + output_type=Table): + """ + Perform asof join of two tables or datasets. + + The result will be an output table with the result of the join operation + + Parameters + ---------- + left_operand : Table or Dataset + The left operand for the join operation. + left_on : str + The left key (or keys) on which the join operation should be performed. + left_by: str or list[str] + The left key (or keys) on which the join operation should be performed. + right_operand : Table or Dataset + The right operand for the join operation. + right_on : str or list[str] + The right key (or keys) on which the join operation should be performed. + right_by: str or list[str] + The right key (or keys) on which the join operation should be performed. + tolerance : int + The tolerance to use for the asof join. The tolerance is interpreted in + the same units as the "on" key. + output_type: Table or InMemoryDataset + The output type for the exec plan result. + + Returns + ------- + result_table : Table or InMemoryDataset + """ + if not isinstance(left_operand, (Table, ds.Dataset)): + raise TypeError(f"Expected Table or Dataset, got {type(left_operand)}") + if not isinstance(right_operand, (Table, ds.Dataset)): + raise TypeError(f"Expected Table or Dataset, got {type(right_operand)}") + + if not isinstance(left_by, (tuple, list)): + left_by = [left_by] + if not isinstance(right_by, (tuple, list)): + right_by = [right_by] + + # AsofJoin does not return on or by columns for right_operand. + right_columns = [ + col for col in right_operand.schema.names + if col not in [right_on] + right_by + ] + columns_collisions = set(left_operand.schema.names) & set(right_columns) + if columns_collisions: + raise ValueError( + "Columns {} present in both tables. AsofJoin does not support " + "column collisions.".format(columns_collisions), + ) + + # Add the join node to the execplan + if isinstance(left_operand, ds.Dataset): + left_source = _dataset_to_decl(left_operand, use_threads=use_threads) + else: + left_source = Declaration( + "table_source", TableSourceNodeOptions(left_operand), + ) + if isinstance(right_operand, ds.Dataset): + right_source = _dataset_to_decl(right_operand, use_threads=use_threads) + else: + right_source = Declaration( + "table_source", TableSourceNodeOptions(right_operand) + ) + + join_opts = AsofJoinNodeOptions( + left_on, left_by, right_on, right_by, tolerance + ) + decl = Declaration( + "asofjoin", options=join_opts, inputs=[left_source, right_source] + ) + + result_table = decl.to_table(use_threads=use_threads) + + if output_type == Table: + return result_table + elif output_type == ds.InMemoryDataset: + return ds.InMemoryDataset(result_table) + else: + raise TypeError("Unsupported output type") + + def _filter_table(table, expression): """Filter rows of a table based on the provided expression. diff --git a/python/pyarrow/includes/libarrow_acero.pxd b/python/pyarrow/includes/libarrow_acero.pxd index bb1e3646c91b..dc9babee190e 100644 --- a/python/pyarrow/includes/libarrow_acero.pxd +++ b/python/pyarrow/includes/libarrow_acero.pxd @@ -76,6 +76,13 @@ cdef extern from "arrow/acero/options.h" namespace "arrow::acero" nogil: c_string output_suffix_for_left, c_string output_suffix_for_right) + cdef struct CAsofJoinKeys "arrow::acero::AsofJoinNodeOptions::Keys": + CFieldRef on_key + vector[CFieldRef] by_key + + cdef cppclass CAsofJoinNodeOptions "arrow::acero::AsofJoinNodeOptions"(CExecNodeOptions): + CAsofJoinNodeOptions(vector[CAsofJoinKeys] keys, int64_t tolerance) + cdef extern from "arrow/acero/exec_plan.h" namespace "arrow::acero" nogil: cdef cppclass CDeclaration "arrow::acero::Declaration": diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 2ec19db1d101..164427d2747a 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5333,6 +5333,92 @@ cdef class Table(_Tabular): output_type=Table ) + def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): + """ + Perform an asof join between this table and another one. + + This is similar to a left-join except that we match on nearest key rather + than equal keys. Both tables must be sorted by the key. This type of join + is most useful for time series data that are not perfectly aligned. + + Optionally match on equivalent keys with "by" before searching with "on". + + Result of the join will be a new Table, where further + operations can be applied. + + Parameters + ---------- + right_table : Table + The table to join to the current one, acting as the right table + in the join operation. + on : str + The column from current table that should be used as the "on" key + of the join operation left side. + + An inexact match is used on the "on" key, i.e. a row is considered a + match if and only if left_on - tolerance <= right_on <= left_on. + + The input dataset must be sorted by the "on" key. Must be a single + field of a common type. + + Currently, the "on" key must be an integer, date, or timestamp type. + by : str or list[str] + The columns from current table that should be used as the keys + of the join operation left side. The join operation is then done + only for the matches in these columns. + tolerance : int + The tolerance for inexact "on" key matching. A right row is considered + a match with the left row ``right.on - left.on <= tolerance``. The + ``tolerance`` may be: + + - negative, in which case a past-as-of-join occurs; + - or positive, in which case a future-as-of-join occurs; + - or zero, in which case an exact-as-of-join occurs. + + The tolerance is interpreted in the same units as the "on" key. + right_on : str or list[str], default None + The columns from the right_table that should be used as the on key + on the join operation right side. + When ``None`` use the same key name as the left table. + right_by : str or list[str], default None + The columns from the right_table that should be used as keys + on the join operation right side. + When ``None`` use the same key names as the left table. + + Returns + ------- + Table + + Example + -------- + >>> import pyarrow as pa + >>> t1 = pa.table({'id': [1, 3, 2, 3, 3], + ... 'year': [2020, 2021, 2022, 2022, 2023]}) + >>> t2 = pa.table({'id': [3, 4], + ... 'year': [2020, 2021], + ... 'n_legs': [5, 100], + ... 'animal': ["Brittle stars", "Centipede"]}) + + >>> t1.join_asof(t2, on='year', by='id', tolerance=-2) + pyarrow.Table + id: int64 + year: int64 + n_legs: int64 + animal: string + ---- + id: [[1,3,2,3,3]] + year: [[2020,2021,2022,2022,2023]] + n_legs: [[null,5,null,5,null]] + animal: [[null,"Brittle stars",null,"Brittle stars",null]] + """ + if right_on is None: + right_on = on + if right_by is None: + right_by = by + return _pac()._perform_join_asof(self, on, by, + right_table, right_on, right_by, + tolerance, output_type=Table) + def __arrow_c_stream__(self, requested_schema=None): """ Export the table as an Arrow C stream PyCapsule. diff --git a/python/pyarrow/tests/test_acero.py b/python/pyarrow/tests/test_acero.py index a43606013027..1bdfabd8f583 100644 --- a/python/pyarrow/tests/test_acero.py +++ b/python/pyarrow/tests/test_acero.py @@ -30,6 +30,7 @@ AggregateNodeOptions, OrderByNodeOptions, HashJoinNodeOptions, + AsofJoinNodeOptions, ) except ImportError: pass @@ -342,6 +343,40 @@ def test_hash_join(): assert result.sort_by("a").equals(expected) +def test_asof_join(): + left = pa.table({'key': [1, 2, 3], 'ts': [1, 1, 1], 'a': [4, 5, 6]}) + left_source = Declaration("table_source", options=TableSourceNodeOptions(left)) + right = pa.table({'key': [2, 3, 4], 'ts': [2, 5, 2], 'b': [4, 5, 6]}) + right_source = Declaration("table_source", options=TableSourceNodeOptions(right)) + + # asof join + join_opts = AsofJoinNodeOptions( + left_on="ts", left_by=["key"], + right_on="ts", right_by=["key"], + tolerance=1, + ) + joined = Declaration( + "asofjoin", options=join_opts, inputs=[left_source, right_source] + ) + result = joined.to_table() + expected = pa.table( + [[1, 2, 3], [1, 1, 1], [4, 5, 6], [None, 4, None]], + names=["key", "ts", "a", "b"]) + assert result == expected + + for by in [field("key"), ["key"], [field("key")]]: + for on in [field("ts"), "ts"]: + join_opts = AsofJoinNodeOptions( + left_on=on, left_by=by, + right_on=on, right_by=by, + tolerance=1, + ) + joined = Declaration( + "asofjoin", options=join_opts, inputs=[left_source, right_source]) + result = joined.to_table() + assert result == expected + + @pytest.mark.dataset def test_scan(tempdir): table = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 8e2039038588..3d77214c174c 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5075,6 +5075,120 @@ def test_dataset_join_collisions(tempdir): ], names=["colA", "colB", "colVals", "colB_r", "colVals_r"]) +@pytest.mark.dataset +def test_dataset_join_asof(tempdir): + t1 = pa.Table.from_pydict({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"] + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.Table.from_pydict({ + "colB": [2, 9, 15], + "col3": ["a", "b", "g"], + "colC": [1., 3., 5.] + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + result = ds1.join_asof( + ds2, on="colA", by="col2", tolerance=1, + right_on="colB", right_by="col3", + ) + assert result.to_table().sort_by("colA") == pa.table({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, None, None], + }) + + +@pytest.mark.dataset +def test_dataset_join_asof_multiple_by(tempdir): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + result = ds1.join_asof( + ds2, on="on", by=["colA", "colB"], tolerance=1 + ) + assert result.to_table().sort_by("colA") == pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": [None, "B", None], + }) + + +@pytest.mark.dataset +def test_dataset_join_asof_empty_by(tempdir): + t1 = pa.table({ + "on": [1, 2, 3], + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "on": [2, 3, 4], + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + result = ds1.join_asof( + ds2, on="on", by=[], tolerance=1 + ) + assert result.to_table() == pa.table({ + "on": [1, 2, 3], + "colVals": ["Z", "Z", "B"], + }) + + +@pytest.mark.dataset +def test_dataset_join_asof_collisions(tempdir): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": ["a", "b", "f"] + }) + ds.write_dataset(t1, tempdir / "t1", format="ipc") + ds1 = ds.dataset(tempdir / "t1", format="ipc") + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + ds.write_dataset(t2, tempdir / "t2", format="ipc") + ds2 = ds.dataset(tempdir / "t2", format="ipc") + + msg = ( + "Columns {'colVals'} present in both tables. " + "AsofJoin does not support column collisions." + ) + with pytest.raises(ValueError, match=msg): + ds1.join_asof( + ds2, on="on", by=["colA", "colB"], tolerance=1, + right_on="on", right_by=["colA", "colB"], + ) + + @pytest.mark.parametrize('dstype', [ "fs", "mem" ]) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 374a29e956e3..d0a7ccacac0d 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2780,6 +2780,169 @@ def test_table_join_many_columns(): }) +@pytest.mark.dataset +def test_table_join_asof(): + t1 = pa.Table.from_pydict({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"] + }) + + t2 = pa.Table.from_pydict({ + "colB": [2, 9, 15], + "col3": ["a", "b", "g"], + "colC": [1., 3., 5.] + }) + + r = t1.join_asof( + t2, on="colA", by="col2", tolerance=1, + right_on="colB", right_by="col3", + ) + assert r.combine_chunks() == pa.table({ + "colA": [1, 1, 5, 6, 7], + "col2": ["a", "b", "a", "b", "f"], + "colC": [1., None, None, None, None], + }) + + +@pytest.mark.dataset +def test_table_join_asof_multiple_by(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + result = t1.join_asof( + t2, on="on", by=["colA", "colB"], tolerance=1 + ) + assert result.sort_by("colA") == pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": [None, "B", None], + }) + + +@pytest.mark.dataset +def test_table_join_asof_empty_by(): + t1 = pa.table({ + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "on": [2, 3, 4], + }) + + result = t1.join_asof( + t2, on="on", by=[], tolerance=1 + ) + assert result == pa.table({ + "on": [1, 2, 3], + "colVals": ["Z", "Z", "B"], + }) + + +@pytest.mark.dataset +def test_table_join_asof_collisions(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + "colVals": ["a", "b", "f"] + }) + + t2 = pa.table({ + "colB": [99, 20, 10], + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + msg = ( + "Columns {'colVals'} present in both tables. " + "AsofJoin does not support column collisions." + ) + with pytest.raises(ValueError, match=msg): + t1.join_asof( + t2, on="on", by=["colA", "colB"], tolerance=1, + right_on="on", right_by=["colA", "colB"], + ) + + +@pytest.mark.dataset +def test_table_join_asof_by_length_mismatch(): + t1 = pa.table({ + "colA": [1, 2, 6], + "colB": [10, 20, 60], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2, 3, 4], + }) + + msg = "inconsistent size of by-key across inputs" + with pytest.raises(pa.lib.ArrowInvalid, match=msg): + t1.join_asof( + t2, on="on", by=["colA", "colB"], tolerance=1, + right_on="on", right_by=["colA"], + ) + + +def test_table_join_asof_by_type_mismatch(): + t1 = pa.table({ + "colA": [1, 2, 6], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99., 2., 1.], + "on": [2, 3, 4], + }) + + msg = "Expected by-key type int64 but got double for field colA in input 1" + with pytest.raises(pa.lib.ArrowInvalid, match=msg): + t1.join_asof( + t2, on="on", by=["colA"], tolerance=1, + right_on="on", right_by=["colA"], + ) + + +def test_table_join_asof_on_type_mismatch(): + t1 = pa.table({ + "colA": [1, 2, 6], + "on": [1, 2, 3], + }) + + t2 = pa.table({ + "colVals": ["Z", "B", "A"], + "colUniq": [100, 200, 300], + "colA": [99, 2, 1], + "on": [2., 3., 4.], + }) + + msg = "Expected on-key type int64 but got double for field on in input 1" + with pytest.raises(pa.lib.ArrowInvalid, match=msg): + t1.join_asof( + t2, on="on", by=["colA"], tolerance=1, + right_on="on", right_by=["colA"], + ) + + @pytest.mark.parametrize( ('cls'), [