Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
078b9a9
Add `join_asof` binding
judahrand Feb 17, 2023
2aa990b
Add test for `join_asof` binding
judahrand Feb 17, 2023
dad0416
Handle removal of special Dataset columns
judahrand Feb 20, 2023
fd33b8c
Explicitly don't handle column collisions
judahrand Feb 20, 2023
09bdd04
Fix linting
judahrand Feb 20, 2023
27e5862
Fix the docs
judahrand Feb 20, 2023
312378e
Add more tests
judahrand Feb 20, 2023
8303310
Fix bug with datasets
judahrand Feb 20, 2023
46a1d2a
Make sure tests pass
judahrand Feb 20, 2023
d399267
Add `AsofJoinNodeOptions` class
judahrand Mar 5, 2023
367a17d
Use `AsofJoinNodeOptions` in `_perform_join_asof`
judahrand Mar 5, 2023
28ea072
Swap to kwargs
judahrand Mar 5, 2023
0a517f3
Fix restructuredtext formatting
judahrand Mar 5, 2023
984b159
Add test cases
judahrand Mar 5, 2023
f68d7b2
Merge remote-tracking branch 'upstream/main' into python/join_asof
judahrand Sep 8, 2023
6f8db25
Fix some typos in doc strings
judahrand Sep 8, 2023
102f965
Add additional test cases
judahrand Sep 8, 2023
f645e65
Fix typo in C++ error message
judahrand Sep 8, 2023
d66a2e8
Add more detail about 'on' to doc strings
judahrand Sep 8, 2023
84e8284
Merge branch 'main' into python/join_asof
judahrand Sep 20, 2023
acb082e
Improve docstrings
judahrand Oct 23, 2023
8a141fe
Merge remote-tracking branch 'upstream/main' into python/join_asof
judahrand Oct 23, 2023
ae7e9e0
Enable `use_threads`
judahrand Oct 23, 2023
a280fda
Merge remote-tracking branch 'upstream/main' into python/join_asof
judahrand Oct 30, 2023
cae3e92
Merge branch 'main' into python/join_asof
judahrand Jan 5, 2024
7f7b877
Apply docstring suggestions from code review
judahrand Mar 4, 2024
5fcce5f
Remove tests from `test_exec_plan.py`
judahrand Mar 4, 2024
7dd4e21
More docstring formatting changes
judahrand Mar 4, 2024
e6da4df
Elaborate on `join_asof` description
judahrand Mar 4, 2024
0553f1d
Make `join_as` example usage more complex
judahrand Mar 4, 2024
df46f9e
Remove unused import
judahrand Mar 4, 2024
6e87867
Merge remote-tracking branch 'upstream/main' into python/join_asof
judahrand Mar 5, 2024
c2339b4
Merge remote-tracking branch 'upstream/main' into python/join_asof
judahrand Mar 11, 2024
adbf06c
Merge remote-tracking branch 'upstream/main' into python/join_asof
jorisvandenbossche Mar 15, 2024
f30de79
clean-up docstrings
jorisvandenbossche Mar 15, 2024
88b3c38
Merge remote-tracking branch 'upstream/main' into python/join_asof
jorisvandenbossche Mar 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/src/arrow/acero/asof_join_node.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1242,7 +1242,7 @@ class AsofJoinNode : public ExecNode {
if (by_key_type[k] == NULLPTR) {
by_key_type[k] = by_field[k]->type().get();
} else if (*by_key_type[k] != *by_field[k]->type()) {
return Status::Invalid("Expected on-key type ", *by_key_type[k], " but got ",
return Status::Invalid("Expected by-key type ", *by_key_type[k], " but got ",
Comment thread
judahrand marked this conversation as resolved.
Resolved
*by_field[k]->type(), " for field ", by_field[k]->name(),
" in input ", j);
}
Expand Down
79 changes: 79 additions & 0 deletions python/pyarrow/_acero.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,85 @@ class HashJoinNodeOptions(_HashJoinNodeOptions):
)


cdef class _AsofJoinNodeOptions(ExecNodeOptions):

def _set_options(self, left_on, left_by, right_on, right_by, tolerance):
cdef:
vector[CFieldRef] c_left_by
vector[CFieldRef] c_right_by
CAsofJoinKeys c_left_keys
CAsofJoinKeys c_right_keys
vector[CAsofJoinKeys] c_input_keys

# Prepare left AsofJoinNodeOption::Keys
if not isinstance(left_by, (list, tuple)):
left_by = [left_by]
for key in left_by:
c_left_by.push_back(_ensure_field_ref(key))

c_left_keys.on_key = _ensure_field_ref(left_on)
c_left_keys.by_key = c_left_by

c_input_keys.push_back(c_left_keys)

# Prepare right AsofJoinNodeOption::Keys
if not isinstance(right_by, (list, tuple)):
right_by = [right_by]
for key in right_by:
c_right_by.push_back(_ensure_field_ref(key))

c_right_keys.on_key = _ensure_field_ref(right_on)
c_right_keys.by_key = c_right_by

c_input_keys.push_back(c_right_keys)

self.wrapped.reset(
new CAsofJoinNodeOptions(
c_input_keys,
tolerance,
)
)


class AsofJoinNodeOptions(_AsofJoinNodeOptions):
"""
Make a node which implements 'as of join' operation.

This is the option class for the "asofjoin" node factory.

Parameters
----------
left_on : str, Expression
The left key on which the join operation should be performed.
Can be a string column name or a field expression.

An inexact match is used on the "on" key, i.e. a row is considered a
match if and only if left_on - tolerance <= right_on <= left_on.

The input dataset must be sorted by the "on" key. Must be a single
field of a common type.

Currently, the "on" key must be an integer, date, or timestamp type.
left_by: str, Expression or list
The left keys on which the join operation should be performed.
Comment thread
judahrand marked this conversation as resolved.
Resolved
Exact equality is used for each field of the "by" keys.
Each key can be a string column name or a field expression,
or a list of such field references.
right_on : str, Expression
The right key on which the join operation should be performed.
See `left_on` for details.
right_by: str, Expression or list
The right keys on which the join operation should be performed.
See `left_by` for details.
tolerance : int
The tolerance to use for the asof join. The tolerance is interpreted in
the same units as the "on" key.
"""

def __init__(self, left_on, left_by, right_on, right_by, tolerance):
self._set_options(left_on, left_by, right_on, right_by, tolerance)


cdef class Declaration(_Weakrefable):
"""
Helper class for declaring the nodes of an ExecPlan.
Expand Down
64 changes: 64 additions & 0 deletions python/pyarrow/_dataset.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -880,6 +880,70 @@ cdef class Dataset(_Weakrefable):
output_type=InMemoryDataset
)

def join_asof(self, right_dataset, on, by, tolerance, right_on=None, right_by=None):
"""
Perform an asof join between this dataset and another one.

This is similar to a left-join except that we match on nearest key rather
than equal keys. Both datasets must be sorted by the key. This type of join
is most useful for time series data that are not perfectly aligned.

Optionally match on equivalent keys with "by" before searching with "on".

Result of the join will be a new Dataset, where further
operations can be applied.

Parameters
----------
right_dataset : dataset
The dataset to join to the current one, acting as the right dataset
in the join operation.
on : str
The column from current dataset that should be used as the "on" key
of the join operation left side.

An inexact match is used on the "on" key, i.e. a row is considered a
match if and only if left_on - tolerance <= right_on <= left_on.

The input table must be sorted by the "on" key. Must be a single
field of a common type.

Currently, the "on" key must be an integer, date, or timestamp type.
by : str or list[str]
The columns from current dataset that should be used as the keys
of the join operation left side. The join operation is then done
only for the matches in these columns.
tolerance : int

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the "on" key is a timestamp column, what value can be used here? (not an int?)

@judahrand judahrand Mar 5, 2023 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good question actually... I'm not 100% sure how this is intended to work. The C++ implementation exclusively accepts an int64_t for the tolerance. It simply states that it will use the same units as the on column... it is unclear what that means. I'd assumed it meant the resolution of the timestamp in a timestamp case.

The tolerance for inexact "on" key matching. A right row is considered
a match with the left row `right.on - left.on <= tolerance`. The
`tolerance` may be:

- negative, in which case a past-as-of-join occurs;
- or positive, in which case a future-as-of-join occurs;
- or zero, in which case an exact-as-of-join occurs.

The tolerance is interpreted in the same units as the "on" key.
right_on : str or list[str], default None
The columns from the right_dataset that should be used as the on key
on the join operation right side.
When ``None`` use the same key name as the left dataset.
right_by : str or list[str], default None
The columns from the right_dataset that should be used as by keys
on the join operation right side.
When ``None`` use the same key names as the left dataset.

Returns
-------
InMemoryDataset
"""
if right_on is None:
right_on = on
if right_by is None:
right_by = by
return _pac()._perform_join_asof(self, on, by,
right_dataset, right_on, right_by,
tolerance, output_type=InMemoryDataset)


cdef class InMemoryDataset(Dataset):
"""
Expand Down
87 changes: 87 additions & 0 deletions python/pyarrow/acero.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
AggregateNodeOptions,
OrderByNodeOptions,
HashJoinNodeOptions,
AsofJoinNodeOptions,
)
except ImportError as exc:
raise ImportError(
Expand Down Expand Up @@ -253,6 +254,92 @@ def _perform_join(join_type, left_operand, left_keys,
raise TypeError("Unsupported output type")


def _perform_join_asof(left_operand, left_on, left_by,
right_operand, right_on, right_by,
tolerance, use_threads=True,
output_type=Table):
"""
Perform asof join of two tables or datasets.

The result will be an output table with the result of the join operation

Parameters
----------
left_operand : Table or Dataset
The left operand for the join operation.
left_on : str
The left key (or keys) on which the join operation should be performed.
left_by: str or list[str]
The left key (or keys) on which the join operation should be performed.
right_operand : Table or Dataset
The right operand for the join operation.
right_on : str or list[str]
The right key (or keys) on which the join operation should be performed.
right_by: str or list[str]
The right key (or keys) on which the join operation should be performed.
tolerance : int
The tolerance to use for the asof join. The tolerance is interpreted in
the same units as the "on" key.
output_type: Table or InMemoryDataset
The output type for the exec plan result.

Returns
-------
result_table : Table or InMemoryDataset
"""
if not isinstance(left_operand, (Table, ds.Dataset)):
raise TypeError(f"Expected Table or Dataset, got {type(left_operand)}")
if not isinstance(right_operand, (Table, ds.Dataset)):
raise TypeError(f"Expected Table or Dataset, got {type(right_operand)}")

if not isinstance(left_by, (tuple, list)):
left_by = [left_by]
if not isinstance(right_by, (tuple, list)):
right_by = [right_by]

# AsofJoin does not return on or by columns for right_operand.
right_columns = [
col for col in right_operand.schema.names
if col not in [right_on] + right_by
]
columns_collisions = set(left_operand.schema.names) & set(right_columns)
if columns_collisions:
raise ValueError(
"Columns {} present in both tables. AsofJoin does not support "
"column collisions.".format(columns_collisions),
)

# Add the join node to the execplan
if isinstance(left_operand, ds.Dataset):
left_source = _dataset_to_decl(left_operand, use_threads=use_threads)
else:
left_source = Declaration(
"table_source", TableSourceNodeOptions(left_operand),
)
if isinstance(right_operand, ds.Dataset):
right_source = _dataset_to_decl(right_operand, use_threads=use_threads)
else:
right_source = Declaration(
"table_source", TableSourceNodeOptions(right_operand)
)

join_opts = AsofJoinNodeOptions(
left_on, left_by, right_on, right_by, tolerance
)
decl = Declaration(
"asofjoin", options=join_opts, inputs=[left_source, right_source]
)

result_table = decl.to_table(use_threads=use_threads)

if output_type == Table:
return result_table
elif output_type == ds.InMemoryDataset:
return ds.InMemoryDataset(result_table)
else:
raise TypeError("Unsupported output type")


def _filter_table(table, expression):
"""Filter rows of a table based on the provided expression.

Expand Down
7 changes: 7 additions & 0 deletions python/pyarrow/includes/libarrow_acero.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,13 @@ cdef extern from "arrow/acero/options.h" namespace "arrow::acero" nogil:
c_string output_suffix_for_left,
c_string output_suffix_for_right)

cdef struct CAsofJoinKeys "arrow::acero::AsofJoinNodeOptions::Keys":
CFieldRef on_key
vector[CFieldRef] by_key

cdef cppclass CAsofJoinNodeOptions "arrow::acero::AsofJoinNodeOptions"(CExecNodeOptions):
CAsofJoinNodeOptions(vector[CAsofJoinKeys] keys, int64_t tolerance)


cdef extern from "arrow/acero/exec_plan.h" namespace "arrow::acero" nogil:
cdef cppclass CDeclaration "arrow::acero::Declaration":
Expand Down
86 changes: 86 additions & 0 deletions python/pyarrow/table.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -5333,6 +5333,92 @@ cdef class Table(_Tabular):
output_type=Table
)

def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None):
"""
Perform an asof join between this table and another one.

Comment on lines +5338 to +5339

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would still like to see a bit more expanded explanation (apart from the individual keyword explanations) about what and asof join exactly is.

Something indicating it does 1) an inexact join, 2) on a sorted dataset, 3) potentially first joining on other attributes, and 4) typically useful for time series data that are not perfectly aligned. Are that the most relevant characteristics?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is similar to a left-join except that we match on nearest key rather
than equal keys. Both tables must be sorted by the key. This type of join
is most useful for time series data that are not perfectly aligned.

Optionally match on equivalent keys with "by" before searching with "on".

Result of the join will be a new Table, where further
operations can be applied.

Parameters
----------
right_table : Table
The table to join to the current one, acting as the right table
in the join operation.
on : str
The column from current table that should be used as the "on" key
of the join operation left side.

An inexact match is used on the "on" key, i.e. a row is considered a
match if and only if left_on - tolerance <= right_on <= left_on.

The input dataset must be sorted by the "on" key. Must be a single
field of a common type.

Currently, the "on" key must be an integer, date, or timestamp type.
by : str or list[str]
The columns from current table that should be used as the keys
of the join operation left side. The join operation is then done
only for the matches in these columns.
tolerance : int
The tolerance for inexact "on" key matching. A right row is considered
a match with the left row ``right.on - left.on <= tolerance``. The
``tolerance`` may be:

- negative, in which case a past-as-of-join occurs;
- or positive, in which case a future-as-of-join occurs;
- or zero, in which case an exact-as-of-join occurs.

The tolerance is interpreted in the same units as the "on" key.
right_on : str or list[str], default None
The columns from the right_table that should be used as the on key
on the join operation right side.
When ``None`` use the same key name as the left table.
right_by : str or list[str], default None
The columns from the right_table that should be used as keys
on the join operation right side.
When ``None`` use the same key names as the left table.

Returns
-------
Table

Example
--------
>>> import pyarrow as pa
>>> t1 = pa.table({'id': [1, 3, 2, 3, 3],
... 'year': [2020, 2021, 2022, 2022, 2023]})
>>> t2 = pa.table({'id': [3, 4],
... 'year': [2020, 2021],
... 'n_legs': [5, 100],
... 'animal': ["Brittle stars", "Centipede"]})

>>> t1.join_asof(t2, on='year', by='id', tolerance=-2)
pyarrow.Table
id: int64
year: int64
n_legs: int64
animal: string
----
id: [[1,3,2,3,3]]
year: [[2020,2021,2022,2022,2023]]
n_legs: [[null,5,null,5,null]]
animal: [[null,"Brittle stars",null,"Brittle stars",null]]
"""
if right_on is None:
right_on = on
if right_by is None:
right_by = by
return _pac()._perform_join_asof(self, on, by,
right_table, right_on, right_by,
tolerance, output_type=Table)

def __arrow_c_stream__(self, requested_schema=None):
"""
Export the table as an Arrow C stream PyCapsule.
Expand Down
Loading