From c53e86b4baf4bba8440cf48af88d9f133e2dbd8b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 25 Jan 2023 12:25:46 +0100 Subject: [PATCH 1/2] GH-33825: [Python] Expose pyarrow.dataset.get_partition_keys publicly (get key/value from partition expression) --- docs/source/python/api/dataset.rst | 8 ++++++++ python/pyarrow/_dataset.pyx | 4 ++-- python/pyarrow/dataset.py | 3 ++- python/pyarrow/tests/test_dataset.py | 11 ++++++----- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/source/python/api/dataset.rst b/docs/source/python/api/dataset.rst index 866c67440c56..d711562a7171 100644 --- a/docs/source/python/api/dataset.rst +++ b/docs/source/python/api/dataset.rst @@ -71,3 +71,11 @@ Classes Expression InMemoryDataset WrittenFile + +Helper functions +----------------- + +.. autosummary:: + :toctree: ../generated/ + + get_partition_keys diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 5f1610c384fb..c258eb4160a7 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -1134,7 +1134,7 @@ cdef class FileFragment(Fragment): else: # parquet has a subclass -> type embedded in class name typ = "" - partition_dict = _get_partition_keys(self.partition_expression) + partition_dict = get_partition_keys(self.partition_expression) partition = ", ".join( [f"{key}={val}" for key, val in partition_dict.items()] ) @@ -2889,7 +2889,7 @@ cdef class Scanner(_Weakrefable): return reader -def _get_partition_keys(Expression partition_expression): +def get_partition_keys(Expression partition_expression): """ Extract partition keys (equality constraints between a field and a scalar) from an expression as a dict mapping the field's name to its value. diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index de9469de445b..c4da9686ecf0 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -47,7 +47,8 @@ UnionDataset, UnionDatasetFactory, WrittenFile, - _get_partition_keys, + get_partition_keys, + get_partition_keys as _get_partition_keys, # keep for backwards compatibility _filesystemdataset_write, ) # keep Expression functionality exposed here for backwards compatibility diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 27edc8afadd4..54a40944d8fb 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -740,15 +740,16 @@ def test_expression_arithmetic_operators(): def test_partition_keys(): a, b, c = [ds.field(f) == f for f in 'abc'] - assert ds._get_partition_keys(a) == {'a': 'a'} - assert ds._get_partition_keys(a & b & c) == {f: f for f in 'abc'} + assert ds.get_partition_keys(a) == {'a': 'a'} + assert ds.get_partition_keys(a) == ds._get_partition_keys(a) + assert ds.get_partition_keys(a & b & c) == {f: f for f in 'abc'} nope = ds.field('d') >= 3 - assert ds._get_partition_keys(nope) == {} - assert ds._get_partition_keys(a & nope) == {'a': 'a'} + assert ds.get_partition_keys(nope) == {} + assert ds.get_partition_keys(a & nope) == {'a': 'a'} null = ds.field('a').is_null() - assert ds._get_partition_keys(null) == {'a': None} + assert ds.get_partition_keys(null) == {'a': None} @pytest.mark.parquet From 337939ccd0a53bceccb99b9e6699d7c19bb106b2 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 25 Jan 2023 13:08:46 +0100 Subject: [PATCH 2/2] update docstring --- python/pyarrow/_dataset.pyx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index c258eb4160a7..d176e055e96a 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -2898,6 +2898,17 @@ def get_partition_keys(Expression partition_expression): will be conjunctions of equality conditions and are accessible through this function. Other subexpressions will be ignored. + Parameters + ---------- + partition_expression : pyarrow.dataset.Expression + + Returns + ------- + dict + + Examples + -------- + For example, an expression of is converted to {'part': 'A', 'year': 2016}