diff --git a/docs/source/python/api/dataset.rst b/docs/source/python/api/dataset.rst index 866c67440c56..d711562a7171 100644 --- a/docs/source/python/api/dataset.rst +++ b/docs/source/python/api/dataset.rst @@ -71,3 +71,11 @@ Classes Expression InMemoryDataset WrittenFile + +Helper functions +----------------- + +.. autosummary:: + :toctree: ../generated/ + + get_partition_keys diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index d5385fd40254..ba0a2860ecda 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -1824,7 +1824,7 @@ cdef class FileFragment(Fragment): else: # parquet has a subclass -> type embedded in class name typ = "" - partition_dict = _get_partition_keys(self.partition_expression) + partition_dict = get_partition_keys(self.partition_expression) partition = ", ".join( [f"{key}={val}" for key, val in partition_dict.items()] ) @@ -3520,7 +3520,7 @@ cdef class Scanner(_Weakrefable): return reader -def _get_partition_keys(Expression partition_expression): +def get_partition_keys(Expression partition_expression): """ Extract partition keys (equality constraints between a field and a scalar) from an expression as a dict mapping the field's name to its value. @@ -3529,6 +3529,17 @@ def _get_partition_keys(Expression partition_expression): will be conjunctions of equality conditions and are accessible through this function. Other subexpressions will be ignored. + Parameters + ---------- + partition_expression : pyarrow.dataset.Expression + + Returns + ------- + dict + + Examples + -------- + For example, an expression of is converted to {'part': 'A', 'year': 2016} diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index de9469de445b..c4da9686ecf0 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -47,7 +47,8 @@ UnionDataset, UnionDatasetFactory, WrittenFile, - _get_partition_keys, + get_partition_keys, + get_partition_keys as _get_partition_keys, # keep for backwards compatibility _filesystemdataset_write, ) # keep Expression functionality exposed here for backwards compatibility diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index ebacd0235916..4f12bc893f67 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -714,15 +714,16 @@ def test_expression_arithmetic_operators(): def test_partition_keys(): a, b, c = [ds.field(f) == f for f in 'abc'] - assert ds._get_partition_keys(a) == {'a': 'a'} - assert ds._get_partition_keys(a & b & c) == {f: f for f in 'abc'} + assert ds.get_partition_keys(a) == {'a': 'a'} + assert ds.get_partition_keys(a) == ds._get_partition_keys(a) + assert ds.get_partition_keys(a & b & c) == {f: f for f in 'abc'} nope = ds.field('d') >= 3 - assert ds._get_partition_keys(nope) == {} - assert ds._get_partition_keys(a & nope) == {'a': 'a'} + assert ds.get_partition_keys(nope) == {} + assert ds.get_partition_keys(a & nope) == {'a': 'a'} null = ds.field('a').is_null() - assert ds._get_partition_keys(null) == {'a': None} + assert ds.get_partition_keys(null) == {'a': None} @pytest.mark.parquet