From 69ba5710829c584af3a82e19965daba4361c2f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 13 Jan 2025 12:37:44 +0100 Subject: [PATCH 1/9] GH-45237: [Python] Raise minimum supported cython to >=3 --- ci/conda_env_python.txt | 2 +- ci/docker/conda-python-cython2.dockerfile | 24 ---------------------- docker-compose.yml | 25 ----------------------- python/pyproject.toml | 2 +- python/requirements-build.txt | 2 +- python/requirements-wheel-build.txt | 2 +- python/setup.py | 4 ++-- 7 files changed, 6 insertions(+), 55 deletions(-) delete mode 100644 ci/docker/conda-python-cython2.dockerfile diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index bf915493de30..b467cc1ddd3b 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -18,7 +18,7 @@ # don't add pandas here, because it is not a mandatory test dependency boto3 # not a direct dependency of s3fs, but needed for our s3fs fixture cffi -cython>=0.29.31 +cython>=3 cloudpickle fsspec hypothesis diff --git a/ci/docker/conda-python-cython2.dockerfile b/ci/docker/conda-python-cython2.dockerfile deleted file mode 100644 index 859ad868b0c7..000000000000 --- a/ci/docker/conda-python-cython2.dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -ARG python=3.9 -FROM ${repo}:${arch}-conda-python-${python} - -RUN mamba install -q -y "cython<3" && \ - mamba clean --all diff --git a/docker-compose.yml b/docker-compose.yml index 43dd3511fcf1..a13072346120 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,7 +123,6 @@ x-hierarchy: - conda-python-pandas: - conda-python-docs - conda-python-cpython-debug - - conda-python-cython2 - conda-python-dask - conda-python-emscripten - conda-python-hdfs @@ -1499,30 +1498,6 @@ services: /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/integration_substrait.sh"] - conda-python-cython2: - # Usage: - # docker compose build conda - # docker compose build conda-cpp - # docker compose build conda-python - # docker compose build conda-python-cython2 - # docker compose run --rm conda-python-cython2 - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2 - build: - context: . - dockerfile: ci/docker/conda-python-cython2.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2 - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - shm_size: *shm-size - environment: - <<: [*common, *ccache] - PYTEST_ARGS: # inherit - volumes: *conda-volumes - command: *python-conda-command - conda-python-cpython-debug: # Usage: # docker compose build conda diff --git a/python/pyproject.toml b/python/pyproject.toml index 85bdbec0915e..be7c16726702 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -17,7 +17,7 @@ [build-system] requires = [ - "cython >= 0.29.31", + "cython >= 3", # Starting with NumPy 1.25, NumPy is (by default) as far back compatible # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION # define). For older Python versions (where NumPy 1.25 is not yet available) diff --git a/python/requirements-build.txt b/python/requirements-build.txt index c150c842a0cc..00b523ee5212 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -1,4 +1,4 @@ -cython>=0.29.31 +cython>=3 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' setuptools_scm>=8 diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index faa078d3d7fe..38e59ce45ab8 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,4 +1,4 @@ -cython>=0.29.31 +cython>=3 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0; python_version>='3.9' setuptools_scm diff --git a/python/setup.py b/python/setup.py index 60b9a696d978..bd658b91a53a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -48,9 +48,9 @@ ) -if Cython.__version__ < '0.29.31': +if Cython.__version__ < '3': raise Exception( - 'Please update your Cython version. Supported Cython >= 0.29.31') + 'Please update your Cython version. Supported Cython >= 3') setup_dir = os.path.abspath(os.path.dirname(__file__)) From 0158ad5216a5e16c2096f10f65761f07ccea8c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 13 Jan 2025 12:40:33 +0100 Subject: [PATCH 2/9] pytest-cython 0.3.1 has been released with the required fix. This pin should not be required anymore --- ci/conda_env_sphinx.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index 4665a32e24bb..751df9b2f3c0 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -30,9 +30,5 @@ sphinx-lint sphinxcontrib-jquery sphinxcontrib-mermaid sphinx==6.2 -# Requirement for doctest-cython -# Needs upper pin of 0.3.0, see: -# https://github.com/lgpage/pytest-cython/issues/67 -# With 0.3.* bug fix release, the pin can be removed -pytest-cython==0.2.2 +pytest-cython pandas From bbc2382cf83339f539279b9d9f30d4c18802a009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 13 Jan 2025 12:48:30 +0100 Subject: [PATCH 3/9] Remove unnecessary cython dependency to test merge_pr script --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index d59da447612a..9a7c0722fcef 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -124,7 +124,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools pytest requests setuptools-scm + pip install setuptools pytest requests setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 388818c40725172c048f088fa6f98ab1c14c03ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 13 Jan 2025 13:03:17 +0100 Subject: [PATCH 4/9] Re-add cython for dev.yml as is required on Ruby release and merge script tests --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 9a7c0722fcef..f9718cbf7bb1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -124,7 +124,7 @@ jobs: shell: bash run: | gem install test-unit - pip install setuptools pytest requests setuptools-scm + pip install "cython>=3" setuptools pytest requests setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c6bf8d454aded62baaf6463790b5c229169b9c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 13 Jan 2025 13:05:35 +0100 Subject: [PATCH 5/9] Remove cython 2 task. I forgot to save the file and commit --- dev/tasks/tasks.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 93450df155c9..cade2d51a0f9 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1116,14 +1116,6 @@ tasks: PYTHON: "3.10" image: conda-python-substrait - test-conda-python-3.10-cython2: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: "3.10" - image: conda-python-cython2 - test-ubuntu-22.04-python-313-freethreading: ci: github template: docker-tests/github.linux.yml From 2889c63a3a9c4c525a3884709a4ac1ef934f1d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 13 Jan 2025 14:17:10 +0100 Subject: [PATCH 6/9] Revert "pytest-cython 0.3.1 has been released with the required fix. This pin should not be required anymore" This reverts commit 0158ad5216a5e16c2096f10f65761f07ccea8c18. --- ci/conda_env_sphinx.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index 751df9b2f3c0..4665a32e24bb 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -30,5 +30,9 @@ sphinx-lint sphinxcontrib-jquery sphinxcontrib-mermaid sphinx==6.2 -pytest-cython +# Requirement for doctest-cython +# Needs upper pin of 0.3.0, see: +# https://github.com/lgpage/pytest-cython/issues/67 +# With 0.3.* bug fix release, the pin can be removed +pytest-cython==0.2.2 pandas From c44d53cfdd84c6b04ce24c95b8a1640763d31349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 14 Jan 2025 09:41:13 +0100 Subject: [PATCH 7/9] Remove flag due to unsafe use of type 'bool' in operation raised on Cython 0.29.1 --- python/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 80d1cd31ac23..12fba9067bc7 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -200,10 +200,6 @@ if(MSVC) # seem harmless, and probably not worth the effort of working around it string(APPEND CMAKE_CXX_FLAGS " /wd4800") - # See https://github.com/cython/cython/issues/2731. Change introduced in - # Cython 0.29.1 causes "unsafe use of type 'bool' in operation" - string(APPEND CMAKE_CXX_FLAGS " /wd4804") - # See https://github.com/cython/cython/issues/4445. # # Cython 3 emits "(void)__Pyx_PyObject_CallMethod0;" to suppress a From 59f308b2c511e5bb09136923c24945894452dc4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 14 Jan 2025 10:48:44 +0100 Subject: [PATCH 8/9] Remove unnecessary binding decorator --- python/pyarrow/_azurefs.pyx | 3 --- python/pyarrow/_dataset_parquet.pyx | 1 - python/pyarrow/_fs.pyx | 2 -- python/pyarrow/_gcsfs.pyx | 3 --- python/pyarrow/_hdfs.pyx | 3 --- python/pyarrow/_s3fs.pyx | 3 --- python/pyarrow/scalar.pxi | 2 -- 7 files changed, 17 deletions(-) diff --git a/python/pyarrow/_azurefs.pyx b/python/pyarrow/_azurefs.pyx index 5cd6300c18c6..60c326d280e2 100644 --- a/python/pyarrow/_azurefs.pyx +++ b/python/pyarrow/_azurefs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.lib import frombytes, tobytes from pyarrow.includes.libarrow_fs cimport * @@ -115,7 +113,6 @@ cdef class AzureFileSystem(FileSystem): self.azurefs = wrapped.get() @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 8fe9f30d33af..6d89e6084fd1 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -19,7 +19,6 @@ """Dataset support for Parquet file format.""" -from cython cimport binding from cython.operator cimport dereference as deref import os diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index e315dd6381f4..2b1f0e7eb318 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -18,7 +18,6 @@ # cython: language_level = 3 from cpython.datetime cimport datetime, PyDateTime_DateTime -from cython cimport binding from pyarrow.includes.common cimport * from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint @@ -422,7 +421,6 @@ cdef class FileSystem(_Weakrefable): "SubTreeFileSystem") @staticmethod - @binding(True) # Required for cython < 3 def _from_uri(uri): fs, _path = FileSystem.from_uri(uri) return fs diff --git a/python/pyarrow/_gcsfs.pyx b/python/pyarrow/_gcsfs.pyx index 5e69413cea95..a5a366fe9cc6 100644 --- a/python/pyarrow/_gcsfs.pyx +++ b/python/pyarrow/_gcsfs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.lib cimport (pyarrow_wrap_metadata, pyarrow_unwrap_metadata) from pyarrow.lib import frombytes, tobytes, ensure_metadata @@ -164,7 +162,6 @@ cdef class GcsFileSystem(FileSystem): return datetime.fromtimestamp(expiration_ns / 1.0e9, timezone.utc) @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_hdfs.pyx b/python/pyarrow/_hdfs.pyx index c426337a12ec..0a1a5483bda3 100644 --- a/python/pyarrow/_hdfs.pyx +++ b/python/pyarrow/_hdfs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_fs cimport * @@ -137,7 +135,6 @@ replication=1)`` return self @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index 038f9109d815..2a7cf6dfdd14 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.lib cimport (check_status, pyarrow_wrap_metadata, pyarrow_unwrap_metadata) from pyarrow.lib import frombytes, tobytes, KeyValueMetadata @@ -419,7 +417,6 @@ cdef class S3FileSystem(FileSystem): self.s3fs = wrapped.get() @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index 2235cd0b981a..e877b0965d13 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -16,7 +16,6 @@ # under the License. import collections -from cython cimport binding from uuid import UUID @@ -895,7 +894,6 @@ cdef class DictionaryScalar(Scalar): """ @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(type, is_valid, index, dictionary): cdef: CDictionaryScalarIndexAndDictionary value From 87bd74fa48fba23b00024e11df0d6c4bd3e7b145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 14 Jan 2025 10:54:59 +0100 Subject: [PATCH 9/9] Remove stray binding --- python/pyarrow/_dataset_parquet.pyx | 1 - python/pyarrow/io.pxi | 1 - 2 files changed, 2 deletions(-) diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 6d89e6084fd1..863c92859193 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -889,7 +889,6 @@ cdef class ParquetFragmentScanOptions(FragmentScanOptions): return attrs == other_attrs @staticmethod - @binding(True) # Required for Cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index b3de15067fbf..89015dadba5a 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -2370,7 +2370,6 @@ cdef class CacheOptions(_Weakrefable): ideal_bandwidth_utilization_frac, max_ideal_request_size_mib)) @staticmethod - @binding(True) # Required for Cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper.