From 4be495b92035702015a8309d38809572301b2670 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 5 Feb 2025 18:10:10 -0800 Subject: [PATCH 01/10] Create python-wheel-windows-vs2022-base.dockerfile --- ...ython-wheel-windows-vs2022-base.dockerfile | 118 ++++++++++++++++++ ci/scripts/install_vcpkg.bat | 79 ++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 ci/docker/python-wheel-windows-vs2022-base.dockerfile create mode 100644 ci/scripts/install_vcpkg.bat diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile new file mode 100644 index 000000000000..1957ca326b9a --- /dev/null +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -0,0 +1,118 @@ +# escape=` + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# WIP: GH-45156: [Python][Packaging] Revisit Windows wheel Docker setup +# +# Reconfigure for building +# +# docker run --storage-opt "size=50GB" mcr.microsoft.com/windows/servercore:ltsc2022 cmd +# +# See https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2022#troubleshoot-build-tools-containers +# +# Build (using -m is important) +# +# docker build -t buildtools -m 4GB --file .\ci\docker\python-wheel-windows-vs2022-base.dockerfile . +# +# RESOURCES +# +# - https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2022 +# - https://learn.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2022 +# +# TODOS +# +# - I don't think we need to disable BUILDKIT anymore in other places so look +# at whether that's true and remove any references + +# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env +# when you update this file. + +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +# Ensure we in a command shell and not Powershell +SHELL ["cmd", "/S", "/C"] + +# Install MSVC BuildTools +# +# The set of components below (lines starting with --add) is the most minimal +# set we could find that would still compile Arrow C++. +RUN ` + curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe ` + && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache ` + --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" ` + --add Microsoft.VisualStudio.Component.VC.CoreBuildTools ` + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + --add Microsoft.VisualStudio.Component.Windows10SDK.20348 ` + --add Microsoft.VisualStudio.Component.VC.CMake.Project ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0) ` + && del /q vs_buildtools.exe + +# Install choco CLI +# +# We switch into Powershell just for this command and switch back to cmd +# See https://chocolatey.org/install#completely-offline-install +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] +RUN ` + Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) +SHELL ["cmd", "/S", "/C"] + +# Install CMake and other tools +ARG cmake=3.31.2 +RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs 'ADD_CMAKE_TO_PATH=System' +RUN choco install --no-progress -r -y git gzip ninja wget + +# Install vcpkg +# +# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has +# started to ship precompiled binaries for the vcpkg-tool. +ARG vcpkg +COPY ci/vcpkg/*.patch ` + ci/vcpkg/*windows*.cmake ` + arrow/ci/vcpkg/ +COPY ci/scripts/install_vcpkg.bat arrow/ci/scripts/ +ENV VCPKG_ROOT=C:\\vcpkg +RUN cmd /c arrow\ci\scripts\install_vcpkg.bat C:\\vcpkg %vcpkg% +RUN setx PATH "%PATH%;%VCPKG_ROOT%" + +# Configure vcpkg and install dependencies +# NOTE: use windows batch environment notation for build arguments in RUN +# statements but bash notation in ENV statements +# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system +# cmake's and ninja's versions are recent enough +ARG build_type=release +ENV CMAKE_BUILD_TYPE=${build_type} ` + VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg ` + VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` + VCPKG_FEATURE_FLAGS="manifests" +COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ +# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains +# ssl related fixes as well as we can patch the vcpkg portfile to support +# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186 +# but we cannot patch those portfiles since vcpkg-tool handles the checkout of +# previous versions => use bundled S3 build +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=flight` + --x-feature=gcs` + --x-feature=json` + --x-feature=orc` + --x-feature=parquet` + --x-feature=s3 diff --git a/ci/scripts/install_vcpkg.bat b/ci/scripts/install_vcpkg.bat new file mode 100644 index 000000000000..2b0eafb570a7 --- /dev/null +++ b/ci/scripts/install_vcpkg.bat @@ -0,0 +1,79 @@ +@rem # Licensed to the Apache Software Foundation (ASF) under one +@rem # or more contributor license agreements. See the NOTICE file +@rem # distributed with this work for additional information +@rem # regarding copyright ownership. The ASF licenses this file +@rem # to you under the Apache License, Version 2.0 (the +@rem # "License"); you may not use this file except in compliance +@rem # with the License. You may obtain a copy of the License at +@rem # +@rem # http://www.apache.org/licenses/LICENSE-2.0 +@rem # +@rem # Unless required by applicable law or agreed to in writing, +@rem # software distributed under the License is distributed on an +@rem # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem # KIND, either express or implied. See the License for the +@rem # specific language governing permissions and limitations +@rem # under the License. + +@echo off + +SET orig_dir=%CD% +@REM TODO: In PR review, decide whether this needs to be fancier +SET arrow_dir=arrow +SET default_vcpkg_ports_patch=%arrow_dir%/ci/vcpkg/ports.patch +SET vcpkg_destination=%1 +SET env_file=.env + +set USAGE_STRING=Usage: .\%~n0.bat target-directory [vcpkg-version [vcpkg-ports-patch]] +if "%1"=="" ( + echo %USAGE_STRING% + exit \b 1 +) + +@rem Extract value of VCPKG from .env file if it exists +set VCPKG= +if exist .env ( + @REM Iterate through each line, split on '=' into %%i and %%j halves + for /f "usebackq tokens=1,* delims==" %%i in (%env_file%) do ( + @REM %%i is storing the first token on each line + if "%%i"=="VCPKG" ( + @REM Iterate over the second half of line (%%j), first token from each + for /f "tokens=1 delims= " %%A in ("%%j") do ( + @REM ~ removes surrounding quotes from %%A + set VCPKG=%%~A + ) + ) + ) +) + +@REM Handle vcpkg_version argument or fall back to environment variable +@REM +@REM The "else if" here works around behavior I couldn't find a workaround for +@REM with how Windows Docker handles the empty `ARG vcpkg` in associated +@REM Dockerfiles. +if "%2"=="" ( + SET vcpkg_version=%VCPKG% +) else if "%2"=="%%vcpkg%%" ( + SET vcpkg_version=%VCPKG% +) else ( + SET vcpkg_version=%2 +) + +@REM Handle vcpkg_ports_patch path +if "%3"=="" ( + SET vcpkg_ports_patch=%default_vcpkg_ports_patch% +) else ( + SET vcpkg_ports_patch=%3 +) + +git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg %vcpkg_destination% +cd %vcpkg_destination% +if not "%vcpkg_version%"=="" ( + git checkout %vcpkg_version% +) +./bootstrap-vcpkg.bat -disableMetrics +if exist "%default_vcpkg_ports_patch%" ( + git apply --verbose --ignore-whitespace ${vcpkg_ports_patch} +) + +cd %orig_dir% From ed312418f2e907d62f8794a08a07e7ff20cf0362 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Fri, 7 Feb 2025 18:03:05 -0800 Subject: [PATCH 02/10] Update all windows docker image to use new base --- .env | 4 +- ...aded-wheel-windows-test-vs2022.dockerfile} | 0 ...-threaded-wheel-windows-vs2022.dockerfile} | 0 ...-wheel-windows-test-vs2019-base.dockerfile | 51 ------------ ...-wheel-windows-test-vs2022-base.dockerfile | 67 ++++++++++++++++ ...thon-wheel-windows-test-vs2022.dockerfile} | 0 ...ython-wheel-windows-vs2019-base.dockerfile | 79 ------------------- ...=> python-wheel-windows-vs2022.dockerfile} | 0 dev/tasks/python-wheels/github.windows.yml | 10 +-- docker-compose.yml | 68 ++++++++-------- 10 files changed, 108 insertions(+), 171 deletions(-) rename ci/docker/{python-free-threaded-wheel-windows-test-vs2019.dockerfile => python-free-threaded-wheel-windows-test-vs2022.dockerfile} (100%) rename ci/docker/{python-free-threaded-wheel-windows-vs2019.dockerfile => python-free-threaded-wheel-windows-vs2022.dockerfile} (100%) delete mode 100644 ci/docker/python-wheel-windows-test-vs2019-base.dockerfile create mode 100644 ci/docker/python-wheel-windows-test-vs2022-base.dockerfile rename ci/docker/{python-wheel-windows-test-vs2019.dockerfile => python-wheel-windows-test-vs2022.dockerfile} (100%) delete mode 100644 ci/docker/python-wheel-windows-vs2019-base.dockerfile rename ci/docker/{python-wheel-windows-vs2019.dockerfile => python-wheel-windows-vs2022.dockerfile} (100%) diff --git a/.env b/.env index a94803cc3277..7a78e2dc483c 100644 --- a/.env +++ b/.env @@ -95,8 +95,8 @@ VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release # ci/docker/python-*-windows-*.dockerfile or the vcpkg config. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-02-03 -PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-01-27 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=ltsc2022-KB5049983 +PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=ltsc2022-KB5049983 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan". # See https://github.com/conan-io/conan-docker-tools#readme and diff --git a/ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile similarity index 100% rename from ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile rename to ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile diff --git a/ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile b/ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile similarity index 100% rename from ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile rename to ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile diff --git a/ci/docker/python-wheel-windows-test-vs2019-base.dockerfile b/ci/docker/python-wheel-windows-test-vs2019-base.dockerfile deleted file mode 100644 index 73a78da30b90..000000000000 --- a/ci/docker/python-wheel-windows-test-vs2019-base.dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env -# when you update this file. - -# based on mcr.microsoft.com/windows/servercore:ltsc2019 -# contains choco and vs2019 preinstalled -FROM abrarov/msvc-2019:2.11.0 - -# hadolint shell=cmd.exe - -# Add unix tools to path -RUN setx path "%path%;C:\Program Files\Git\usr\bin" - -# 1. Remove previous installations of Python from the base image -# NOTE: a more recent base image (tried with 2.12.1) comes with Python 3.9.7 -# and the MSI installers are failing to remove pip and tcl/tk "products" making -# the subsequent choco python installation step failing for installing Python -# version 3.9.* due to existing python version -# 2. Install Minio for S3 testing. -RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ - rm -rf Python* && \ - curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z \ - --output "C:\Windows\Minio.exe" - -# Install archiver to extract xz archives (for timezone database). -# Install the GCS testbench using a well-known Python version. -# NOTE: cannot use pipx's `--fetch-missing-python` because of -# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. -RUN choco install --no-progress -r -y archiver && \ - choco install -r -y --pre --no-progress python --version=3.11.9 -ENV PIPX_BIN_DIR=C:\\Windows\\ -ENV PIPX_PYTHON="C:\Python311\python.exe" -COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ -RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \ - storage-testbench -h diff --git a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile new file mode 100644 index 000000000000..59df5d7d3137 --- /dev/null +++ b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile @@ -0,0 +1,67 @@ +# escape=` + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env +# when you update this file. + +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +# Ensure we in a command shell and not Powershell +SHELL ["cmd", "/S", "/C"] + +# Install MSVC BuildTools +# +# The set of components below (lines starting with --add) is the most minimal +# set we could find that would still compile Arrow C++. +RUN ` + curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe ` + && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache ` + --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" ` + --add Microsoft.VisualStudio.Component.VC.CoreBuildTools ` + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + --add Microsoft.VisualStudio.Component.Windows10SDK.20348 ` + --add Microsoft.VisualStudio.Component.VC.CMake.Project ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0) ` + && del /q vs_buildtools.exe + +# Install choco CLI +# +# We switch into Powershell just for this command and switch back to cmd +# See https://chocolatey.org/install#completely-offline-install +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] +RUN ` + Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) +SHELL ["cmd", "/S", "/C"] + +# Install git, wget, minio +RUN choco install --no-progress -r -y git wget +RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z ` + --output "C:\Windows\Minio.exe" + +# Install archiver to extract xz archives (for timezone database). +# Install the GCS testbench using a well-known Python version. +# NOTE: cannot use pipx's `--fetch-missing-python` because of +# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. +RUN choco install --no-progress -r -y archiver && ` + choco install -r -y --pre --no-progress python --version=3.11.9 +ENV PIPX_BIN_DIR=C:\\Windows\\ +ENV PIPX_PYTHON="C:\Python311\python.exe" +COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ +RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` + storage-testbench -h diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2022.dockerfile similarity index 100% rename from ci/docker/python-wheel-windows-test-vs2019.dockerfile rename to ci/docker/python-wheel-windows-test-vs2022.dockerfile diff --git a/ci/docker/python-wheel-windows-vs2019-base.dockerfile b/ci/docker/python-wheel-windows-vs2019-base.dockerfile deleted file mode 100644 index bd91f01bf9b6..000000000000 --- a/ci/docker/python-wheel-windows-vs2019-base.dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env -# when you update this file. - -# based on mcr.microsoft.com/windows/servercore:ltsc2019 -# contains choco and vs2019 preinstalled -FROM abrarov/msvc-2019:2.11.0 - -# Install CMake and Ninja -ARG cmake=3.31.2 -RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs 'ADD_CMAKE_TO_PATH=System' && \ - choco install --no-progress -r -y gzip wget ninja - -# Add unix tools to path -RUN setx path "%path%;C:\Program Files\Git\usr\bin" - -# Install vcpkg -# -# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has -# started to ship precompiled binaries for the vcpkg-tool. -ARG vcpkg -COPY ci/vcpkg/*.patch \ - ci/vcpkg/*windows*.cmake \ - arrow/ci/vcpkg/ -COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/ -ENV VCPKG_ROOT=C:\\vcpkg -RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \ - setx PATH "%PATH%;%VCPKG_ROOT%" - -# Configure vcpkg and install dependencies -# NOTE: use windows batch environment notation for build arguments in RUN -# statements but bash notation in ENV statements -# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system -# cmake's and ninja's versions are recent enough -ARG build_type=release -ENV CMAKE_BUILD_TYPE=${build_type} \ - VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \ - VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \ - VCPKG_FEATURE_FLAGS="manifests" -COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ -# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains -# ssl related fixes as well as we can patch the vcpkg portfile to support -# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186 -# but we cannot patch those portfiles since vcpkg-tool handles the checkout of -# previous versions => use bundled S3 build -RUN vcpkg install \ - --clean-after-build \ - --x-install-root=%VCPKG_ROOT%\installed \ - --x-manifest-root=arrow/ci/vcpkg \ - --x-feature=flight \ - --x-feature=gcs \ - --x-feature=json \ - --x-feature=orc \ - --x-feature=parquet \ - --x-feature=s3 - -# Remove previous installations of Python from the base image -# NOTE: a more recent base image (tried with 2.12.1) comes with Python 3.9.7 -# and the MSI installers are failing to remove pip and tcl/tk "products" making -# the subsequent choco python installation step failing for installing Python -# version 3.9.* due to existing Python version -RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ - rm -rf Python* diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile b/ci/docker/python-wheel-windows-vs2022.dockerfile similarity index 100% rename from ci/docker/python-wheel-windows-vs2019.dockerfile rename to ci/docker/python-wheel-windows-vs2022.dockerfile diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 2ac942d5bdb4..02f28a1edb0f 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -22,7 +22,7 @@ jobs: build: name: "Build wheel for Windows" - runs-on: windows-2019 + runs-on: windows-2022 env: # archery uses this environment variable PYTHON: "{{ python_version }}" @@ -30,7 +30,7 @@ jobs: # this is a private repository at the moment (mostly because of licensing # consideration of windows images with visual studio), but anyone can # recreate the image by manually building it via: - # `archery build python-wheel-windows-vs2019` + # `archery build python-wheel-windows-vs2022` # note that we don't run docker build since there wouldn't be a cache hit # and rebuilding the dependencies takes a fair amount of time REPO: ghcr.io/ursacomputing/arrow @@ -62,15 +62,15 @@ jobs: run: | cd arrow @rem We want to use only - @rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 + @rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2022 @rem but it doesn't use pulled caches. @rem It always build an image from scratch. @rem We can remove this workaround once we find a way to use @rem pulled caches when build an image. echo on - archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 + archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 if errorlevel 1 ( - archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 || exit /B 1 + archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 ) archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 diff --git a/docker-compose.yml b/docker-compose.yml index bf8093ec6030..0285de2df322 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -175,9 +175,9 @@ x-hierarchy: - python-free-threaded-wheel-manylinux-test-imports - python-wheel-manylinux-test-unittests - python-free-threaded-wheel-manylinux-test-unittests - - python-wheel-windows-vs2019-base: - - python-wheel-windows-vs2019 - - python-free-threaded-wheel-windows-vs2019 + - python-wheel-windows-vs2022-base: + - python-wheel-windows-vs2022 + - python-free-threaded-wheel-windows-vs2022 - python-wheel-windows-test-base: - python-wheel-windows-test - python-free-threaded-wheel-windows-test @@ -1266,81 +1266,81 @@ services: CHECK_UNITTESTS: "ON" command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow - python-wheel-windows-vs2019-base: - image: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + python-wheel-windows-vs2022-base: + image: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} build: args: vcpkg: ${VCPKG} context: . - dockerfile: ci/docker/python-wheel-windows-vs2019-base.dockerfile + dockerfile: ci/docker/python-wheel-windows-vs2022-base.dockerfile # This should make the pushed images reusable, but the image gets rebuilt. # Uncomment if no local cache is available. # cache_from: - # - abrarov/msvc-2019:2.11.0 - # - ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} - volumes: &python-wheel-windows-vs2019-volumes + # - mcr.microsoft.com/windows/servercore:ltsc2022 + # - ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + volumes: &python-wheel-windows-vs2022-volumes - type: bind source: . target: "C:/arrow" - python-wheel-windows-vs2019: - image: ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + python-wheel-windows-vs2022: + image: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} build: args: - base: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} context: . - dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile + dockerfile: ci/docker/python-wheel-windows-vs2022.dockerfile # This should make the pushed images reusable, but the image gets rebuilt. # Uncomment if no local cache is available. # cache_from: - # - abrarov/msvc-2019:2.11.0 - # - ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} - volumes: *python-wheel-windows-vs2019-volumes + # - mcr.microsoft.com/windows/servercore:ltsc2022 + # - ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + volumes: *python-wheel-windows-vs2022-volumes command: arrow\\ci\\scripts\\python_wheel_windows_build.bat - python-free-threaded-wheel-windows-vs2019: - image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + python-free-threaded-wheel-windows-vs2022: + image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} build: args: - base: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} context: . - dockerfile: ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile + dockerfile: ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile # This should make the pushed images reusable, but the image gets rebuilt. # Uncomment if no local cache is available. # cache_from: - # - abrarov/msvc-2019:2.11.0 - # - ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} - volumes: *python-wheel-windows-vs2019-volumes + # - mcr.microsoft.com/windows/servercore:ltsc2022 + # - ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + volumes: *python-wheel-windows-vs2022-volumes command: arrow\\ci\\scripts\\python_wheel_windows_build.bat python-wheel-windows-test-base: - image: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + image: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: context: . - dockerfile: ci/docker/python-wheel-windows-test-vs2019-base.dockerfile - volumes: *python-wheel-windows-vs2019-volumes + dockerfile: ci/docker/python-wheel-windows-test-vs2022-base.dockerfile + volumes: *python-wheel-windows-vs2022-volumes python-wheel-windows-test: - image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} python: ${PYTHON} context: . - dockerfile: ci/docker/python-wheel-windows-test-vs2019.dockerfile - volumes: *python-wheel-windows-vs2019-volumes + dockerfile: ci/docker/python-wheel-windows-test-vs2022.dockerfile + volumes: *python-wheel-windows-vs2022-volumes command: arrow\\ci\\scripts\\python_wheel_windows_test.bat python-free-threaded-wheel-windows-test: - image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} context: . - dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile - volumes: *python-wheel-windows-vs2019-volumes + dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile + volumes: *python-wheel-windows-vs2022-volumes command: arrow\\ci\\scripts\\python_wheel_windows_test.bat ############################## Integration ################################# From 1ce8c8f8da95b397715e83f8c25044d03a99521b Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Sat, 22 Feb 2025 13:02:11 -0800 Subject: [PATCH 03/10] Switch vcpkg .bat for existing .sh script --- ...ython-wheel-windows-vs2022-base.dockerfile | 9 ++- ci/scripts/install_vcpkg.bat | 79 ------------------- 2 files changed, 6 insertions(+), 82 deletions(-) delete mode 100644 ci/scripts/install_vcpkg.bat diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 1957ca326b9a..ce48557c1a11 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -77,6 +77,9 @@ ARG cmake=3.31.2 RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs 'ADD_CMAKE_TO_PATH=System' RUN choco install --no-progress -r -y git gzip ninja wget +# Add unix tools to path +RUN setx path "%path%;C:\Program Files\Git\usr\bin" + # Install vcpkg # # Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has @@ -85,10 +88,10 @@ ARG vcpkg COPY ci/vcpkg/*.patch ` ci/vcpkg/*windows*.cmake ` arrow/ci/vcpkg/ -COPY ci/scripts/install_vcpkg.bat arrow/ci/scripts/ +COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/ ENV VCPKG_ROOT=C:\\vcpkg -RUN cmd /c arrow\ci\scripts\install_vcpkg.bat C:\\vcpkg %vcpkg% -RUN setx PATH "%PATH%;%VCPKG_ROOT%" +RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` + setx PATH "%PATH%;%VCPKG_ROOT%" # Configure vcpkg and install dependencies # NOTE: use windows batch environment notation for build arguments in RUN diff --git a/ci/scripts/install_vcpkg.bat b/ci/scripts/install_vcpkg.bat deleted file mode 100644 index 2b0eafb570a7..000000000000 --- a/ci/scripts/install_vcpkg.bat +++ /dev/null @@ -1,79 +0,0 @@ -@rem # Licensed to the Apache Software Foundation (ASF) under one -@rem # or more contributor license agreements. See the NOTICE file -@rem # distributed with this work for additional information -@rem # regarding copyright ownership. The ASF licenses this file -@rem # to you under the Apache License, Version 2.0 (the -@rem # "License"); you may not use this file except in compliance -@rem # with the License. You may obtain a copy of the License at -@rem # -@rem # http://www.apache.org/licenses/LICENSE-2.0 -@rem # -@rem # Unless required by applicable law or agreed to in writing, -@rem # software distributed under the License is distributed on an -@rem # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem # KIND, either express or implied. See the License for the -@rem # specific language governing permissions and limitations -@rem # under the License. - -@echo off - -SET orig_dir=%CD% -@REM TODO: In PR review, decide whether this needs to be fancier -SET arrow_dir=arrow -SET default_vcpkg_ports_patch=%arrow_dir%/ci/vcpkg/ports.patch -SET vcpkg_destination=%1 -SET env_file=.env - -set USAGE_STRING=Usage: .\%~n0.bat target-directory [vcpkg-version [vcpkg-ports-patch]] -if "%1"=="" ( - echo %USAGE_STRING% - exit \b 1 -) - -@rem Extract value of VCPKG from .env file if it exists -set VCPKG= -if exist .env ( - @REM Iterate through each line, split on '=' into %%i and %%j halves - for /f "usebackq tokens=1,* delims==" %%i in (%env_file%) do ( - @REM %%i is storing the first token on each line - if "%%i"=="VCPKG" ( - @REM Iterate over the second half of line (%%j), first token from each - for /f "tokens=1 delims= " %%A in ("%%j") do ( - @REM ~ removes surrounding quotes from %%A - set VCPKG=%%~A - ) - ) - ) -) - -@REM Handle vcpkg_version argument or fall back to environment variable -@REM -@REM The "else if" here works around behavior I couldn't find a workaround for -@REM with how Windows Docker handles the empty `ARG vcpkg` in associated -@REM Dockerfiles. -if "%2"=="" ( - SET vcpkg_version=%VCPKG% -) else if "%2"=="%%vcpkg%%" ( - SET vcpkg_version=%VCPKG% -) else ( - SET vcpkg_version=%2 -) - -@REM Handle vcpkg_ports_patch path -if "%3"=="" ( - SET vcpkg_ports_patch=%default_vcpkg_ports_patch% -) else ( - SET vcpkg_ports_patch=%3 -) - -git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg %vcpkg_destination% -cd %vcpkg_destination% -if not "%vcpkg_version%"=="" ( - git checkout %vcpkg_version% -) -./bootstrap-vcpkg.bat -disableMetrics -if exist "%default_vcpkg_ports_patch%" ( - git apply --verbose --ignore-whitespace ${vcpkg_ports_patch} -) - -cd %orig_dir% From f7dc625a41746bc75af94783e0c5357f78bbb549 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Sat, 22 Feb 2025 16:30:36 -0800 Subject: [PATCH 04/10] Clean up vs2022 dockerfile a bit --- ...ython-wheel-windows-vs2022-base.dockerfile | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index ce48557c1a11..7f683487a8c0 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -18,27 +18,36 @@ # under the License. -# WIP: GH-45156: [Python][Packaging] Revisit Windows wheel Docker setup +# NOTE: To build this Dockerfile, you probably need to do the following two +# things: # -# Reconfigure for building +# 1. Increase your container image size to a higher value. # -# docker run --storage-opt "size=50GB" mcr.microsoft.com/windows/servercore:ltsc2022 cmd +# e.g., # -# See https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2022#troubleshoot-build-tools-containers +# Set a custom 'storage-opts' value in your Windows Docker config and restart +# Docker: # -# Build (using -m is important) +# "storage-opts": [ +# "size=50GB" +# ] # -# docker build -t buildtools -m 4GB --file .\ci\docker\python-wheel-windows-vs2022-base.dockerfile . +# See # -# RESOURCES +# https://learn.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#example # -# - https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2022 -# - https://learn.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2022 +# for details on this step and # -# TODOS +# https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2022#troubleshoot-build-tools-containers # -# - I don't think we need to disable BUILDKIT anymore in other places so look -# at whether that's true and remove any references +# for more information. +# +# 2. Increase the memory limit for the build container to at least 4GB. +# +# e.g., +# +# docker build -t sometag -m 4GB --file ` +# .\ci\docker\python-wheel-windows-vs2022-base.dockerfile . # NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env # when you update this file. @@ -65,7 +74,9 @@ RUN ` # Install choco CLI # -# We switch into Powershell just for this command and switch back to cmd +# Switch into Powershell just for this command because choco only provides a +# Powershell installation script. After, we switch back to cmd. +# # See https://chocolatey.org/install#completely-offline-install SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] RUN ` @@ -77,7 +88,7 @@ ARG cmake=3.31.2 RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs 'ADD_CMAKE_TO_PATH=System' RUN choco install --no-progress -r -y git gzip ninja wget -# Add unix tools to path +# Add UNIX tools to PATH RUN setx path "%path%;C:\Program Files\Git\usr\bin" # Install vcpkg From ffa8d79a4ce12170f8c7fc219a40905fe05216e5 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Sat, 22 Feb 2025 16:34:32 -0800 Subject: [PATCH 05/10] Remove remaining references to vs2019 --- .pre-commit-config.yaml | 9 ++------- dev/tasks/python-wheels/github.windows.yml | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e386b87be6e9..8e63648e8dbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: files: >- ( ?^ci/docker/conda-python-emscripten\.dockerfile$| - ?^ci/docker/python-.*-wheel-windows-test-vs2019.*\.dockerfile$| + ?^ci/docker/python-.*-wheel-windows-test-vs2022.*\.dockerfile$| ) types: [] - repo: https://github.com/pycqa/flake8 @@ -179,12 +179,7 @@ repos: - id: sphinx-lint files: ^docs/source exclude: ^docs/source/python/generated - args: [ - '--enable', - 'all', - '--disable', - 'dangling-hyphen,line-too-long', - ] + args: ["--enable", "all", "--disable", "dangling-hyphen,line-too-long"] - repo: https://github.com/koalaman/shellcheck-precommit rev: v0.10.0 hooks: diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 02f28a1edb0f..063ec1d632cc 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -72,7 +72,7 @@ jobs: if errorlevel 1 ( archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 ) - archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 + archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 - uses: actions/upload-artifact@v4 with: @@ -98,6 +98,6 @@ jobs: shell: cmd run: | cd arrow - archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 + archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-test {% endif %} From bb517c4d518787945ee5c0328b2eeeb32f7cd9da Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Sat, 22 Feb 2025 19:40:48 -0800 Subject: [PATCH 06/10] Fix path in python wheel build script --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 833d31059c71..8d9a897dba5e 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -24,7 +24,7 @@ py -0p %PYTHON_CMD% -m sysconfig || exit /B 1 -call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat" +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" @echo on echo "=== (%PYTHON%) Clear output directories and leftovers ===" From b42001490c2075374d0017adbcf1efb28b77aba7 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Sun, 23 Feb 2025 09:52:56 -0800 Subject: [PATCH 07/10] Change windows wheel cmake generator --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 8d9a897dba5e..fd6d0591661c 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -56,7 +56,7 @@ set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON set CMAKE_UNITY_BUILD=ON -set CMAKE_GENERATOR=Visual Studio 16 2019 +set CMAKE_GENERATOR=Visual Studio 17 2022 set CMAKE_PLATFORM=x64 set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests From d5918102f9f85b5ef8eaf789b15937063b07ba92 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Sun, 23 Feb 2025 12:30:03 -0800 Subject: [PATCH 08/10] Revert reformat in .pre-commit-comit.yaml --- .pre-commit-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8e63648e8dbc..d877429e47e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -179,7 +179,12 @@ repos: - id: sphinx-lint files: ^docs/source exclude: ^docs/source/python/generated - args: ["--enable", "all", "--disable", "dangling-hyphen,line-too-long"] + args: [ + '--enable', + 'all', + '--disable', + 'dangling-hyphen,line-too-long', + ] - repo: https://github.com/koalaman/shellcheck-precommit rev: v0.10.0 hooks: From 53ea036af559e57a8f8eaadfbcb5ec3687545f29 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Mon, 24 Feb 2025 11:04:29 -0800 Subject: [PATCH 09/10] Update .env --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 7a78e2dc483c..1877bba9f167 100644 --- a/.env +++ b/.env @@ -95,8 +95,8 @@ VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release # ci/docker/python-*-windows-*.dockerfile or the vcpkg config. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=ltsc2022-KB5049983 -PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=ltsc2022-KB5049983 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-02-25 +PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-02-25 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan". # See https://github.com/conan-io/conan-docker-tools#readme and From 9651440ca1a276662a07cb29bdfa29dde086fd59 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Mon, 24 Feb 2025 13:41:40 -0800 Subject: [PATCH 10/10] Update ci/docker/python-wheel-windows-test-vs2022-base.dockerfile Co-authored-by: Sutou Kouhei --- ci/docker/python-wheel-windows-test-vs2022-base.dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile index 59df5d7d3137..1d1602c03a23 100644 --- a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile @@ -54,12 +54,10 @@ RUN choco install --no-progress -r -y git wget RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z ` --output "C:\Windows\Minio.exe" -# Install archiver to extract xz archives (for timezone database). # Install the GCS testbench using a well-known Python version. # NOTE: cannot use pipx's `--fetch-missing-python` because of # https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. -RUN choco install --no-progress -r -y archiver && ` - choco install -r -y --pre --no-progress python --version=3.11.9 +RUN choco install -r -y --pre --no-progress python --version=3.11.9 ENV PIPX_BIN_DIR=C:\\Windows\\ ENV PIPX_PYTHON="C:\Python311\python.exe" COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/