From f7ff1c4e33f04ed48c9b7a85c5a6469c3d185797 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Tue, 20 Oct 2026 15:58:32 +0800 Subject: [PATCH] Add Arrow and Parquet C++ to the Rocky build images contrib/datalake_fdw builds against Arrow and Parquet C++. Install them in the Rocky 8, 9 and 10 build images from the Arrow project's own repository, pinned so the version moves only with a change here: 22.0.0 on Rocky 8, the last release whose headers are C++17 (from 23 on they need C++20, which gcc 8 does not have), and 26.0.0 on Rocky 9 and 10. The repository is left disabled afterwards, as EPEL is. The release package goes in with every repository disabled: it needs nothing from them, and loading their metadata under QEMU took eight minutes of an arm64 build. Testinfra checks that pkg-config reports the pinned version of both. The arm64 half of each build runs under QEMU, and the Rocky builds already took 35 to 60 minutes before Arrow joined them, so the job timeout goes from 60 to 90 minutes. The Ubuntu images are left alone: Apache's libarrow-dev pulls in libcurl4-openssl-dev, which replaces the libcurl4-gnutls-dev the deb packaging depends on. --- .../docker-cbdb-build-containers.yml | 6 +++-- devops/deploy/docker/build/rocky10/Dockerfile | 20 +++++++++++++++++ .../tests/testinfra/test_cloudberry_db_env.py | 10 +++++++++ devops/deploy/docker/build/rocky8/Dockerfile | 22 +++++++++++++++++++ .../tests/testinfra/test_cloudberry_db_env.py | 10 +++++++++ devops/deploy/docker/build/rocky9/Dockerfile | 20 +++++++++++++++++ .../tests/testinfra/test_cloudberry_db_env.py | 10 +++++++++ 7 files changed, 96 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 9601fdb83cc..b14c4c45303 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -76,8 +76,10 @@ concurrency: jobs: build-and-push: - # Set timeout to prevent hanging builds - timeout-minutes: 60 + # Set timeout to prevent hanging builds. The arm64 half runs under + # QEMU, and the Rocky builds already took 35 to 60 minutes before + # Arrow joined them, so 60 left no room. + timeout-minutes: 90 runs-on: ubuntu-latest # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 diff --git a/devops/deploy/docker/build/rocky10/Dockerfile b/devops/deploy/docker/build/rocky10/Dockerfile index 3ab40229020..bb86df1f980 100644 --- a/devops/deploy/docker/build/rocky10/Dockerfile +++ b/devops/deploy/docker/build/rocky10/Dockerfile @@ -170,6 +170,26 @@ RUN dnf makecache && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +# -------------------------------------------------------------------- +# Install Apache Arrow and Parquet C++ for contrib/datalake_fdw +# -------------------------------------------------------------------- +# From the Arrow project's own repository, pinned: datalake_fdw shares +# its ABI with Arrow, so the version moves only with a change here. +# EPEL and CRB carry what the packages depend on (re2, thrift, ...). +# The repository is left disabled afterwards, as EPEL is. +# The release package needs nothing from a repository, so it goes in +# with them all disabled: loading their metadata under QEMU took eight +# minutes of an arm64 build. +# -------------------------------------------------------------------- +ARG ARROW_VERSION=26.0.0 +RUN dnf install -y --disablerepo='*' \ + https://packages.apache.org/artifactory/arrow/almalinux/10/apache-arrow-release-latest.rpm && \ + dnf install -y --enablerepo=epel --enablerepo=crb \ + arrow-devel-${ARROW_VERSION}-1.el10 \ + parquet-devel-${ARROW_VERSION}-1.el10 && \ + dnf config-manager --disable 'apache-arrow-*' && \ + dnf clean all + # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py index 445318f5335..ac528027fae 100644 --- a/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py +++ b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py @@ -125,3 +125,13 @@ def test_locale_generated(host): locale = host.run("locale -a | grep en_US.utf8") assert locale.exit_status == 0 assert "en_US.utf8" in locale.stdout + + +def test_arrow_for_datalake_fdw(host): + """ + Test if Arrow and Parquet C++ are installed at the pinned version. + """ + for module in ("arrow", "parquet"): + version = host.run(f"pkg-config --modversion {module}") + assert version.exit_status == 0 + assert version.stdout.strip() == "26.0.0" diff --git a/devops/deploy/docker/build/rocky8/Dockerfile b/devops/deploy/docker/build/rocky8/Dockerfile index eed55e4b171..bda83e27e99 100644 --- a/devops/deploy/docker/build/rocky8/Dockerfile +++ b/devops/deploy/docker/build/rocky8/Dockerfile @@ -171,6 +171,28 @@ RUN dnf makecache && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +# -------------------------------------------------------------------- +# Install Apache Arrow and Parquet C++ for contrib/datalake_fdw +# -------------------------------------------------------------------- +# From the Arrow project's own repository, pinned: datalake_fdw shares +# its ABI with Arrow, so the version moves only with a change here. +# 22.0.0 is the last release whose headers are C++17: from 23 on they +# need C++20, which this image's gcc 8 does not have. +# EPEL and PowerTools carry what the packages depend on (re2, thrift, ...). +# The repository is left disabled afterwards, as EPEL is. +# The release package needs nothing from a repository, so it goes in +# with them all disabled: loading their metadata under QEMU took eight +# minutes of an arm64 build. +# -------------------------------------------------------------------- +ARG ARROW_VERSION=22.0.0 +RUN dnf install -y --disablerepo='*' \ + https://packages.apache.org/artifactory/arrow/almalinux/8/apache-arrow-release-latest.rpm && \ + dnf install -y --enablerepo=epel --enablerepo=powertools \ + arrow-devel-${ARROW_VERSION}-1.el8 \ + parquet-devel-${ARROW_VERSION}-1.el8 && \ + dnf config-manager --disable 'apache-arrow-*' && \ + dnf clean all + # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py index c484c5b9408..df2c2a3c9b6 100644 --- a/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py +++ b/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py @@ -124,3 +124,13 @@ def test_locale_generated(host): locale = host.run("locale -a | grep en_US.utf8") assert locale.exit_status == 0 assert "en_US.utf8" in locale.stdout + + +def test_arrow_for_datalake_fdw(host): + """ + Test if Arrow and Parquet C++ are installed at the pinned version. + """ + for module in ("arrow", "parquet"): + version = host.run(f"pkg-config --modversion {module}") + assert version.exit_status == 0 + assert version.stdout.strip() == "22.0.0" diff --git a/devops/deploy/docker/build/rocky9/Dockerfile b/devops/deploy/docker/build/rocky9/Dockerfile index 592d9b2e10a..202016ab85f 100644 --- a/devops/deploy/docker/build/rocky9/Dockerfile +++ b/devops/deploy/docker/build/rocky9/Dockerfile @@ -172,6 +172,26 @@ RUN dnf makecache && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +# -------------------------------------------------------------------- +# Install Apache Arrow and Parquet C++ for contrib/datalake_fdw +# -------------------------------------------------------------------- +# From the Arrow project's own repository, pinned: datalake_fdw shares +# its ABI with Arrow, so the version moves only with a change here. +# EPEL and CRB carry what the packages depend on (re2, thrift, ...). +# The repository is left disabled afterwards, as EPEL is. +# The release package needs nothing from a repository, so it goes in +# with them all disabled: loading their metadata under QEMU took eight +# minutes of an arm64 build. +# -------------------------------------------------------------------- +ARG ARROW_VERSION=26.0.0 +RUN dnf install -y --disablerepo='*' \ + https://packages.apache.org/artifactory/arrow/almalinux/9/apache-arrow-release-latest.rpm && \ + dnf install -y --enablerepo=epel --enablerepo=crb \ + arrow-devel-${ARROW_VERSION}-1.el9 \ + parquet-devel-${ARROW_VERSION}-1.el9 && \ + dnf config-manager --disable 'apache-arrow-*' && \ + dnf clean all + # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py index 9da7929ff98..643ef60db30 100644 --- a/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py +++ b/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py @@ -127,3 +127,13 @@ def test_locale_generated(host): locale = host.run("locale -a | grep en_US.utf8") assert locale.exit_status == 0 assert "en_US.utf8" in locale.stdout + + +def test_arrow_for_datalake_fdw(host): + """ + Test if Arrow and Parquet C++ are installed at the pinned version. + """ + for module in ("arrow", "parquet"): + version = host.run(f"pkg-config --modversion {module}") + assert version.exit_status == 0 + assert version.stdout.strip() == "26.0.0"