diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 9601fdb83cc..b14c4c45303 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -76,8 +76,10 @@ concurrency: jobs: build-and-push: - # Set timeout to prevent hanging builds - timeout-minutes: 60 + # Set timeout to prevent hanging builds. The arm64 half runs under + # QEMU, and the Rocky builds already took 35 to 60 minutes before + # Arrow joined them, so 60 left no room. + timeout-minutes: 90 runs-on: ubuntu-latest # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 diff --git a/devops/deploy/docker/build/rocky10/Dockerfile b/devops/deploy/docker/build/rocky10/Dockerfile index 3ab40229020..bb86df1f980 100644 --- a/devops/deploy/docker/build/rocky10/Dockerfile +++ b/devops/deploy/docker/build/rocky10/Dockerfile @@ -170,6 +170,26 @@ RUN dnf makecache && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +# -------------------------------------------------------------------- +# Install Apache Arrow and Parquet C++ for contrib/datalake_fdw +# -------------------------------------------------------------------- +# From the Arrow project's own repository, pinned: datalake_fdw shares +# its ABI with Arrow, so the version moves only with a change here. +# EPEL and CRB carry what the packages depend on (re2, thrift, ...). +# The repository is left disabled afterwards, as EPEL is. +# The release package needs nothing from a repository, so it goes in +# with them all disabled: loading their metadata under QEMU took eight +# minutes of an arm64 build. +# -------------------------------------------------------------------- +ARG ARROW_VERSION=26.0.0 +RUN dnf install -y --disablerepo='*' \ + https://packages.apache.org/artifactory/arrow/almalinux/10/apache-arrow-release-latest.rpm && \ + dnf install -y --enablerepo=epel --enablerepo=crb \ + arrow-devel-${ARROW_VERSION}-1.el10 \ + parquet-devel-${ARROW_VERSION}-1.el10 && \ + dnf config-manager --disable 'apache-arrow-*' && \ + dnf clean all + # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py index 445318f5335..ac528027fae 100644 --- a/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py +++ b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py @@ -125,3 +125,13 @@ def test_locale_generated(host): locale = host.run("locale -a | grep en_US.utf8") assert locale.exit_status == 0 assert "en_US.utf8" in locale.stdout + + +def test_arrow_for_datalake_fdw(host): + """ + Test if Arrow and Parquet C++ are installed at the pinned version. + """ + for module in ("arrow", "parquet"): + version = host.run(f"pkg-config --modversion {module}") + assert version.exit_status == 0 + assert version.stdout.strip() == "26.0.0" diff --git a/devops/deploy/docker/build/rocky8/Dockerfile b/devops/deploy/docker/build/rocky8/Dockerfile index eed55e4b171..bda83e27e99 100644 --- a/devops/deploy/docker/build/rocky8/Dockerfile +++ b/devops/deploy/docker/build/rocky8/Dockerfile @@ -171,6 +171,28 @@ RUN dnf makecache && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +# -------------------------------------------------------------------- +# Install Apache Arrow and Parquet C++ for contrib/datalake_fdw +# -------------------------------------------------------------------- +# From the Arrow project's own repository, pinned: datalake_fdw shares +# its ABI with Arrow, so the version moves only with a change here. +# 22.0.0 is the last release whose headers are C++17: from 23 on they +# need C++20, which this image's gcc 8 does not have. +# EPEL and PowerTools carry what the packages depend on (re2, thrift, ...). +# The repository is left disabled afterwards, as EPEL is. +# The release package needs nothing from a repository, so it goes in +# with them all disabled: loading their metadata under QEMU took eight +# minutes of an arm64 build. +# -------------------------------------------------------------------- +ARG ARROW_VERSION=22.0.0 +RUN dnf install -y --disablerepo='*' \ + https://packages.apache.org/artifactory/arrow/almalinux/8/apache-arrow-release-latest.rpm && \ + dnf install -y --enablerepo=epel --enablerepo=powertools \ + arrow-devel-${ARROW_VERSION}-1.el8 \ + parquet-devel-${ARROW_VERSION}-1.el8 && \ + dnf config-manager --disable 'apache-arrow-*' && \ + dnf clean all + # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py index c484c5b9408..df2c2a3c9b6 100644 --- a/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py +++ b/devops/deploy/docker/build/rocky8/tests/testinfra/test_cloudberry_db_env.py @@ -124,3 +124,13 @@ def test_locale_generated(host): locale = host.run("locale -a | grep en_US.utf8") assert locale.exit_status == 0 assert "en_US.utf8" in locale.stdout + + +def test_arrow_for_datalake_fdw(host): + """ + Test if Arrow and Parquet C++ are installed at the pinned version. + """ + for module in ("arrow", "parquet"): + version = host.run(f"pkg-config --modversion {module}") + assert version.exit_status == 0 + assert version.stdout.strip() == "22.0.0" diff --git a/devops/deploy/docker/build/rocky9/Dockerfile b/devops/deploy/docker/build/rocky9/Dockerfile index 592d9b2e10a..202016ab85f 100644 --- a/devops/deploy/docker/build/rocky9/Dockerfile +++ b/devops/deploy/docker/build/rocky9/Dockerfile @@ -172,6 +172,26 @@ RUN dnf makecache && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +# -------------------------------------------------------------------- +# Install Apache Arrow and Parquet C++ for contrib/datalake_fdw +# -------------------------------------------------------------------- +# From the Arrow project's own repository, pinned: datalake_fdw shares +# its ABI with Arrow, so the version moves only with a change here. +# EPEL and CRB carry what the packages depend on (re2, thrift, ...). +# The repository is left disabled afterwards, as EPEL is. +# The release package needs nothing from a repository, so it goes in +# with them all disabled: loading their metadata under QEMU took eight +# minutes of an arm64 build. +# -------------------------------------------------------------------- +ARG ARROW_VERSION=26.0.0 +RUN dnf install -y --disablerepo='*' \ + https://packages.apache.org/artifactory/arrow/almalinux/9/apache-arrow-release-latest.rpm && \ + dnf install -y --enablerepo=epel --enablerepo=crb \ + arrow-devel-${ARROW_VERSION}-1.el9 \ + parquet-devel-${ARROW_VERSION}-1.el9 && \ + dnf config-manager --disable 'apache-arrow-*' && \ + dnf clean all + # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py index 9da7929ff98..643ef60db30 100644 --- a/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py +++ b/devops/deploy/docker/build/rocky9/tests/testinfra/test_cloudberry_db_env.py @@ -127,3 +127,13 @@ def test_locale_generated(host): locale = host.run("locale -a | grep en_US.utf8") assert locale.exit_status == 0 assert "en_US.utf8" in locale.stdout + + +def test_arrow_for_datalake_fdw(host): + """ + Test if Arrow and Parquet C++ are installed at the pinned version. + """ + for module in ("arrow", "parquet"): + version = host.run(f"pkg-config --modversion {module}") + assert version.exit_status == 0 + assert version.stdout.strip() == "26.0.0"