From 1518ee0240c7d528d5f200d70fcf5856e366d6da Mon Sep 17 00:00:00 2001 From: Jayanth Kumar Date: Thu, 13 Apr 2023 23:26:35 +0530 Subject: [PATCH 1/2] warpsql: Add support for Citus extension 1. Update Dockerfile 2. Install Citus dependencies (curl, jq) 3. Add build dependencies for Citus (gcc, libc-dev, make, curl-dev, lz4-dev, libxslt-dev, libxml2-dev) 4. Download and install Citus with specified version 5. Remove build dependencies and cleanup temporary files after Citus installation Fixes: #3 --- Dockerfile | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Dockerfile b/Dockerfile index 78d8c632..9aca912d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,3 +93,38 @@ RUN apk add --no-cache --virtual .build-deps \ && ls \ && make \ && make install + +## Adding Citus + +# Install Citus dependencies +RUN apk add --no-cache --virtual .citus-deps \ + curl \ + jq + +# Install Citus +ARG CITUS_VERSION="11.2.0" +RUN set -ex \ + && apk add --no-cache --virtual .citus-build-deps \ + gcc \ + libc-dev \ + make \ + curl-dev \ + lz4-dev \ + zstd-dev \ + clang \ + krb5-dev \ + icu-dev \ + libxslt-dev \ + libxml2-dev \ + llvm15-dev \ + && CITUS_DOWNLOAD_URL="https://github.com/citusdata/citus/archive/refs/tags/v${CITUS_VERSION}.tar.gz" \ + && curl -L -o /tmp/citus.tar.gz "${CITUS_DOWNLOAD_URL}" \ + && tar -C /tmp -xvf /tmp/citus.tar.gz \ + && chown -R postgres:postgres /tmp/citus-${CITUS_VERSION} \ + && cd /tmp/citus-${CITUS_VERSION} \ + && PATH="/usr/local/pgsql/bin:$PATH" ./configure \ + && make \ + && make install \ + && cd ~ \ + && rm -rf /tmp/citus.tar.gz /tmp/citus-${CITUS_VERSION} \ + && apk del .citus-deps .citus-build-deps From a2cfc9b1ea97601a2575596f5a02778ca153a44a Mon Sep 17 00:00:00 2001 From: Jayanth Kumar Date: Fri, 14 Apr 2023 00:17:14 +0530 Subject: [PATCH 2/2] warpsql: Update smoke tests 1. Add Citus extension smoke tests to GitHub Actions workflow 2. Update Bitnami entrypoint script to preload Citus extension 3. Test Citus extension and distributed table creation in smoke tests 4. Ensure logs are displayed for debugging purposes Fixes: #11 --- .github/workflows/smoke-test.yml | 10 ++++++++++ Dockerfile | 3 +++ bitnami/Dockerfile | 24 +++++++++++++++++++++-- bitnami/timescaledb-bitnami-entrypoint.sh | 8 ++++---- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index e5151234..8214ef9e 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -52,6 +52,16 @@ jobs: fi if psql -c "select 1" then + echo "Test Citus Extension" + psql -c "CREATE EXTENSION citus;" + psql -c "SELECT * FROM citus_version();" + + echo "Test Citus Distributed Table" + psql -c "CREATE TABLE test_distributed_table (id serial primary key, data text);" + psql -c "SELECT create_distributed_table('test_distributed_table', 'id');" + psql -c "INSERT INTO test_distributed_table (data) VALUES ('test data');" + psql -c "SELECT * FROM test_distributed_table;" + break fi sleep 1 diff --git a/Dockerfile b/Dockerfile index 9aca912d..ab6831ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,6 +73,9 @@ RUN set -ex \ && rm -rf /build \ && sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" /usr/local/share/postgresql/postgresql.conf.sample +# Add Citus to shared_preload_libraries +RUN echo "shared_preload_libraries = 'citus,timescaledb'" >> /usr/local/share/postgresql/postgresql.conf.sample + # Adding PG Vector RUN cd /tmp diff --git a/bitnami/Dockerfile b/bitnami/Dockerfile index 1042b37a..7f8eb3ea 100644 --- a/bitnami/Dockerfile +++ b/bitnami/Dockerfile @@ -51,6 +51,7 @@ RUN set +o pipefail \ ARG PG_VERSION FROM bitnami/postgresql:${PG_VERSION} ARG PG_VERSION +ARG CITUS_VERSION="11.2.0" LABEL maintainer="Timescale https://www.timescale.com" @@ -89,6 +90,22 @@ RUN set -ex \ && cd build && make install \ && cd ~ \ \ + # Install Citus + && apt-get update \ + && apt-get install -y curl liblz4-dev libzstd-dev clang libkrb5-dev libicu-dev libxslt1-dev libxml2-dev llvm-dev libcurl4-openssl-dev \ + && CITUS_DOWNLOAD_URL="https://github.com/citusdata/citus/archive/refs/tags/v${CITUS_VERSION}.tar.gz" \ + && curl -L -o /tmp/citus.tar.gz "${CITUS_DOWNLOAD_URL}" \ + && tar -C /tmp -xvf /tmp/citus.tar.gz \ + && addgroup --system postgres \ + && adduser --system --ingroup postgres --home /opt/bitnami/postgresql --no-create-home postgres \ + && chown -R postgres:postgres /tmp/citus-${CITUS_VERSION} \ + && cd /tmp/citus-${CITUS_VERSION} \ + && PATH="/opt/bitnami/postgresql/bin:$PATH" ./configure \ + && make \ + && make install \ + && cd ~ \ + && rm -rf /tmp/citus.tar.gz /tmp/citus-${CITUS_VERSION} \ + \ && apt-get autoremove --purge -y \ \ build-essential \ @@ -105,8 +122,11 @@ RUN set -ex \ "${HOME}/.cache" \ /var/lib/apt/lists/* \ /tmp/* \ - /var/tmp/* - + /var/tmp/* \ + \ + # Update shared_preload_libraries + && sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'citus,timescaledb'/g" /opt/bitnami/postgresql/conf/postgresql.conf + USER 1001 ENTRYPOINT [ "/opt/bitnami/scripts/postgresql/timescaledb-bitnami-entrypoint.sh" ] diff --git a/bitnami/timescaledb-bitnami-entrypoint.sh b/bitnami/timescaledb-bitnami-entrypoint.sh index 62b94610..6b82dc10 100755 --- a/bitnami/timescaledb-bitnami-entrypoint.sh +++ b/bitnami/timescaledb-bitnami-entrypoint.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# We have to use the bitnami configuration variable to add timescaledb to -# shared preload list, or else it gets overwritten. +# We have to use the Bitnami configuration variable to add TimescaleDB and Citus to +# the shared preload list, or else it gets overwritten. if [ -z "$POSTGRESQL_SHARED_PRELOAD_LIBRARIES" ] then - POSTGRESQL_SHARED_PRELOAD_LIBRARIES=timescaledb + POSTGRESQL_SHARED_PRELOAD_LIBRARIES="citus,timescaledb" else - POSTGRESQL_SHARED_PRELOAD_LIBRARIES="$POSTGRESQL_SHARED_PRELOAD_LIBRARIES,timescaledb" + POSTGRESQL_SHARED_PRELOAD_LIBRARIES="citus,timescaledb,$POSTGRESQL_SHARED_PRELOAD_LIBRARIES" fi export POSTGRESQL_SHARED_PRELOAD_LIBRARIES