From 937f3bcc6c4ffb053afc7d21e9b3e0f33d74511a Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 4 Jul 2021 13:37:49 -0700 Subject: [PATCH 01/22] run ballista integration test --- .github/workflows/integration_tests.yml | 27 +++++++++++++++++++++++++ benchmarks/tpch-gen.sh | 6 +++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/integration_tests.yml diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 0000000000000..ae473870fa7ee --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Integration tests +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + ./dev/integration-tests.sh diff --git a/benchmarks/tpch-gen.sh b/benchmarks/tpch-gen.sh index a46f65111069e..9d4abdaba2638 100755 --- a/benchmarks/tpch-gen.sh +++ b/benchmarks/tpch-gen.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -#set -e +set -e pushd .. . ./dev/build-set-env.sh @@ -29,6 +29,6 @@ if test -f "$FILE"; then echo "$FILE exists." else mkdir data 2>/dev/null - docker run -v `pwd`/data:/data -it --rm ballista-tpchgen:$BALLISTA_VERSION + docker run -v `pwd`/data:/data -t --rm ballista-tpchgen:$BALLISTA_VERSION ls -l data -fi \ No newline at end of file +fi From 2fee15dc1497877bd99d6b7dbade42f5819edde4 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 5 Jul 2021 21:57:43 -0700 Subject: [PATCH 02/22] reduce openssl build verbosity --- dev/docker/ballista-base.dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dev/docker/ballista-base.dockerfile b/dev/docker/ballista-base.dockerfile index 4dc7a065e6907..f5b9ea1ada077 100644 --- a/dev/docker/ballista-base.dockerfile +++ b/dev/docker/ballista-base.dockerfile @@ -42,18 +42,17 @@ ARG OPENSSL_VERSION=1.1.1b # component. It's possible that this will cause bizarre and terrible things to # happen. There may be "sanitized" header RUN echo "Building OpenSSL" && \ - ls /usr/include/linux && \ mkdir -p /usr/local/musl/include && \ ln -s /usr/include/linux /usr/local/musl/include/linux && \ ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ cd /tmp && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ - tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ + tar xzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make && \ - make install && \ + make install 1>/dev/null && \ rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ rm -r /tmp/* From 331d62e6ada7b09c07a54f2c409904e9ee97330c Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 5 Jul 2021 22:22:06 -0700 Subject: [PATCH 03/22] suppress all make command echos --- dev/docker/ballista-base.dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dev/docker/ballista-base.dockerfile b/dev/docker/ballista-base.dockerfile index f5b9ea1ada077..96f7b647092b5 100644 --- a/dev/docker/ballista-base.dockerfile +++ b/dev/docker/ballista-base.dockerfile @@ -50,9 +50,9 @@ RUN echo "Building OpenSSL" && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ tar xzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ - env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ - env C_INCLUDE_PATH=/usr/local/musl/include/ make && \ - make install 1>/dev/null && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make -s depend && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make -s && \ + make -s install && \ rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ rm -r /tmp/* @@ -62,7 +62,7 @@ RUN echo "Building zlib" && \ curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \ - make && make install && \ + make -s && make -s install && \ rm -r /tmp/* RUN echo "Building libpq" && \ @@ -71,8 +71,8 @@ RUN echo "Building libpq" && \ curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ CC=musl-gcc CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ - cd src/interfaces/libpq && make all-static-lib && make install-lib-static && \ - cd ../../bin/pg_config && make && make install && \ + cd src/interfaces/libpq && make -s all-static-lib && make -s install-lib-static && \ + cd ../../bin/pg_config && make -s && make -s install && \ rm -r /tmp/* ENV OPENSSL_DIR=/usr/local/musl/ \ @@ -95,4 +95,4 @@ RUN cargo install cargo-build-deps # prepare toolchain RUN rustup update && \ - rustup component add rustfmt \ No newline at end of file + rustup component add rustfmt From 6e3e60b00ab54688a18a48d7daf97c75e5439c0a Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 5 Jul 2021 22:23:15 -0700 Subject: [PATCH 04/22] suppress openssl install log & tpch build log --- benchmarks/tpchgen.dockerfile | 2 +- dev/docker/ballista-base.dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/tpchgen.dockerfile b/benchmarks/tpchgen.dockerfile index 69434708bf9a7..e333297a6790f 100644 --- a/benchmarks/tpchgen.dockerfile +++ b/benchmarks/tpchgen.dockerfile @@ -22,7 +22,7 @@ RUN apt-get update && \ RUN git clone https://github.com/databricks/tpch-dbgen.git && \ cd tpch-dbgen && \ - make + make -s WORKDIR /tpch-dbgen ADD entrypoint.sh /tpch-dbgen/ diff --git a/dev/docker/ballista-base.dockerfile b/dev/docker/ballista-base.dockerfile index 96f7b647092b5..8cfd0683f6013 100644 --- a/dev/docker/ballista-base.dockerfile +++ b/dev/docker/ballista-base.dockerfile @@ -52,7 +52,7 @@ RUN echo "Building OpenSSL" && \ env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make -s depend && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make -s && \ - make -s install && \ + make -s install 1>/dev/null && \ rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ rm -r /tmp/* @@ -62,7 +62,7 @@ RUN echo "Building zlib" && \ curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \ - make -s && make -s install && \ + make -s && make -s install 1>/dev/null && \ rm -r /tmp/* RUN echo "Building libpq" && \ From 4ea832eb875679cd0fa9a0dbe9c6f725a0cbbc74 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 5 Jul 2021 23:05:33 -0700 Subject: [PATCH 05/22] build integration test binaries in debug mode to reduce build time --- dev/build-ballista-docker.sh | 9 ++++++++- dev/docker/ballista.dockerfile | 22 +++++++++++----------- dev/integration-tests.sh | 3 ++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh index bc028da9e716a..260a6ecc9337c 100755 --- a/dev/build-ballista-docker.sh +++ b/dev/build-ballista-docker.sh @@ -21,4 +21,11 @@ set -e . ./dev/build-set-env.sh docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . -docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . + +if [[ -z "${TEST}" ]]; then + CARGO_FLAG="" +else + # build ballista in debug mode to reduce build time when TEST env var is set + CARGO_FLAG="--build-arg RELEASE_FLAG=" +fi +docker build -t ballista:$BALLISTA_VERSION ${CARGO_FLAG} -f dev/docker/ballista.dockerfile . diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 730e86749a63b..141c404567d2f 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -63,17 +63,17 @@ ARG RELEASE_FLAG=--release ENV FORCE_REBUILD='true' RUN cargo build $RELEASE_FLAG -# put the executor on /executor (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi - -# put the scheduler on /scheduler (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi - -# put the tpch on /tpch (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi +SHELL ["/bin/bash", "-c"] +# binaries need to be copied from different places depending on RELEASE_FLAG +RUN if [[ "$RELEASE_FLAG" == *"--release"* ]]; then \ + mv /tmp/ballista/target/release/ballista-executor /executor && \ + mv /tmp/ballista/target/release/ballista-scheduler /scheduler && \ + mv /tmp/ballista/target/release/tpch /tpch; \ + else \ + mv /tmp/ballista/target/debug/ballista-executor /executor && \ + mv /tmp/ballista/target/debug/ballista-scheduler /scheduler && \ + mv /tmp/ballista/target/debug/tpch /tpch; \ + fi # Copy the binary into a new container for a smaller docker image FROM ballista-base:0.5.0-SNAPSHOT diff --git a/dev/integration-tests.sh b/dev/integration-tests.sh index 3c1306319c369..3d7e8e67f8d29 100755 --- a/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -17,7 +17,8 @@ # specific language governing permissions and limitations # under the License. set -e -./dev/build-ballista-docker.sh + +TEST=1 ./dev/build-ballista-docker.sh pushd benchmarks ./tpch-gen.sh From deadb0d935261d03e2e1b375204835a4db6261d7 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 5 Jul 2021 23:54:42 -0700 Subject: [PATCH 06/22] do not print query result in integration test --- benchmarks/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 8e36424da89f0..11d40bb165f39 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -22,5 +22,5 @@ set -e cd / for query in 1 3 5 6 10 12 do - /tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1 --debug + /tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1 done From 93a261d50504c84af3bb2bbf15eb5de25372987a Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Tue, 6 Jul 2021 00:19:24 -0700 Subject: [PATCH 07/22] Revert "build integration test binaries in debug mode to reduce build time" This reverts commit 4ea832eb875679cd0fa9a0dbe9c6f725a0cbbc74. --- dev/build-ballista-docker.sh | 9 +-------- dev/docker/ballista.dockerfile | 22 +++++++++++----------- dev/integration-tests.sh | 3 +-- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh index 260a6ecc9337c..bc028da9e716a 100755 --- a/dev/build-ballista-docker.sh +++ b/dev/build-ballista-docker.sh @@ -21,11 +21,4 @@ set -e . ./dev/build-set-env.sh docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . - -if [[ -z "${TEST}" ]]; then - CARGO_FLAG="" -else - # build ballista in debug mode to reduce build time when TEST env var is set - CARGO_FLAG="--build-arg RELEASE_FLAG=" -fi -docker build -t ballista:$BALLISTA_VERSION ${CARGO_FLAG} -f dev/docker/ballista.dockerfile . +docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 141c404567d2f..730e86749a63b 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -63,17 +63,17 @@ ARG RELEASE_FLAG=--release ENV FORCE_REBUILD='true' RUN cargo build $RELEASE_FLAG -SHELL ["/bin/bash", "-c"] -# binaries need to be copied from different places depending on RELEASE_FLAG -RUN if [[ "$RELEASE_FLAG" == *"--release"* ]]; then \ - mv /tmp/ballista/target/release/ballista-executor /executor && \ - mv /tmp/ballista/target/release/ballista-scheduler /scheduler && \ - mv /tmp/ballista/target/release/tpch /tpch; \ - else \ - mv /tmp/ballista/target/debug/ballista-executor /executor && \ - mv /tmp/ballista/target/debug/ballista-scheduler /scheduler && \ - mv /tmp/ballista/target/debug/tpch /tpch; \ - fi +# put the executor on /executor (need to be copied from different places depending on FLAG) +ENV RELEASE_FLAG=${RELEASE_FLAG} +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi + +# put the scheduler on /scheduler (need to be copied from different places depending on FLAG) +ENV RELEASE_FLAG=${RELEASE_FLAG} +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi + +# put the tpch on /tpch (need to be copied from different places depending on FLAG) +ENV RELEASE_FLAG=${RELEASE_FLAG} +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi # Copy the binary into a new container for a smaller docker image FROM ballista-base:0.5.0-SNAPSHOT diff --git a/dev/integration-tests.sh b/dev/integration-tests.sh index 3d7e8e67f8d29..3c1306319c369 100755 --- a/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -17,8 +17,7 @@ # specific language governing permissions and limitations # under the License. set -e - -TEST=1 ./dev/build-ballista-docker.sh +./dev/build-ballista-docker.sh pushd benchmarks ./tpch-gen.sh From a795f9bd508cae44dd8e17a09efa63b741f43882 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sat, 10 Jul 2021 14:42:08 -0700 Subject: [PATCH 08/22] make tpch data gen less verbose in CI --- benchmarks/entrypoint.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/entrypoint.sh b/benchmarks/entrypoint.sh index 71c04324afd14..19c1ee769d0ed 100755 --- a/benchmarks/entrypoint.sh +++ b/benchmarks/entrypoint.sh @@ -18,5 +18,11 @@ set -e cd /tpch-dbgen -./dbgen -vf -s 1 -mv *.tbl /data \ No newline at end of file +if [[ $- == *i* ]]; then + ./dbgen -vf -s 1 +else + # do not run in verbose mode when running within an non-interactive + # terminal, for example Github Action job + ./dbgen -f -s 1 +fi +mv *.tbl /data From 1d9809e681272e7cfffd4079323b30d0587b8c97 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 18 Jul 2021 23:16:39 -0700 Subject: [PATCH 09/22] test caching build cache with buildx docker-container driver --- .github/workflows/integration_tests.yml | 14 ++++++++++ dev/build-ballista-docker.sh | 24 ++++++++++++++++- dev/docker/ballista-base.dockerfile | 35 +++++++++++++++++++------ dev/integration-tests.sh | 1 + 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ae473870fa7ee..a8596c14b6a28 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -23,5 +23,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: docker-builder + uses: docker/setup-buildx-action@v1 + - name: Cache Buildx + uses: actions/cache@v2 + id: buildx-cache + with: + path: "${{ runner.temp }}/buildx-cache" + key: ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.lock') }} + ${{ runner.os }}-buildx- - run: | ./dev/integration-tests.sh + env: + BUILDX_CACHE_DIR: "${{ runner.temp }}/buildx-cache" diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh index bc028da9e716a..c99b52ed238be 100755 --- a/dev/build-ballista-docker.sh +++ b/dev/build-ballista-docker.sh @@ -20,5 +20,27 @@ set -e . ./dev/build-set-env.sh -docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . + +# If running within CI, use docker-container driver to cache build cache into +# local folders so we can share them between runs. +if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then + echo "building docker image in CI, saving build cache to local folder ${BUILDX_CACHE_DIR}." + BUILDER=ballista-docker-builder + docker buildx inspect "${BUILDER}" &>/dev/null || \ + docker buildx create --driver docker-container --name "${BUILDER}" --use + BUILD_ARGS=(buildx build \ + --builder ${BUILDER} \ + --cache-from="type=local,src=${BUILDX_CACHE_DIR}" \ + --cache-to="type=local,mode=max,dest=${BUILDX_CACHE_DIR}" \ + --load) +else + echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=1" + echo "To install docker buildx, follow https://github.com/docker/buildx#installing" + export DOCKER_BUILDKIT=1 + BUILD_ARGS=(build) +fi + +docker ${BUILD_ARGS[@]} -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . +# couldn't use buildx for the ballista app image build due to restrictions of +# the docker-container driver. see https://github.com/docker/buildx/issues/301 docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . diff --git a/dev/docker/ballista-base.dockerfile b/dev/docker/ballista-base.dockerfile index 8cfd0683f6013..d3f25bf9dc76c 100644 --- a/dev/docker/ballista-base.dockerfile +++ b/dev/docker/ballista-base.dockerfile @@ -1,3 +1,5 @@ +# syntax = docker/dockerfile:1.2 + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -27,6 +29,18 @@ FROM rust:1.53.0-buster AS builder RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl +ARG SCCACHE_TAR_URL=https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz + +RUN curl -LsSf ${SCCACHE_TAR_URL} > /tmp/sccache.tar.gz && \ + tar axvf /tmp/sccache.tar.gz --strip-components=1 -C /usr/local/bin --wildcards --no-anchored 'sccache' && \ + chmod +x /usr/local/bin/sccache && \ + sccache --version && \ + rm -rf /tmp/sccache.tar.gz + +ENV RUSTC_WRAPPER=/usr/local/bin/sccache +ENV CARGO_INCREMENTAL=0 +ENV SCCACHE_CACHE_SIZE=1G + #NOTE: the following was copied from https://github.com/emk/rust-musl-builder/blob/master/Dockerfile under Apache 2.0 license # The OpenSSL version to use. We parameterize this because many Rust @@ -41,7 +55,8 @@ ARG OPENSSL_VERSION=1.1.1b # necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine" # component. It's possible that this will cause bizarre and terrible things to # happen. There may be "sanitized" header -RUN echo "Building OpenSSL" && \ +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building OpenSSL" && \ mkdir -p /usr/local/musl/include && \ ln -s /usr/include/linux /usr/local/musl/include/linux && \ ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ @@ -49,28 +64,30 @@ RUN echo "Building OpenSSL" && \ cd /tmp && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ tar xzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ - env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ + env CC="sccache musl-gcc" ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make -s depend && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make -s && \ make -s install 1>/dev/null && \ rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ rm -r /tmp/* -RUN echo "Building zlib" && \ +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building zlib" && \ cd /tmp && \ ZLIB_VERSION=1.2.11 && \ curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ - CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \ + CC="sccache musl-gcc" ./configure --static --prefix=/usr/local/musl && \ make -s && make -s install 1>/dev/null && \ rm -r /tmp/* -RUN echo "Building libpq" && \ +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building libpq" && \ cd /tmp && \ POSTGRESQL_VERSION=11.2 && \ curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ - CC=musl-gcc CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ + CC="sccache musl-gcc" CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ cd src/interfaces/libpq && make -s all-static-lib && make -s install-lib-static && \ cd ../../bin/pg_config && make -s && make -s install && \ rm -r /tmp/* @@ -91,8 +108,10 @@ ENV OPENSSL_DIR=/usr/local/musl/ \ ## Download the target for static linking. RUN rustup target add x86_64-unknown-linux-musl -RUN cargo install cargo-build-deps +RUN --mount=type=cache,target=/root/.cache/sccache \ + cargo install cargo-build-deps # prepare toolchain -RUN rustup update && \ +RUN --mount=type=cache,target=/root/.cache/sccache \ + rustup update && \ rustup component add rustfmt diff --git a/dev/integration-tests.sh b/dev/integration-tests.sh index 3c1306319c369..a995f6d6bd6aa 100755 --- a/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -17,6 +17,7 @@ # specific language governing permissions and limitations # under the License. set -e + ./dev/build-ballista-docker.sh pushd benchmarks ./tpch-gen.sh From 633cb18738ae282aa8fa0b6db8b4824d49a4243e Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Tue, 20 Jul 2021 23:27:22 -0700 Subject: [PATCH 10/22] cache tpch gen docker image build --- benchmarks/tpch-gen.sh | 2 +- dev/build-ballista-docker.sh | 19 ------------------- dev/build-set-env.sh | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/benchmarks/tpch-gen.sh b/benchmarks/tpch-gen.sh index 9d4abdaba2638..c5aec143083d7 100755 --- a/benchmarks/tpch-gen.sh +++ b/benchmarks/tpch-gen.sh @@ -21,7 +21,7 @@ set -e pushd .. . ./dev/build-set-env.sh popd -docker build -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile . +docker ${BUILD_ARGS[@]} -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile . # Generate data into the ./data directory if it does not already exist FILE=./data/supplier.tbl diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh index c99b52ed238be..87d66d3569708 100755 --- a/dev/build-ballista-docker.sh +++ b/dev/build-ballista-docker.sh @@ -21,25 +21,6 @@ set -e . ./dev/build-set-env.sh -# If running within CI, use docker-container driver to cache build cache into -# local folders so we can share them between runs. -if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then - echo "building docker image in CI, saving build cache to local folder ${BUILDX_CACHE_DIR}." - BUILDER=ballista-docker-builder - docker buildx inspect "${BUILDER}" &>/dev/null || \ - docker buildx create --driver docker-container --name "${BUILDER}" --use - BUILD_ARGS=(buildx build \ - --builder ${BUILDER} \ - --cache-from="type=local,src=${BUILDX_CACHE_DIR}" \ - --cache-to="type=local,mode=max,dest=${BUILDX_CACHE_DIR}" \ - --load) -else - echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=1" - echo "To install docker buildx, follow https://github.com/docker/buildx#installing" - export DOCKER_BUILDKIT=1 - BUILD_ARGS=(build) -fi - docker ${BUILD_ARGS[@]} -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . # couldn't use buildx for the ballista app image build due to restrictions of # the docker-container driver. see https://github.com/docker/buildx/issues/301 diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh index 3eb29e7ce1443..99c9820e33717 100755 --- a/dev/build-set-env.sh +++ b/dev/build-set-env.sh @@ -18,3 +18,22 @@ # under the License. export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista/rust/core/Cargo.toml) + +# If running within CI, use docker-container driver to cache build cache into +# local folders so we can share them between runs. +if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then + echo "building docker image in CI, saving build cache to local folder ${BUILDX_CACHE_DIR}." + BUILDER=ballista-docker-builder + docker buildx inspect "${BUILDER}" &>/dev/null || \ + docker buildx create --driver docker-container --name "${BUILDER}" --use + BUILD_ARGS=(buildx build \ + --builder ${BUILDER} \ + --cache-from="type=local,src=${BUILDX_CACHE_DIR}" \ + --cache-to="type=local,mode=max,dest=${BUILDX_CACHE_DIR}" \ + --load) +else + echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=1" + echo "To install docker buildx, follow https://github.com/docker/buildx#installing" + export DOCKER_BUILDKIT=1 + BUILD_ARGS=(build) +fi From f3539dd26019a95b86981999cbe4372fd8b64da2 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Wed, 21 Jul 2021 21:24:37 -0700 Subject: [PATCH 11/22] use cargo.toml and dockerfile as cache hashkey --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a8596c14b6a28..5dc4a2e1c77c0 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -31,9 +31,9 @@ jobs: id: buildx-cache with: path: "${{ runner.temp }}/buildx-cache" - key: ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.lock') }} + ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} ${{ runner.os }}-buildx- - run: | ./dev/integration-tests.sh From 686089e7c47df63bb19eab9c341bc4feab321c4c Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Wed, 21 Jul 2021 23:39:48 -0700 Subject: [PATCH 12/22] merge base into main ballista dockerfile --- dev/build-ballista-docker.sh | 5 +- dev/docker/ballista-base.dockerfile | 117 ---------------------------- dev/docker/ballista.dockerfile | 101 +++++++++++++++++++++++- 3 files changed, 98 insertions(+), 125 deletions(-) delete mode 100644 dev/docker/ballista-base.dockerfile diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh index 87d66d3569708..cd3b1be4b2ebe 100755 --- a/dev/build-ballista-docker.sh +++ b/dev/build-ballista-docker.sh @@ -21,7 +21,4 @@ set -e . ./dev/build-set-env.sh -docker ${BUILD_ARGS[@]} -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . -# couldn't use buildx for the ballista app image build due to restrictions of -# the docker-container driver. see https://github.com/docker/buildx/issues/301 -docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . +docker ${BUILD_ARGS[@]} -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . diff --git a/dev/docker/ballista-base.dockerfile b/dev/docker/ballista-base.dockerfile deleted file mode 100644 index d3f25bf9dc76c..0000000000000 --- a/dev/docker/ballista-base.dockerfile +++ /dev/null @@ -1,117 +0,0 @@ -# syntax = docker/dockerfile:1.2 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Turn .dockerignore to .dockerallow by excluding everything and explicitly -# allowing specific files and directories. This enables us to quickly add -# dependency files to the docker content without scanning the whole directory. -# This setup requires to all of our docker containers have arrow's source -# as a mounted directory. - - -# Base image extends debian:buster-slim -FROM rust:1.53.0-buster AS builder - -RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl - -ARG SCCACHE_TAR_URL=https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz - -RUN curl -LsSf ${SCCACHE_TAR_URL} > /tmp/sccache.tar.gz && \ - tar axvf /tmp/sccache.tar.gz --strip-components=1 -C /usr/local/bin --wildcards --no-anchored 'sccache' && \ - chmod +x /usr/local/bin/sccache && \ - sccache --version && \ - rm -rf /tmp/sccache.tar.gz - -ENV RUSTC_WRAPPER=/usr/local/bin/sccache -ENV CARGO_INCREMENTAL=0 -ENV SCCACHE_CACHE_SIZE=1G - -#NOTE: the following was copied from https://github.com/emk/rust-musl-builder/blob/master/Dockerfile under Apache 2.0 license - -# The OpenSSL version to use. We parameterize this because many Rust -# projects will fail to build with 1.1. -#ARG OPENSSL_VERSION=1.0.2r -ARG OPENSSL_VERSION=1.1.1b - -# Build a static library version of OpenSSL using musl-libc. This is needed by -# the popular Rust `hyper` crate. -# -# We point /usr/local/musl/include/linux at some Linux kernel headers (not -# necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine" -# component. It's possible that this will cause bizarre and terrible things to -# happen. There may be "sanitized" header -RUN --mount=type=cache,target=/root/.cache/sccache \ - echo "Building OpenSSL" && \ - mkdir -p /usr/local/musl/include && \ - ln -s /usr/include/linux /usr/local/musl/include/linux && \ - ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ - ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ - cd /tmp && \ - curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ - tar xzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ - env CC="sccache musl-gcc" ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ - env C_INCLUDE_PATH=/usr/local/musl/include/ make -s depend && \ - env C_INCLUDE_PATH=/usr/local/musl/include/ make -s && \ - make -s install 1>/dev/null && \ - rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ - rm -r /tmp/* - -RUN --mount=type=cache,target=/root/.cache/sccache \ - echo "Building zlib" && \ - cd /tmp && \ - ZLIB_VERSION=1.2.11 && \ - curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ - tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ - CC="sccache musl-gcc" ./configure --static --prefix=/usr/local/musl && \ - make -s && make -s install 1>/dev/null && \ - rm -r /tmp/* - -RUN --mount=type=cache,target=/root/.cache/sccache \ - echo "Building libpq" && \ - cd /tmp && \ - POSTGRESQL_VERSION=11.2 && \ - curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ - tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ - CC="sccache musl-gcc" CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ - cd src/interfaces/libpq && make -s all-static-lib && make -s install-lib-static && \ - cd ../../bin/pg_config && make -s && make -s install && \ - rm -r /tmp/* - -ENV OPENSSL_DIR=/usr/local/musl/ \ - OPENSSL_INCLUDE_DIR=/usr/local/musl/include/ \ - DEP_OPENSSL_INCLUDE=/usr/local/musl/include/ \ - OPENSSL_LIB_DIR=/usr/local/musl/lib/ \ - OPENSSL_STATIC=1 \ - PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=1 \ - PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - PKG_CONFIG_ALLOW_CROSS=true \ - PKG_CONFIG_ALL_STATIC=true \ - LIBZ_SYS_STATIC=1 \ - TARGET=musl - -# The content copied mentioned in the NOTE above ends here. - -## Download the target for static linking. -RUN rustup target add x86_64-unknown-linux-musl -RUN --mount=type=cache,target=/root/.cache/sccache \ - cargo install cargo-build-deps - -# prepare toolchain -RUN --mount=type=cache,target=/root/.cache/sccache \ - rustup update && \ - rustup component add rustfmt diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 730e86749a63b..bb11659cf3e42 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -21,11 +21,104 @@ # This setup requires to all of our docker containers have arrow's source # as a mounted directory. -ARG RELEASE_FLAG=--release -FROM ballista-base:0.5.0-SNAPSHOT AS base +# Base image extends debian:buster-slim +FROM rust:1.53.0-buster AS base + +RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl + +ARG SCCACHE_TAR_URL=https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz + +RUN curl -LsSf ${SCCACHE_TAR_URL} > /tmp/sccache.tar.gz && \ + tar axvf /tmp/sccache.tar.gz --strip-components=1 -C /usr/local/bin --wildcards --no-anchored 'sccache' && \ + chmod +x /usr/local/bin/sccache && \ + sccache --version && \ + rm -rf /tmp/sccache.tar.gz + +ENV RUSTC_WRAPPER=/usr/local/bin/sccache +ENV CARGO_INCREMENTAL=0 +ENV SCCACHE_CACHE_SIZE=1G + +#NOTE: the following was copied from https://github.com/emk/rust-musl-builder/blob/master/Dockerfile under Apache 2.0 license + +# The OpenSSL version to use. We parameterize this because many Rust +# projects will fail to build with 1.1. +#ARG OPENSSL_VERSION=1.0.2r +ARG OPENSSL_VERSION=1.1.1b + +# Build a static library version of OpenSSL using musl-libc. This is needed by +# the popular Rust `hyper` crate. +# +# We point /usr/local/musl/include/linux at some Linux kernel headers (not +# necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine" +# component. It's possible that this will cause bizarre and terrible things to +# happen. There may be "sanitized" header +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building OpenSSL" && \ + mkdir -p /usr/local/musl/include && \ + ln -s /usr/include/linux /usr/local/musl/include/linux && \ + ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ + ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ + cd /tmp && \ + curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ + tar xzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ + env CC="sccache musl-gcc" ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make -s depend && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make -s && \ + make -s install 1>/dev/null && \ + rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ + rm -r /tmp/* + +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building zlib" && \ + cd /tmp && \ + ZLIB_VERSION=1.2.11 && \ + curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ + tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ + CC="sccache musl-gcc" ./configure --static --prefix=/usr/local/musl && \ + make -s && make -s install 1>/dev/null && \ + rm -r /tmp/* + +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building libpq" && \ + cd /tmp && \ + POSTGRESQL_VERSION=11.2 && \ + curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ + tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ + CC="sccache musl-gcc" CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ + cd src/interfaces/libpq && make -s all-static-lib && make -s install-lib-static && \ + cd ../../bin/pg_config && make -s && make -s install && \ + rm -r /tmp/* + +ENV OPENSSL_DIR=/usr/local/musl/ \ + OPENSSL_INCLUDE_DIR=/usr/local/musl/include/ \ + DEP_OPENSSL_INCLUDE=/usr/local/musl/include/ \ + OPENSSL_LIB_DIR=/usr/local/musl/lib/ \ + OPENSSL_STATIC=1 \ + PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=1 \ + PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ + PKG_CONFIG_ALLOW_CROSS=true \ + PKG_CONFIG_ALL_STATIC=true \ + LIBZ_SYS_STATIC=1 \ + TARGET=musl + +# The content copied mentioned in the NOTE above ends here. + +## Download the target for static linking. +RUN rustup target add x86_64-unknown-linux-musl +RUN --mount=type=cache,target=/root/.cache/sccache \ + cargo install cargo-build-deps + +# prepare toolchain +RUN --mount=type=cache,target=/root/.cache/sccache \ + rustup update && \ + rustup component add rustfmt + WORKDIR /tmp/ballista RUN apt-get -y install cmake -RUN cargo install cargo-chef +RUN cargo install cargo-chef + + +ARG RELEASE_FLAG=--release FROM base as planner RUN mkdir /tmp/ballista/ballista @@ -76,7 +169,7 @@ ENV RELEASE_FLAG=${RELEASE_FLAG} RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi # Copy the binary into a new container for a smaller docker image -FROM ballista-base:0.5.0-SNAPSHOT +FROM base COPY --from=builder /executor / From aa475968920d7dfa38ad3f72ea5fb95390e3b808 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 25 Jul 2021 13:28:33 -0700 Subject: [PATCH 13/22] enable sccache for appliation build --- dev/docker/ballista.dockerfile | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index bb11659cf3e42..0dbdbbd05ae0a 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -114,8 +114,13 @@ RUN --mount=type=cache,target=/root/.cache/sccache \ rustup component add rustfmt WORKDIR /tmp/ballista -RUN apt-get -y install cmake -RUN cargo install cargo-chef +RUN apt-get -y install cmake \ + && apt-get clean \ + && apt-get autoremove \ + && rm -rf /var/cache/apt/archives/* \ + && rm -rf /var/lib/apt/lists/* +RUN --mount=type=cache,target=/root/.cache/sccache \ + cargo install cargo-chef ARG RELEASE_FLAG=--release @@ -135,7 +140,8 @@ RUN cargo chef prepare --recipe-path recipe.json FROM base as cacher COPY --from=planner /tmp/ballista/recipe.json recipe.json -RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json +RUN --mount=type=cache,target=/root/.cache/sccache \ + cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder RUN mkdir /tmp/ballista/ballista @@ -154,7 +160,8 @@ ARG RELEASE_FLAG=--release # force build.rs to run to generate configure_me code. ENV FORCE_REBUILD='true' -RUN cargo build $RELEASE_FLAG +RUN --mount=type=cache,target=/root/.cache/sccache \ + cargo build $RELEASE_FLAG # put the executor on /executor (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} From 4cb60fa406429fd338ec0ed9a31378176911756e Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 26 Jul 2021 21:50:05 -0700 Subject: [PATCH 14/22] cache cargo registry download --- dev/docker/ballista.dockerfile | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 0dbdbbd05ae0a..43dab86d8d4be 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -36,7 +36,7 @@ RUN curl -LsSf ${SCCACHE_TAR_URL} > /tmp/sccache.tar.gz && \ ENV RUSTC_WRAPPER=/usr/local/bin/sccache ENV CARGO_INCREMENTAL=0 -ENV SCCACHE_CACHE_SIZE=1G +ENV SCCACHE_CACHE_SIZE=2G #NOTE: the following was copied from https://github.com/emk/rust-musl-builder/blob/master/Dockerfile under Apache 2.0 license @@ -106,10 +106,16 @@ ENV OPENSSL_DIR=/usr/local/musl/ \ ## Download the target for static linking. RUN rustup target add x86_64-unknown-linux-musl RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ cargo install cargo-build-deps # prepare toolchain RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ rustup update && \ rustup component add rustfmt @@ -120,6 +126,9 @@ RUN apt-get -y install cmake \ && rm -rf /var/cache/apt/archives/* \ && rm -rf /var/lib/apt/lists/* RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ cargo install cargo-chef @@ -141,6 +150,9 @@ RUN cargo chef prepare --recipe-path recipe.json FROM base as cacher COPY --from=planner /tmp/ballista/recipe.json recipe.json RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder @@ -161,6 +173,9 @@ ARG RELEASE_FLAG=--release # force build.rs to run to generate configure_me code. ENV FORCE_REBUILD='true' RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ cargo build $RELEASE_FLAG # put the executor on /executor (need to be copied from different places depending on FLAG) From c48760a1b9f1eb343ab8c86ab8ece4c0a94ec48e Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sat, 7 Aug 2021 18:58:44 -0700 Subject: [PATCH 15/22] copy cargo_home from cacher layer --- dev/docker/ballista.dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 43dab86d8d4be..dc8f3bf4175c8 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -167,6 +167,7 @@ COPY datafusion ./datafusion/ COPY ballista ./ballista/ COPY datafusion-cli ./datafusion-cli/ COPY datafusion-examples ./datafusion-examples/ +COPY --from=cacher $CARGO_HOME $CARGO_HOME COPY --from=cacher /tmp/ballista/target target ARG RELEASE_FLAG=--release @@ -178,17 +179,17 @@ RUN --mount=type=cache,target=/root/.cache/sccache \ --mount=type=cache,target=/usr/local/cargo/git/db \ cargo build $RELEASE_FLAG -# put the executor on /executor (need to be copied from different places depending on FLAG) +# copy the binaries into / (need to be copied from different places depending on RELEASE_FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi - -# put the scheduler on /scheduler (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi - -# put the tpch on /tpch (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi +RUN if [ -z "$RELEASE_FLAG" ]; then \ + mv /tmp/ballista/target/debug/ballista-executor /executor; \ + mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; \ + mv /tmp/ballista/target/debug/tpch /tpch; \ + else \ + mv /tmp/ballista/target/release/ballista-executor /executor; \ + mv /tmp/ballista/target/release/ballista-scheduler /scheduler; \ + mv /tmp/ballista/target/release/tpch /tpch; \ + fi # Copy the binary into a new container for a smaller docker image FROM base From 4e4cfd950df8c77f4d248529f9ca791107f627e6 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sat, 7 Aug 2021 19:58:02 -0700 Subject: [PATCH 16/22] debug buildx builder cache --- .github/workflows/integration_tests.yml | 2 ++ dev/build-set-env.sh | 8 ++++---- dev/docker/ballista.dockerfile | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 5dc4a2e1c77c0..8ee306e012d38 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -37,5 +37,7 @@ jobs: ${{ runner.os }}-buildx- - run: | ./dev/integration-tests.sh + docker buildx --builder "${BUILDX_BUILDER}" du env: + BUILDX_BUILDER: ballista-docker-builder BUILDX_CACHE_DIR: "${{ runner.temp }}/buildx-cache" diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh index 99c9820e33717..d3af4c50cc21a 100755 --- a/dev/build-set-env.sh +++ b/dev/build-set-env.sh @@ -23,11 +23,11 @@ export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista # local folders so we can share them between runs. if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then echo "building docker image in CI, saving build cache to local folder ${BUILDX_CACHE_DIR}." - BUILDER=ballista-docker-builder - docker buildx inspect "${BUILDER}" &>/dev/null || \ - docker buildx create --driver docker-container --name "${BUILDER}" --use + BUILDX_BUILDER="${BUILDX_BUILDER:-ballista-docker-builder}" + docker buildx inspect "${BUILDX_BUILDER}" &>/dev/null || \ + docker buildx create --driver docker-container --name "${BUILDX_BUILDER}" --use BUILD_ARGS=(buildx build \ - --builder ${BUILDER} \ + --builder ${BUILDX_BUILDER} \ --cache-from="type=local,src=${BUILDX_CACHE_DIR}" \ --cache-to="type=local,mode=max,dest=${BUILDX_CACHE_DIR}" \ --load) diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index dc8f3bf4175c8..61fd2d2482bf4 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -169,7 +169,6 @@ COPY datafusion-cli ./datafusion-cli/ COPY datafusion-examples ./datafusion-examples/ COPY --from=cacher $CARGO_HOME $CARGO_HOME COPY --from=cacher /tmp/ballista/target target -ARG RELEASE_FLAG=--release # force build.rs to run to generate configure_me code. ENV FORCE_REBUILD='true' From 05204c3baf36078ae67c12f3ec98652266cb5a24 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sat, 7 Aug 2021 21:15:41 -0700 Subject: [PATCH 17/22] cache buildkit cache --- .github/workflows/integration_tests.yml | 64 +++++++++++++++++++++---- dev/build-set-env.sh | 6 +-- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 8ee306e012d38..a902cd65421e0 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -26,18 +26,62 @@ jobs: - name: Set up Docker Buildx id: docker-builder uses: docker/setup-buildx-action@v1 - - name: Cache Buildx + + - name: Cache docker layer cache + uses: actions/cache@v2 + id: buildx-layer-cache + with: + path: "${{ runner.temp }}/buildx-cache/layer" + key: ${{ runner.os }}-buildx-layer-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} + restore-keys: | + ${{ runner.os }}-buildx-layer-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} + ${{ runner.os }}-buildx-layer- + + - name: Cache docker buildkit cache uses: actions/cache@v2 - id: buildx-cache + id: buildx-buildkit-cache with: - path: "${{ runner.temp }}/buildx-cache" - key: ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} + path: "${{ runner.temp }}/buildx-cache/buildkit" + key: ${{ runner.os }}-buildx-buildkit-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} - ${{ runner.os }}-buildx- - - run: | + ${{ runner.os }}-buildx-buildkit-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} + ${{ runner.os }}-buildx-buildkit- + + - name: Restore docker buildx cache + # Handle cache mount in buildx + # See https://github.com/docker/buildx/issues/156#issuecomment-537492942 + if: steps.buildx-buildkit-cache.outputs.cache-hit == 'true' + env: + BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} + run: | + export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0" + docker buildx stop --builder "${BUILDX_BUILDER}" + docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \ + -v ${{ runner.temp }}/buildx-cache/buildkit:/backup \ + alpine:3.10 /bin/sh -c "cd / && tar xf /backup/backup.tar.gz" + docker buildx inspect --builder "${BUILDX_BUILDER}" --bootstrap + docker buildx --builder "${BUILDX_BUILDER}" du --verbose + + - name: Run integration test + env: + BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} + BUILDX_LAYER_CACHE_DIR: "${{ runner.temp }}/buildx-cache/layer" + run: | ./dev/integration-tests.sh - docker buildx --builder "${BUILDX_BUILDER}" du + + - name: Backup docker buildx cache env: - BUILDX_BUILDER: ballista-docker-builder - BUILDX_CACHE_DIR: "${{ runner.temp }}/buildx-cache" + KEEP_STORAGE: "3221225472.0" # 3GB + BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} + run: | + export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0" + echo "Start pruning builder ${BUILDX_BUILDER_CONTAINER}" + docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=frontend + docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=regular + docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=source.local + docker buildx --builder "${BUILDX_BUILDER}" prune --force --keep-storage ${KEEP_STORAGE} + docker buildx --builder "${BUILDX_BUILDER}" du --verbose + + docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \ + -v ${{ runner.temp }}/buildx-cache/buildkit:/backup \ + alpine:3.10 /bin/sh -c "cd / && tar cf /backup/backup.tar.gz /var/lib/buildkit" diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh index d3af4c50cc21a..1a79f17fac0d2 100755 --- a/dev/build-set-env.sh +++ b/dev/build-set-env.sh @@ -22,14 +22,14 @@ export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista # If running within CI, use docker-container driver to cache build cache into # local folders so we can share them between runs. if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then - echo "building docker image in CI, saving build cache to local folder ${BUILDX_CACHE_DIR}." + echo "building docker image in CI, saving layer cache to local folder ${BUILDX_LAYER_CACHE_DIR}." BUILDX_BUILDER="${BUILDX_BUILDER:-ballista-docker-builder}" docker buildx inspect "${BUILDX_BUILDER}" &>/dev/null || \ docker buildx create --driver docker-container --name "${BUILDX_BUILDER}" --use BUILD_ARGS=(buildx build \ --builder ${BUILDX_BUILDER} \ - --cache-from="type=local,src=${BUILDX_CACHE_DIR}" \ - --cache-to="type=local,mode=max,dest=${BUILDX_CACHE_DIR}" \ + --cache-from="type=local,src=${BUILDX_LAYER_CACHE_DIR}" \ + --cache-to="type=local,mode=max,dest=${BUILDX_LAYER_CACHE_DIR}" \ --load) else echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=1" From a8c842b95c53d1eee32f9b4f2ca4c1f4b4eabd6a Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sat, 7 Aug 2021 22:19:41 -0700 Subject: [PATCH 18/22] rename ballista integration test job --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a902cd65421e0..72a6855f1beaf 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Integration tests +name: Ballista Integration Tests on: [push, pull_request] jobs: From 99d2f1abb4f113a99d8608f4981c54917a833e52 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sat, 7 Aug 2021 22:36:55 -0700 Subject: [PATCH 19/22] cancel duplicate ballista integration test runs --- ...ntegration_tests.yml => ballista_integration_tests.yml} | 0 .github/workflows/cancel.yml | 7 +++++++ 2 files changed, 7 insertions(+) rename .github/workflows/{integration_tests.yml => ballista_integration_tests.yml} (100%) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/ballista_integration_tests.yml similarity index 100% rename from .github/workflows/integration_tests.yml rename to .github/workflows/ballista_integration_tests.yml diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index e1c6ed98ae8b3..4d8c77c787069 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -44,3 +44,10 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} workflowFileName: rust.yml skipEventTypes: '["push", "schedule"]' + - uses: potiuk/cancel-workflow-runs@master + name: "Cancel stale Ballista integration runs" + with: + cancelMode: allDuplicates + token: ${{ secrets.GITHUB_TOKEN }} + workflowFileName: ballista_integration_tests.yml + skipEventTypes: '["push", "schedule"]' From 9469dacddbb7b5ca448985299b13f39dbbb16133 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 8 Aug 2021 18:16:38 -0700 Subject: [PATCH 20/22] fix local docker image build --- dev/build-set-env.sh | 4 ++-- dev/docker/ballista.dockerfile | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh index 1a79f17fac0d2..f068d29a9c76c 100755 --- a/dev/build-set-env.sh +++ b/dev/build-set-env.sh @@ -32,8 +32,8 @@ if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then --cache-to="type=local,mode=max,dest=${BUILDX_LAYER_CACHE_DIR}" \ --load) else - echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=1" - echo "To install docker buildx, follow https://github.com/docker/buildx#installing" export DOCKER_BUILDKIT=1 + echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=${DOCKER_BUILDKIT}" + echo "To install docker buildx, follow https://github.com/docker/buildx#installing" BUILD_ARGS=(build) fi diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 4833b67327087..74fbf2eb26974 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.3 + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information From c12ae642770fa5d63e4e24ff00481e0adb8e2a4b Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 8 Aug 2021 21:29:10 -0700 Subject: [PATCH 21/22] avoid only restore buildkit cache on exact cache hit --- .github/workflows/ballista_integration_tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ballista_integration_tests.yml b/.github/workflows/ballista_integration_tests.yml index 72a6855f1beaf..60456657a78de 100644 --- a/.github/workflows/ballista_integration_tests.yml +++ b/.github/workflows/ballista_integration_tests.yml @@ -50,10 +50,14 @@ jobs: - name: Restore docker buildx cache # Handle cache mount in buildx # See https://github.com/docker/buildx/issues/156#issuecomment-537492942 - if: steps.buildx-buildkit-cache.outputs.cache-hit == 'true' env: BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} run: | + if [[ ! -d ${{ runner.temp }}/buildx-cache/buildkit ]]; then + # this happens on the first run + echo "buildkit cache dir not found, skipping cache restore" + exit 0 + fi export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0" docker buildx stop --builder "${BUILDX_BUILDER}" docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \ From 1faec3a2d2ee6247c7a94312de2d8871fe2b8214 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 9 Aug 2021 18:43:13 -0700 Subject: [PATCH 22/22] remove redundant restore-keys in action --- .github/workflows/ballista_integration_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ballista_integration_tests.yml b/.github/workflows/ballista_integration_tests.yml index 60456657a78de..95f0b33689ed8 100644 --- a/.github/workflows/ballista_integration_tests.yml +++ b/.github/workflows/ballista_integration_tests.yml @@ -34,7 +34,6 @@ jobs: path: "${{ runner.temp }}/buildx-cache/layer" key: ${{ runner.os }}-buildx-layer-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx-layer-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} ${{ runner.os }}-buildx-layer- - name: Cache docker buildkit cache @@ -44,7 +43,6 @@ jobs: path: "${{ runner.temp }}/buildx-cache/buildkit" key: ${{ runner.os }}-buildx-buildkit-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx-buildkit-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} ${{ runner.os }}-buildx-buildkit- - name: Restore docker buildx cache