diff --git a/.github/workflows/ballista_integration_tests.yml b/.github/workflows/ballista_integration_tests.yml new file mode 100644 index 0000000000000..95f0b33689ed8 --- /dev/null +++ b/.github/workflows/ballista_integration_tests.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Ballista Integration Tests +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: docker-builder + uses: docker/setup-buildx-action@v1 + + - name: Cache docker layer cache + uses: actions/cache@v2 + id: buildx-layer-cache + with: + path: "${{ runner.temp }}/buildx-cache/layer" + key: ${{ runner.os }}-buildx-layer-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} + restore-keys: | + ${{ runner.os }}-buildx-layer- + + - name: Cache docker buildkit cache + uses: actions/cache@v2 + id: buildx-buildkit-cache + with: + path: "${{ runner.temp }}/buildx-cache/buildkit" + key: ${{ runner.os }}-buildx-buildkit-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }} + restore-keys: | + ${{ runner.os }}-buildx-buildkit- + + - name: Restore docker buildx cache + # Handle cache mount in buildx + # See https://github.com/docker/buildx/issues/156#issuecomment-537492942 + env: + BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} + run: | + if [[ ! -d ${{ runner.temp }}/buildx-cache/buildkit ]]; then + # this happens on the first run + echo "buildkit cache dir not found, skipping cache restore" + exit 0 + fi + export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0" + docker buildx stop --builder "${BUILDX_BUILDER}" + docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \ + -v ${{ runner.temp }}/buildx-cache/buildkit:/backup \ + alpine:3.10 /bin/sh -c "cd / && tar xf /backup/backup.tar.gz" + docker buildx inspect --builder "${BUILDX_BUILDER}" --bootstrap + docker buildx --builder "${BUILDX_BUILDER}" du --verbose + + - name: Run integration test + env: + BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} + BUILDX_LAYER_CACHE_DIR: "${{ runner.temp }}/buildx-cache/layer" + run: | + ./dev/integration-tests.sh + + - name: Backup docker buildx cache + env: + KEEP_STORAGE: "3221225472.0" # 3GB + BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }} + run: | + export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0" + echo "Start pruning builder ${BUILDX_BUILDER_CONTAINER}" + docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=frontend + docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=regular + docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=source.local + docker buildx --builder "${BUILDX_BUILDER}" prune --force --keep-storage ${KEEP_STORAGE} + docker buildx --builder "${BUILDX_BUILDER}" du --verbose + + docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \ + -v ${{ runner.temp }}/buildx-cache/buildkit:/backup \ + alpine:3.10 /bin/sh -c "cd / && tar cf /backup/backup.tar.gz /var/lib/buildkit" diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index e1c6ed98ae8b3..4d8c77c787069 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -44,3 +44,10 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} workflowFileName: rust.yml skipEventTypes: '["push", "schedule"]' + - uses: potiuk/cancel-workflow-runs@master + name: "Cancel stale Ballista integration runs" + with: + cancelMode: allDuplicates + token: ${{ secrets.GITHUB_TOKEN }} + workflowFileName: ballista_integration_tests.yml + skipEventTypes: '["push", "schedule"]' diff --git a/benchmarks/entrypoint.sh b/benchmarks/entrypoint.sh index 71c04324afd14..19c1ee769d0ed 100755 --- a/benchmarks/entrypoint.sh +++ b/benchmarks/entrypoint.sh @@ -18,5 +18,11 @@ set -e cd /tpch-dbgen -./dbgen -vf -s 1 -mv *.tbl /data \ No newline at end of file +if [[ $- == *i* ]]; then + ./dbgen -vf -s 1 +else + # do not run in verbose mode when running within an non-interactive + # terminal, for example Github Action job + ./dbgen -f -s 1 +fi +mv *.tbl /data diff --git a/benchmarks/run.sh b/benchmarks/run.sh index b1f47a24c2d8d..b4cbf9daea9f5 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -22,5 +22,5 @@ set -e cd / for query in 1 3 5 6 7 10 12 do - /tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1 --debug + /tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1 done diff --git a/benchmarks/tpch-gen.sh b/benchmarks/tpch-gen.sh index a46f65111069e..c5aec143083d7 100755 --- a/benchmarks/tpch-gen.sh +++ b/benchmarks/tpch-gen.sh @@ -16,12 +16,12 @@ # specific language governing permissions and limitations # under the License. -#set -e +set -e pushd .. . ./dev/build-set-env.sh popd -docker build -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile . +docker ${BUILD_ARGS[@]} -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile . # Generate data into the ./data directory if it does not already exist FILE=./data/supplier.tbl @@ -29,6 +29,6 @@ if test -f "$FILE"; then echo "$FILE exists." else mkdir data 2>/dev/null - docker run -v `pwd`/data:/data -it --rm ballista-tpchgen:$BALLISTA_VERSION + docker run -v `pwd`/data:/data -t --rm ballista-tpchgen:$BALLISTA_VERSION ls -l data -fi \ No newline at end of file +fi diff --git a/benchmarks/tpchgen.dockerfile b/benchmarks/tpchgen.dockerfile index 69434708bf9a7..e333297a6790f 100644 --- a/benchmarks/tpchgen.dockerfile +++ b/benchmarks/tpchgen.dockerfile @@ -22,7 +22,7 @@ RUN apt-get update && \ RUN git clone https://github.com/databricks/tpch-dbgen.git && \ cd tpch-dbgen && \ - make + make -s WORKDIR /tpch-dbgen ADD entrypoint.sh /tpch-dbgen/ diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh index bc028da9e716a..cd3b1be4b2ebe 100755 --- a/dev/build-ballista-docker.sh +++ b/dev/build-ballista-docker.sh @@ -20,5 +20,5 @@ set -e . ./dev/build-set-env.sh -docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . -docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . + +docker ${BUILD_ARGS[@]} -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh index 3eb29e7ce1443..f068d29a9c76c 100755 --- a/dev/build-set-env.sh +++ b/dev/build-set-env.sh @@ -18,3 +18,22 @@ # under the License. export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista/rust/core/Cargo.toml) + +# If running within CI, use docker-container driver to cache build cache into +# local folders so we can share them between runs. +if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then + echo "building docker image in CI, saving layer cache to local folder ${BUILDX_LAYER_CACHE_DIR}." + BUILDX_BUILDER="${BUILDX_BUILDER:-ballista-docker-builder}" + docker buildx inspect "${BUILDX_BUILDER}" &>/dev/null || \ + docker buildx create --driver docker-container --name "${BUILDX_BUILDER}" --use + BUILD_ARGS=(buildx build \ + --builder ${BUILDX_BUILDER} \ + --cache-from="type=local,src=${BUILDX_LAYER_CACHE_DIR}" \ + --cache-to="type=local,mode=max,dest=${BUILDX_LAYER_CACHE_DIR}" \ + --load) +else + export DOCKER_BUILDKIT=1 + echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=${DOCKER_BUILDKIT}" + echo "To install docker buildx, follow https://github.com/docker/buildx#installing" + BUILD_ARGS=(build) +fi diff --git a/dev/docker/ballista-base.dockerfile b/dev/docker/ballista-base.dockerfile deleted file mode 100644 index 4dc7a065e6907..0000000000000 --- a/dev/docker/ballista-base.dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Turn .dockerignore to .dockerallow by excluding everything and explicitly -# allowing specific files and directories. This enables us to quickly add -# dependency files to the docker content without scanning the whole directory. -# This setup requires to all of our docker containers have arrow's source -# as a mounted directory. - - -# Base image extends debian:buster-slim -FROM rust:1.53.0-buster AS builder - -RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl - -#NOTE: the following was copied from https://github.com/emk/rust-musl-builder/blob/master/Dockerfile under Apache 2.0 license - -# The OpenSSL version to use. We parameterize this because many Rust -# projects will fail to build with 1.1. -#ARG OPENSSL_VERSION=1.0.2r -ARG OPENSSL_VERSION=1.1.1b - -# Build a static library version of OpenSSL using musl-libc. This is needed by -# the popular Rust `hyper` crate. -# -# We point /usr/local/musl/include/linux at some Linux kernel headers (not -# necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine" -# component. It's possible that this will cause bizarre and terrible things to -# happen. There may be "sanitized" header -RUN echo "Building OpenSSL" && \ - ls /usr/include/linux && \ - mkdir -p /usr/local/musl/include && \ - ln -s /usr/include/linux /usr/local/musl/include/linux && \ - ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ - ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ - cd /tmp && \ - curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ - tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ - env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ - env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ - env C_INCLUDE_PATH=/usr/local/musl/include/ make && \ - make install && \ - rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ - rm -r /tmp/* - -RUN echo "Building zlib" && \ - cd /tmp && \ - ZLIB_VERSION=1.2.11 && \ - curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ - tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ - CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \ - make && make install && \ - rm -r /tmp/* - -RUN echo "Building libpq" && \ - cd /tmp && \ - POSTGRESQL_VERSION=11.2 && \ - curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ - tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ - CC=musl-gcc CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ - cd src/interfaces/libpq && make all-static-lib && make install-lib-static && \ - cd ../../bin/pg_config && make && make install && \ - rm -r /tmp/* - -ENV OPENSSL_DIR=/usr/local/musl/ \ - OPENSSL_INCLUDE_DIR=/usr/local/musl/include/ \ - DEP_OPENSSL_INCLUDE=/usr/local/musl/include/ \ - OPENSSL_LIB_DIR=/usr/local/musl/lib/ \ - OPENSSL_STATIC=1 \ - PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=1 \ - PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - PKG_CONFIG_ALLOW_CROSS=true \ - PKG_CONFIG_ALL_STATIC=true \ - LIBZ_SYS_STATIC=1 \ - TARGET=musl - -# The content copied mentioned in the NOTE above ends here. - -## Download the target for static linking. -RUN rustup target add x86_64-unknown-linux-musl -RUN cargo install cargo-build-deps - -# prepare toolchain -RUN rustup update && \ - rustup component add rustfmt \ No newline at end of file diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index 11b788e7a2b1a..74fbf2eb26974 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.3 + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,11 +23,118 @@ # This setup requires to all of our docker containers have arrow's source # as a mounted directory. -ARG RELEASE_FLAG=--release -FROM ballista-base:0.5.0-SNAPSHOT AS base +# Base image extends debian:buster-slim +FROM rust:1.53.0-buster AS base + +RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl + +ARG SCCACHE_TAR_URL=https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz + +RUN curl -LsSf ${SCCACHE_TAR_URL} > /tmp/sccache.tar.gz && \ + tar axvf /tmp/sccache.tar.gz --strip-components=1 -C /usr/local/bin --wildcards --no-anchored 'sccache' && \ + chmod +x /usr/local/bin/sccache && \ + sccache --version && \ + rm -rf /tmp/sccache.tar.gz + +ENV RUSTC_WRAPPER=/usr/local/bin/sccache +ENV CARGO_INCREMENTAL=0 +ENV SCCACHE_CACHE_SIZE=2G + +#NOTE: the following was copied from https://github.com/emk/rust-musl-builder/blob/master/Dockerfile under Apache 2.0 license + +# The OpenSSL version to use. We parameterize this because many Rust +# projects will fail to build with 1.1. +#ARG OPENSSL_VERSION=1.0.2r +ARG OPENSSL_VERSION=1.1.1b + +# Build a static library version of OpenSSL using musl-libc. This is needed by +# the popular Rust `hyper` crate. +# +# We point /usr/local/musl/include/linux at some Linux kernel headers (not +# necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine" +# component. It's possible that this will cause bizarre and terrible things to +# happen. There may be "sanitized" header +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building OpenSSL" && \ + mkdir -p /usr/local/musl/include && \ + ln -s /usr/include/linux /usr/local/musl/include/linux && \ + ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ + ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ + cd /tmp && \ + curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ + tar xzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ + env CC="sccache musl-gcc" ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make -s depend && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make -s && \ + make -s install 1>/dev/null && \ + rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ + rm -r /tmp/* + +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building zlib" && \ + cd /tmp && \ + ZLIB_VERSION=1.2.11 && \ + curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ + tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ + CC="sccache musl-gcc" ./configure --static --prefix=/usr/local/musl && \ + make -s && make -s install 1>/dev/null && \ + rm -r /tmp/* + +RUN --mount=type=cache,target=/root/.cache/sccache \ + echo "Building libpq" && \ + cd /tmp && \ + POSTGRESQL_VERSION=11.2 && \ + curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ + tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ + CC="sccache musl-gcc" CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ + cd src/interfaces/libpq && make -s all-static-lib && make -s install-lib-static && \ + cd ../../bin/pg_config && make -s && make -s install && \ + rm -r /tmp/* + +ENV OPENSSL_DIR=/usr/local/musl/ \ + OPENSSL_INCLUDE_DIR=/usr/local/musl/include/ \ + DEP_OPENSSL_INCLUDE=/usr/local/musl/include/ \ + OPENSSL_LIB_DIR=/usr/local/musl/lib/ \ + OPENSSL_STATIC=1 \ + PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=1 \ + PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ + PKG_CONFIG_ALLOW_CROSS=true \ + PKG_CONFIG_ALL_STATIC=true \ + LIBZ_SYS_STATIC=1 \ + TARGET=musl + +# The content copied mentioned in the NOTE above ends here. + +## Download the target for static linking. +RUN rustup target add x86_64-unknown-linux-musl +RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + cargo install cargo-build-deps + +# prepare toolchain +RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + rustup update && \ + rustup component add rustfmt + WORKDIR /tmp/ballista -RUN apt-get -y install cmake -RUN cargo install cargo-chef +RUN apt-get -y install cmake \ + && apt-get clean \ + && apt-get autoremove \ + && rm -rf /var/cache/apt/archives/* \ + && rm -rf /var/lib/apt/lists/* +RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + cargo install cargo-chef + + +ARG RELEASE_FLAG=--release FROM base as planner RUN mkdir /tmp/ballista/ballista @@ -44,7 +153,11 @@ RUN cargo chef prepare --recipe-path recipe.json FROM base as cacher COPY --from=planner /tmp/ballista/recipe.json recipe.json -RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json +RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder RUN mkdir /tmp/ballista/ballista @@ -60,27 +173,31 @@ COPY benchmarks ./benchmarks/ COPY datafusion ./datafusion/ COPY datafusion-cli ./datafusion-cli/ COPY datafusion-examples ./datafusion-examples/ +COPY --from=cacher $CARGO_HOME $CARGO_HOME COPY --from=cacher /tmp/ballista/target target -ARG RELEASE_FLAG=--release # force build.rs to run to generate configure_me code. ENV FORCE_REBUILD='true' -RUN cargo build $RELEASE_FLAG - -# put the executor on /executor (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi - -# put the scheduler on /scheduler (need to be copied from different places depending on FLAG) -ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi +RUN --mount=type=cache,target=/root/.cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + cargo build $RELEASE_FLAG -# put the tpch on /tpch (need to be copied from different places depending on FLAG) +# copy the binaries into / (need to be copied from different places depending on RELEASE_FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi +RUN if [ -z "$RELEASE_FLAG" ]; then \ + mv /tmp/ballista/target/debug/ballista-executor /executor; \ + mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; \ + mv /tmp/ballista/target/debug/tpch /tpch; \ + else \ + mv /tmp/ballista/target/release/ballista-executor /executor; \ + mv /tmp/ballista/target/release/ballista-scheduler /scheduler; \ + mv /tmp/ballista/target/release/tpch /tpch; \ + fi # Copy the binary into a new container for a smaller docker image -FROM ballista-base:0.5.0-SNAPSHOT +FROM base COPY --from=builder /executor / diff --git a/dev/integration-tests.sh b/dev/integration-tests.sh index 3c1306319c369..a995f6d6bd6aa 100755 --- a/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -17,6 +17,7 @@ # specific language governing permissions and limitations # under the License. set -e + ./dev/build-ballista-docker.sh pushd benchmarks ./tpch-gen.sh