Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
937f3bc
run ballista integration test
houqp Jul 4, 2021
2fee15d
reduce openssl build verbosity
houqp Jul 6, 2021
331d62e
suppress all make command echos
houqp Jul 6, 2021
6e3e60b
suppress openssl install log & tpch build log
houqp Jul 6, 2021
4ea832e
build integration test binaries in debug mode to reduce build time
houqp Jul 6, 2021
deadb0d
do not print query result in integration test
houqp Jul 6, 2021
93a261d
Revert "build integration test binaries in debug mode to reduce build…
houqp Jul 6, 2021
a795f9b
make tpch data gen less verbose in CI
houqp Jul 10, 2021
bd25df0
Merge remote-tracking branch 'upstream/master' into qp_test
houqp Jul 19, 2021
1d9809e
test caching build cache with buildx docker-container driver
houqp Jul 19, 2021
e76da7e
Merge remote-tracking branch 'upstream/master' into qp_test
houqp Jul 21, 2021
633cb18
cache tpch gen docker image build
houqp Jul 21, 2021
f3539dd
use cargo.toml and dockerfile as cache hashkey
houqp Jul 22, 2021
686089e
merge base into main ballista dockerfile
houqp Jul 22, 2021
aa47596
enable sccache for appliation build
houqp Jul 25, 2021
5bc9779
Merge remote-tracking branch 'upstream/master' into qp_test
houqp Jul 25, 2021
4cb60fa
cache cargo registry download
houqp Jul 27, 2021
c48760a
copy cargo_home from cacher layer
houqp Aug 8, 2021
4e4cfd9
debug buildx builder cache
houqp Aug 8, 2021
05204c3
cache buildkit cache
houqp Aug 8, 2021
81749a8
Merge remote-tracking branch 'upstream/master' into qp_test
houqp Aug 8, 2021
a8c842b
rename ballista integration test job
houqp Aug 8, 2021
99d2f1a
cancel duplicate ballista integration test runs
houqp Aug 8, 2021
9469dac
fix local docker image build
houqp Aug 9, 2021
c12ae64
avoid only restore buildkit cache on exact cache hit
houqp Aug 9, 2021
f385758
Merge remote-tracking branch 'upstream/master' into qp_test
houqp Aug 10, 2021
1faec3a
remove redundant restore-keys in action
houqp Aug 10, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions .github/workflows/ballista_integration_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Ballista Integration Tests
on: [push, pull_request]

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Docker Buildx
id: docker-builder
uses: docker/setup-buildx-action@v1

- name: Cache docker layer cache
uses: actions/cache@v2
id: buildx-layer-cache
with:
path: "${{ runner.temp }}/buildx-cache/layer"
key: ${{ runner.os }}-buildx-layer-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }}
restore-keys: |
${{ runner.os }}-buildx-layer-

- name: Cache docker buildkit cache
uses: actions/cache@v2
id: buildx-buildkit-cache
with:
path: "${{ runner.temp }}/buildx-cache/buildkit"
key: ${{ runner.os }}-buildx-buildkit-${{ hashFiles('**/Cargo.toml', '**/*.dockerfile') }}
restore-keys: |
${{ runner.os }}-buildx-buildkit-

- name: Restore docker buildx cache
# Handle cache mount in buildx
# See https://github.com/docker/buildx/issues/156#issuecomment-537492942
env:
BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }}
run: |
if [[ ! -d ${{ runner.temp }}/buildx-cache/buildkit ]]; then
# this happens on the first run
echo "buildkit cache dir not found, skipping cache restore"
exit 0
fi
export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0"
docker buildx stop --builder "${BUILDX_BUILDER}"
docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \
-v ${{ runner.temp }}/buildx-cache/buildkit:/backup \
alpine:3.10 /bin/sh -c "cd / && tar xf /backup/backup.tar.gz"
docker buildx inspect --builder "${BUILDX_BUILDER}" --bootstrap
docker buildx --builder "${BUILDX_BUILDER}" du --verbose

- name: Run integration test
env:
BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }}
BUILDX_LAYER_CACHE_DIR: "${{ runner.temp }}/buildx-cache/layer"
run: |
./dev/integration-tests.sh

- name: Backup docker buildx cache
env:
KEEP_STORAGE: "3221225472.0" # 3GB
BUILDX_BUILDER: ${{ steps.docker-builder.outputs.name }}
run: |
export BUILDX_BUILDER_CONTAINER="buildx_buildkit_${BUILDX_BUILDER}0"
echo "Start pruning builder ${BUILDX_BUILDER_CONTAINER}"
docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=frontend
docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=regular
docker buildx --builder "${BUILDX_BUILDER}" prune --force --filter type=source.local
docker buildx --builder "${BUILDX_BUILDER}" prune --force --keep-storage ${KEEP_STORAGE}
docker buildx --builder "${BUILDX_BUILDER}" du --verbose

docker run --rm --volumes-from "${BUILDX_BUILDER_CONTAINER}" \
-v ${{ runner.temp }}/buildx-cache/buildkit:/backup \
alpine:3.10 /bin/sh -c "cd / && tar cf /backup/backup.tar.gz /var/lib/buildkit"
7 changes: 7 additions & 0 deletions .github/workflows/cancel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,10 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
workflowFileName: rust.yml
skipEventTypes: '["push", "schedule"]'
- uses: potiuk/cancel-workflow-runs@master
name: "Cancel stale Ballista integration runs"
with:
cancelMode: allDuplicates
token: ${{ secrets.GITHUB_TOKEN }}
workflowFileName: ballista_integration_tests.yml
skipEventTypes: '["push", "schedule"]'
10 changes: 8 additions & 2 deletions benchmarks/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,11 @@

set -e
cd /tpch-dbgen
./dbgen -vf -s 1
mv *.tbl /data
if [[ $- == *i* ]]; then
./dbgen -vf -s 1
else
# do not run in verbose mode when running within an non-interactive
# terminal, for example Github Action job
./dbgen -f -s 1
fi
mv *.tbl /data
2 changes: 1 addition & 1 deletion benchmarks/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,5 @@ set -e
cd /
for query in 1 3 5 6 7 10 12
do
/tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1 --debug
/tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1
done
8 changes: 4 additions & 4 deletions benchmarks/tpch-gen.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,19 @@
# specific language governing permissions and limitations
# under the License.

#set -e
set -e

pushd ..
. ./dev/build-set-env.sh
popd
docker build -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile .
docker ${BUILD_ARGS[@]} -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile .

# Generate data into the ./data directory if it does not already exist
FILE=./data/supplier.tbl
if test -f "$FILE"; then
echo "$FILE exists."
else
mkdir data 2>/dev/null
docker run -v `pwd`/data:/data -it --rm ballista-tpchgen:$BALLISTA_VERSION
docker run -v `pwd`/data:/data -t --rm ballista-tpchgen:$BALLISTA_VERSION
ls -l data
fi
fi
2 changes: 1 addition & 1 deletion benchmarks/tpchgen.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN apt-get update && \

RUN git clone https://github.com/databricks/tpch-dbgen.git && \
cd tpch-dbgen && \
make
make -s

WORKDIR /tpch-dbgen
ADD entrypoint.sh /tpch-dbgen/
Expand Down
4 changes: 2 additions & 2 deletions dev/build-ballista-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@
set -e

. ./dev/build-set-env.sh
docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile .
docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .

docker ${BUILD_ARGS[@]} -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
19 changes: 19 additions & 0 deletions dev/build-set-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,22 @@
# under the License.

export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista/rust/core/Cargo.toml)

# If running within CI, use docker-container driver to cache build cache into
# local folders so we can share them between runs.
if [[ "${CI}" = "true" ]] && docker buildx &>/dev/null; then
echo "building docker image in CI, saving layer cache to local folder ${BUILDX_LAYER_CACHE_DIR}."
BUILDX_BUILDER="${BUILDX_BUILDER:-ballista-docker-builder}"
docker buildx inspect "${BUILDX_BUILDER}" &>/dev/null || \
docker buildx create --driver docker-container --name "${BUILDX_BUILDER}" --use
BUILD_ARGS=(buildx build \
--builder ${BUILDX_BUILDER} \
--cache-from="type=local,src=${BUILDX_LAYER_CACHE_DIR}" \
--cache-to="type=local,mode=max,dest=${BUILDX_LAYER_CACHE_DIR}" \
--load)
else
export DOCKER_BUILDKIT=1
echo "No docker buildx plugin found, fallback to default docker build command with DOCKER_BUILDKIT=${DOCKER_BUILDKIT}"
echo "To install docker buildx, follow https://github.com/docker/buildx#installing"
BUILD_ARGS=(build)
fi
99 changes: 0 additions & 99 deletions dev/docker/ballista-base.dockerfile

This file was deleted.

Loading