From 1da00e68e1b9de0b1b156177c6789094e07cd578 Mon Sep 17 00:00:00 2001 From: James Troup Date: Fri, 23 Oct 2026 01:13:40 +0100 Subject: [PATCH 1/4] chore(ci): build and publish the Stacklet cubejs/cube image --- .github/CODEOWNERS | 1 + .github/workflows/stacklet-image.yml | 332 +++++++++++++++++++++++ .github/workflows/stacklet-lint.yml | 51 ++++ .pre-commit-config.yaml | 18 ++ mise.toml | 2 + packages/cubejs-docker/DEVELOPMENT.md | 42 ++- packages/cubejs-docker/latest.Dockerfile | 4 +- scripts/build-docker.sh | 75 +++-- scripts/stacklet-overlay-guard.sh | 90 ++++++ 9 files changed, 574 insertions(+), 41 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/stacklet-image.yml create mode 100644 .github/workflows/stacklet-lint.yml create mode 100644 .pre-commit-config.yaml create mode 100644 mise.toml create mode 100755 scripts/stacklet-overlay-guard.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000000..6931f70f7a8c4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stacklet/engineering diff --git a/.github/workflows/stacklet-image.yml b/.github/workflows/stacklet-image.yml new file mode 100644 index 0000000000000..b665991529e31 --- /dev/null +++ b/.github/workflows/stacklet-image.yml @@ -0,0 +1,332 @@ +# Build the Stacklet cubejs/cube image with scripts/build-docker.sh and publish +# it. Registry and role come from repo Actions variables. +# +# Tags: v-stacklet- for pushes, and +# v-stacklet-pr- for same-repo PRs, where comes +# from packages/cubejs-docker/package.json and is the commit built. +# +# Only the build job runs third-party code (yarn install, docker build), so it +# gets no cloud credentials and no OIDC token. It hands the image to the +# publish job as an artifact. +name: Stacklet Image + +on: + push: + branches: + - stacklet/integration + - release/** + pull_request: + branches: + - stacklet/integration + - release/** + +permissions: {} + +# PR runs supersede each other. Push runs are grouped by commit alone: GitHub +# drops a pending run whenever a newer one joins its group, so a per-branch +# group would lose images. The same commit pushed to two branches (a release +# branch cut from integration) queues instead of racing to the immutable tag. +concurrency: + group: stacklet-image-${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || format('push-{0}', github.sha) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + overlay-guard: + name: Overlay guard + runs-on: ubuntu-24.04 + # Fork PRs don't run at all. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: read + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + use-policy-store: true + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: .nvmrc + package-manager-cache: false + + - name: Check changed sources reach the image + run: ./scripts/stacklet-overlay-guard.sh + + # The other jobs skip themselves without the role, which would leave a + # push to a publishing branch green with no image. + - name: Check publish configuration + if: github.event_name == 'push' + env: + PUBLISH_ROLE_ARN: ${{ vars.PUBLISH_ROLE_ARN }} + run: | + if [ -z "$PUBLISH_ROLE_ARN" ]; then + echo "ERROR: repo variable PUBLISH_ROLE_ARN is not set; nothing would be published" + exit 1 + fi + + resolve: + name: Resolve image + needs: overlay-guard + runs-on: ubuntu-24.04 + if: vars.PUBLISH_ROLE_ARN != '' + permissions: + contents: read + id-token: write + outputs: + sha: ${{ steps.meta.outputs.sha }} + tag: ${{ steps.meta.outputs.tag }} + image: ${{ steps.meta.outputs.image }} + exists: ${{ steps.exists.outputs.exists }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + use-policy-store: true + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: packages/cubejs-docker/package.json + sparse-checkout-cone-mode: false + + - name: Resolve image tag + id: meta + env: + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REGISTRY: ${{ vars.ECR_REGISTRY }} + REPOSITORY: ${{ vars.ECR_REPOSITORY }} + run: | + version="v$(jq -r .version packages/cubejs-docker/package.json)" + tag="${version}-stacklet-${PR_NUMBER:+pr${PR_NUMBER}-}${SHA:0:9}" + { + echo "sha=$SHA" + echo "tag=$tag" + echo "image=${REGISTRY}/${REPOSITORY}:${tag}" + } >> "$GITHUB_OUTPUT" + echo "Image: ${REGISTRY}/${REPOSITORY}:${tag}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.PUBLISH_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + + # Immutable registry: an existing tag can't be re-pushed, so skip the + # build. Publish still checks the existing tag's manifest. + - name: Check whether image already exists + id: exists + env: + TAG: ${{ steps.meta.outputs.tag }} + REPOSITORY: ${{ vars.ECR_REPOSITORY }} + run: | + if aws ecr describe-images \ + --repository-name "$REPOSITORY" \ + --image-ids imageTag="$TAG" >/dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "Image $TAG already exists; skipping build." + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + build: + name: Build + needs: resolve + runs-on: ubuntu-24.04 + if: needs.resolve.outputs.exists == 'false' + permissions: + contents: read + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + use-policy-store: true + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: .nvmrc + # No dependency cache in a publishing workflow (cache poisoning). + package-manager-cache: false + + - name: Install dependencies + env: + CUBESTORE_SKIP_POST_INSTALL: "true" + run: yarn install --frozen-lockfile + + # Single-platform image with no attestations, so ECR stores a plain + # image manifest, as for the upstream mirror tags. + - name: Build image + env: + IMAGE: ${{ needs.resolve.outputs.image }} + TAG: ${{ needs.resolve.outputs.tag }} + SHA: ${{ needs.resolve.outputs.sha }} + run: | + ./scripts/build-docker.sh "$IMAGE" \ + --platform linux/amd64 \ + --provenance=false \ + --sbom=false \ + --build-arg "IMAGE_VERSION=$TAG" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=$SHA" + + # /livez only checks data sources that are already connected, so it + # answers without a database; /readyz would need one. + - name: Smoke test + env: + IMAGE: ${{ needs.resolve.outputs.image }} + run: | + docker run -d --name cube-smoke \ + -e CUBEJS_API_SECRET=smoke-test \ + -e CUBEJS_DB_TYPE=postgres \ + -e CUBEJS_CACHE_AND_QUEUE_DRIVER=memory \ + "$IMAGE" + probe="require('http').get('http://localhost:4000/livez',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))" + ok=false + for _ in $(seq 1 60); do + if docker exec cube-smoke node -e "$probe" 2>/dev/null; then + ok=true + break + fi + sleep 2 + done + docker logs cube-smoke 2>&1 | tail -50 + docker rm -f cube-smoke >/dev/null + if [ "$ok" != true ]; then + echo "ERROR: /livez did not return 200 within 120s" + exit 1 + fi + echo "Smoke test OK" + + - name: Save image + shell: bash + env: + IMAGE: ${{ needs.resolve.outputs.image }} + run: | + set -o pipefail + docker save "$IMAGE" | gzip -1 > "$RUNNER_TEMP/image.tar.gz" + + - name: Upload image + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: stacklet-image + path: ${{ runner.temp }}/image.tar.gz + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + # If this job fails, use "Re-run all jobs", not "Re-run failed jobs". A + # failed-jobs re-run keeps build's earlier result, so it pushes again and the + # immutable registry rejects the tag. A full re-run finds the tag in resolve, + # skips build, and only checks the manifest here. + publish: + name: Publish + needs: [resolve, build] + runs-on: ubuntu-24.04 + # Also runs when build skipped because the tag exists, to check that tag. + if: >- + !cancelled() && + needs.resolve.result == 'success' && + (needs.build.result == 'success' || needs.build.result == 'skipped') + permissions: + id-token: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + use-policy-store: true + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Download image + if: needs.build.result == 'success' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: stacklet-image + path: ${{ runner.temp }} + + - name: Load image + if: needs.build.result == 'success' + shell: bash + env: + IMAGE: ${{ needs.resolve.outputs.image }} + run: | + set -o pipefail + gunzip -c "$RUNNER_TEMP/image.tar.gz" | docker load + # A tag pushed in error can't be replaced, so check before pushing. + # Only the containerd image store reports a descriptor; the classic + # store always pushes a single image manifest. + media_type="$(docker image inspect --format '{{if .Descriptor}}{{.Descriptor.MediaType}}{{end}}' "$IMAGE")" + case "$media_type" in + *manifest.list* | *image.index*) + echo "ERROR: $IMAGE is a multi-platform index ($media_type), not a single image" + exit 1 + ;; + esac + echo "$IMAGE: ${media_type:-classic image store}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.PUBLISH_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Push image + if: needs.build.result == 'success' + shell: bash + env: + IMAGE: ${{ needs.resolve.outputs.image }} + REGISTRY: ${{ vars.ECR_REGISTRY }} + run: | + set -o pipefail + aws ecr get-login-password | docker login --username AWS --password-stdin "$REGISTRY" + docker push "$IMAGE" + + - name: Check published manifest + env: + TAG: ${{ needs.resolve.outputs.tag }} + REPOSITORY: ${{ vars.ECR_REPOSITORY }} + run: | + media_type="$(aws ecr describe-images \ + --repository-name "$REPOSITORY" \ + --image-ids imageTag="$TAG" \ + --query 'imageDetails[0].imageManifestMediaType' \ + --output text)" + echo "$TAG: $media_type" + case "$media_type" in + application/vnd.docker.distribution.manifest.v2+json | application/vnd.oci.image.manifest.v1+json) ;; + *) + echo "ERROR: expected a single-platform image manifest, got '$media_type'" + exit 1 + ;; + esac + + - name: Summary + env: + IMAGE: ${{ needs.resolve.outputs.image }} + BUILT: ${{ needs.build.result }} + run: | + if [ "$BUILT" = success ]; then + echo "Published \`$IMAGE\`" >> "$GITHUB_STEP_SUMMARY" + else + echo "Image already existed, not rebuilt: \`$IMAGE\`" >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/stacklet-lint.yml b/.github/workflows/stacklet-lint.yml new file mode 100644 index 0000000000000..e7be83573eb96 --- /dev/null +++ b/.github/workflows/stacklet-lint.yml @@ -0,0 +1,51 @@ +# Run the pre-commit hooks (actionlint, zizmor, shellcheck) over the +# Stacklet-owned workflows and scripts. +name: Stacklet Lint + +on: + push: + branches: + - stacklet/integration + - release/** + pull_request: + branches: + - stacklet/integration + - release/** + +permissions: {} + +concurrency: + group: stacklet-lint-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint + runs-on: ubuntu-24.04 + # Fork PRs don't run at all. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: read + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + use-policy-store: true + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # No tool cache, so a PR run can't seed tools that a later push run uses. + - name: Install mise + uses: jdx/mise-action@9149ea85001c7435d5a66bb127d6a1b6227cb0a5 # v5.0.0 + with: + minimum_release_age: 7d + cache: false + + - name: Lint + run: pre-commit run --all-files --show-diff-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..8034be47ca2be --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +# Lint the Stacklet-owned files only. Upstream cube-js/cube files stay as +# upstream has them, so merges of upstream release tags don't conflict. +files: ^(\.github/workflows/stacklet-.*\.ya?ml|scripts/(build-docker|stacklet-.*)\.sh)$ +repos: + - repo: https://github.com/rhysd/actionlint + rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 + hooks: + - id: actionlint + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: 451b56af716f9f0d0c2b816503a3fd0cf8b036fa # frozen: v1.29.0 + hooks: + - id: zizmor + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: 745eface02aef23e168a8afb6b5737818efbea95 # frozen: v0.11.0.1 + hooks: + - id: shellcheck diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000000000..b4d67bb2b8779 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +pre-commit = "4.6.0" diff --git a/packages/cubejs-docker/DEVELOPMENT.md b/packages/cubejs-docker/DEVELOPMENT.md index 745e330ad44c7..65d25e32e6205 100644 --- a/packages/cubejs-docker/DEVELOPMENT.md +++ b/packages/cubejs-docker/DEVELOPMENT.md @@ -27,36 +27,32 @@ docker buildx build --platform linux/amd64 -t cubejs/cube:dev -f dev.Dockerfile ## Stacklet builds -The Stacklet fork patches certain Cube packages at the TypeScript source level. -Because `latest.Dockerfile` installs packages from npm (not from the local -workspace), a wrapper script is required to compile modified packages and stage -their compiled output before `docker build` runs. - -**Always use the build script — never `docker build` directly:** +The Stacklet fork patches some Cube packages at the TypeScript source level. +`latest.Dockerfile` installs packages from npm, not from the local workspace, +so build the image with the wrapper script rather than `docker build`: ```sh -# From the repo root: -./scripts/build-docker.sh cubejs/cube:v1.6.6-stacklet-$(git rev-parse --short HEAD) +# From the repo root, after `yarn install`: +./scripts/build-docker.sh cubejs/cube:dev ``` -The script: -1. Runs `yarn workspace @cubejs-backend/server-core build` (and any other - patched packages) -2. Copies compiled `dist/` output into `stacklet-patches/-dist/` -3. Runs `docker build`; the Dockerfile overlays those dist files on the - npm-installed versions -4. Cleans up the staged files on exit +The script compiles the workspace (`yarn tsc`), copies the `dist/` of each +package in its `PATCHED_PACKAGES` list into `stacklet-patches/-dist/`, +and runs `docker build`. The Dockerfile overlays every `*-dist/` directory it +finds there onto the npm-installed package. Extra arguments after the tag are +passed to `docker build`. ### Adding a new patched package -1. Make source changes and compile: `yarn workspace @cubejs-backend/ build` -2. Add a compile + stage block to `scripts/build-docker.sh`: - ```bash - mkdir -p "$PATCHES_DIR/-dist" - cp -r "$REPO_ROOT/packages/cubejs-/dist/." "$PATCHES_DIR/-dist/" - ``` -3. No Dockerfile change is needed — `latest.Dockerfile` automatically applies - any `*-dist/` directories it finds in `stacklet-patches/`. +Add the package's `@cubejs-backend/` name, without the scope, to +`PATCHED_PACKAGES` in `scripts/build-docker.sh`. No Dockerfile change is +needed. + +`scripts/stacklet-overlay-guard.sh` fails on any change under `packages/` or +`rust/` that would not reach the image. It allows Markdown, tests, +`packages/cubejs-docker/`, and the TypeScript sources of packages in +`PATCHED_PACKAGES`. Rust ships as prebuilt binaries, so Rust changes always +fail. ### Verifying a patch is in the image diff --git a/packages/cubejs-docker/latest.Dockerfile b/packages/cubejs-docker/latest.Dockerfile index 8802e49c99334..f2770bbceb7ae 100644 --- a/packages/cubejs-docker/latest.Dockerfile +++ b/packages/cubejs-docker/latest.Dockerfile @@ -38,7 +38,9 @@ RUN yarn install --prod \ COPY stacklet-patches/ /cube/stacklet-patches/ RUN find /cube/stacklet-patches -maxdepth 1 -name '*-dist' -type d | while read d; do \ pkg=$(basename "$d" -dist); \ - cp -r "$d/." "/cube/node_modules/@cubejs-backend/$pkg/dist/"; \ + target="/cube/node_modules/@cubejs-backend/$pkg/dist"; \ + [ -d "$target" ] || { echo "stacklet-patches: $target not installed" >&2; exit 1; }; \ + cp -r "$d/." "$target/" || exit 1; \ done && rm -rf /cube/stacklet-patches FROM node:22.22.0-bookworm-slim diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index 0703d8960b8ab..d784625796a02 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -1,25 +1,37 @@ #!/usr/bin/env bash # Build a Stacklet Cube Docker image. # -# Usage: ./scripts/build-docker.sh [TAG] -# TAG defaults to cubejs/cube:dev +# Usage: ./scripts/build-docker.sh [TAG [DOCKER_BUILD_ARGS...]] +# ./scripts/build-docker.sh --list-patched +# TAG defaults to cubejs/cube:dev. Any further arguments are passed through +# to `docker build` (e.g. --label, --build-arg). +# --list-patched prints PATCHED_PACKAGES, one per line, and exits. # -# This script compiles Stacklet-patched packages and stages their dist output -# into packages/cubejs-docker/stacklet-patches/ before invoking docker build. -# The Dockerfile picks up whatever *-dist/ directories it finds there and -# overlays them on the npm-installed versions of those packages. +# latest.Dockerfile installs the npm-published @cubejs-backend/* packages, so +# local source changes don't reach the image on their own. This script compiles +# the monorepo and stages the dist/ of each package in PATCHED_PACKAGES into +# packages/cubejs-docker/stacklet-patches/-dist/. The Dockerfile overlays +# every *-dist/ directory it finds there onto the npm-installed package. # -# To add a new patched package in the future: -# 1. Make your source changes and run: yarn workspace @cubejs-backend/ build -# 2. Add a compile + stage block below (following the server-core example) -# 3. No Dockerfile change is needed. +# To patch another package, add its @cubejs-backend/ name (without the scope) +# to PATCHED_PACKAGES. No Dockerfile change is needed. set -euo pipefail +PATCHED_PACKAGES=( + server-core +) + +if [ "${1:-}" = "--list-patched" ]; then + printf '%s\n' "${PATCHED_PACKAGES[@]}" + exit 0 +fi + REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" DOCKER_DIR="$REPO_ROOT/packages/cubejs-docker" PATCHES_DIR="$DOCKER_DIR/stacklet-patches" TAG="${1:-cubejs/cube:dev}" +shift || true cleanup() { echo "Cleaning up staged patches..." @@ -27,24 +39,53 @@ cleanup() { } trap cleanup EXIT +# Directory under packages/ whose package.json is named @cubejs-backend/. +package_dir() { + local pkg="$1" manifest + for manifest in "$REPO_ROOT"/packages/*/package.json; do + if [ "$(node -p "require('$manifest').name")" = "@cubejs-backend/$pkg" ]; then + dirname "$manifest" + return 0 + fi + done + echo "ERROR: no package named @cubejs-backend/$pkg under packages/" >&2 + return 1 +} + # --------------------------------------------------------------------------- # Compile and stage Stacklet-patched packages # --------------------------------------------------------------------------- -echo "==> Building @cubejs-backend/server-core..." -yarn --cwd "$REPO_ROOT" workspace @cubejs-backend/server-core build +declare -A PATCHED_DIRS=() +for pkg in "${PATCHED_PACKAGES[@]}"; do + PATCHED_DIRS[$pkg]="$(package_dir "$pkg")" + # tsc never deletes stale output, and the package's tsbuildinfo lives in + # dist/, so this forces a clean rebuild of what gets overlaid. + rm -rf "${PATCHED_DIRS[$pkg]}/dist" +done + +# Patched packages compile against the workspace's other packages, so build +# the whole TypeScript project graph (incremental after the first run). +echo "==> Compiling TypeScript packages..." +yarn --cwd "$REPO_ROOT" tsc + +for pkg in "${PATCHED_PACKAGES[@]}"; do + dir="${PATCHED_DIRS[$pkg]}" + echo "==> Staging $pkg dist from ${dir#"$REPO_ROOT"/}..." + mkdir -p "$PATCHES_DIR/$pkg-dist" + cp -r "$dir/dist/." "$PATCHES_DIR/$pkg-dist/" +done -echo "==> Staging server-core dist..." -mkdir -p "$PATCHES_DIR/server-core-dist" -cp -r "$REPO_ROOT/packages/cubejs-server-core/dist/." \ - "$PATCHES_DIR/server-core-dist/" +# The Docker build context is packages/cubejs-docker, so it needs its own copy +# of the root lockfile (which carries the CVE resolutions). +cp "$REPO_ROOT/yarn.lock" "$DOCKER_DIR/yarn.lock" # --------------------------------------------------------------------------- # Docker build # --------------------------------------------------------------------------- echo "==> Building Docker image $TAG..." -docker build -t "$TAG" -f "$DOCKER_DIR/latest.Dockerfile" "$DOCKER_DIR" +docker build -t "$TAG" -f "$DOCKER_DIR/latest.Dockerfile" "$@" "$DOCKER_DIR" echo "==> Done: $TAG" diff --git a/scripts/stacklet-overlay-guard.sh b/scripts/stacklet-overlay-guard.sh new file mode 100755 index 0000000000000..fbbeff6832d72 --- /dev/null +++ b/scripts/stacklet-overlay-guard.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Fail if this tree changes code that won't reach the Docker image. +# +# Usage: ./scripts/stacklet-overlay-guard.sh [BASE_REF] +# BASE_REF defaults to the upstream cube-js/cube release tag matching the +# version in packages/cubejs-docker/package.json, fetched from GitHub. +# +# The image installs npm-published packages and overlays only the compiled +# TypeScript of the @cubejs-backend packages listed by +# `build-docker.sh --list-patched`. Every other change under packages/ or +# rust/ is silently dropped, so it fails here instead. Allowed: Markdown, +# tests, packages/cubejs-docker/ itself, and the TypeScript sources of +# patched packages (src/, or js/ for @cubejs-backend/native, whose src/ is +# Rust). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +if [ $# -ge 1 ]; then + base="$1" +else + version="$(node -p "require('./packages/cubejs-docker/package.json').version")" + # Only the tag's tree is needed. A depth limit in a full clone would make it + # shallow, so only use one where the clone is already shallow (CI). + depth=() + if [ "$(git rev-parse --is-shallow-repository)" = true ]; then + depth=(--depth=1) + fi + echo "==> Fetching upstream tag v$version..." + git fetch --quiet --no-tags "${depth[@]}" https://github.com/cube-js/cube.git "refs/tags/v$version" + base=FETCH_HEAD +fi + +# Process substitution hides a failing command from `set -e`, so check the base +# and capture command output in plain assignments first. +if ! git rev-parse --verify --quiet "$base^{tree}" >/dev/null; then + echo "ERROR: base ref '$base' does not resolve to a tree" + exit 1 +fi +patched_out="$(./scripts/build-docker.sh --list-patched)" +changed_out="$(git diff --name-only "$base" HEAD)" +mapfile -t patched <<< "$patched_out" +mapfile -t changed <<< "$changed_out" + +# Directory of TypeScript sources the overlay ships, per patched package. +declare -A shipped=() +for pkg in "${patched[@]}"; do + if [ "$pkg" = native ]; then + shipped[packages/cubejs-backend-native]=js + else + for manifest in packages/*/package.json; do + if [ "$(node -p "require('./$manifest').name")" = "@cubejs-backend/$pkg" ]; then + shipped[$(dirname "$manifest")]=src + fi + done + fi +done + +declare -A dropped=() +for path in "${changed[@]}"; do + case "$path" in + '' | *.md | packages/cubejs-docker/*) continue ;; + esac + # Top-level test directories only: `case` globs would also match test/ + # directories nested inside shipped code. + if [[ "$path" =~ ^packages/[^/]+/(test|__tests__)/ ]]; then + continue + fi + if [[ "$path" =~ ^(packages/[^/]+)/([^/]+)/ && "${shipped[${BASH_REMATCH[1]}]:-}" = "${BASH_REMATCH[2]}" ]]; then + continue + fi + if [[ "$path" =~ ^((packages|rust)/[^/]+)/ ]]; then + dropped[${BASH_REMATCH[1]}]+=" $path"$'\n' + fi +done + +if [ ${#dropped[@]} -gt 0 ]; then + echo "ERROR: changes that won't reach the image:" + for dir in $(printf '%s\n' "${!dropped[@]}" | sort); do + echo " $dir" + printf '%s' "${dropped[$dir]}" + done + echo "Only the TypeScript sources of packages in PATCHED_PACKAGES" + echo "(scripts/build-docker.sh) are overlaid. Rust ships as prebuilt binaries." + exit 1 +fi + +echo "Overlay guard OK (patched: ${patched[*]})" From a7eb33d6341d020b8223046ea9d427c6fac8ad5c Mon Sep 17 00:00:00 2001 From: James Troup Date: Fri, 23 Oct 2026 02:27:39 +0100 Subject: [PATCH 2/4] chore: fix image existence check and address review feedback --- .github/workflows/stacklet-image.yml | 16 +++++++++++++--- scripts/stacklet-overlay-guard.sh | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/stacklet-image.yml b/.github/workflows/stacklet-image.yml index b665991529e31..1b30921e89eac 100644 --- a/.github/workflows/stacklet-image.yml +++ b/.github/workflows/stacklet-image.yml @@ -131,15 +131,23 @@ jobs: id: exists env: TAG: ${{ steps.meta.outputs.tag }} + REGISTRY: ${{ vars.ECR_REGISTRY }} REPOSITORY: ${{ vars.ECR_REPOSITORY }} + # The registry is in another account than the role, so name it: + # without --registry-id the ECR API looks in the caller's account. + # Only "not found" means build; any other error fails the job. run: | - if aws ecr describe-images \ + if err="$(aws ecr describe-images \ + --registry-id "${REGISTRY%%.*}" \ --repository-name "$REPOSITORY" \ - --image-ids imageTag="$TAG" >/dev/null 2>&1; then + --image-ids imageTag="$TAG" 2>&1 >/dev/null)"; then echo "exists=true" >> "$GITHUB_OUTPUT" echo "Image $TAG already exists; skipping build." - else + elif [[ "$err" == *ImageNotFoundException* ]]; then echo "exists=false" >> "$GITHUB_OUTPUT" + else + echo "$err" + exit 1 fi build: @@ -304,9 +312,11 @@ jobs: - name: Check published manifest env: TAG: ${{ needs.resolve.outputs.tag }} + REGISTRY: ${{ vars.ECR_REGISTRY }} REPOSITORY: ${{ vars.ECR_REPOSITORY }} run: | media_type="$(aws ecr describe-images \ + --registry-id "${REGISTRY%%.*}" \ --repository-name "$REPOSITORY" \ --image-ids imageTag="$TAG" \ --query 'imageDetails[0].imageManifestMediaType' \ diff --git a/scripts/stacklet-overlay-guard.sh b/scripts/stacklet-overlay-guard.sh index fbbeff6832d72..b69c70594126e 100755 --- a/scripts/stacklet-overlay-guard.sh +++ b/scripts/stacklet-overlay-guard.sh @@ -65,7 +65,7 @@ for path in "${changed[@]}"; do esac # Top-level test directories only: `case` globs would also match test/ # directories nested inside shipped code. - if [[ "$path" =~ ^packages/[^/]+/(test|__tests__)/ ]]; then + if [[ "$path" =~ ^packages/[^/]+/(test|tests|__tests__)/ ]]; then continue fi if [[ "$path" =~ ^(packages/[^/]+)/([^/]+)/ && "${shipped[${BASH_REMATCH[1]}]:-}" = "${BASH_REMATCH[2]}" ]]; then From 9e6199c36163b46fe71220f939e540f45b1b092d Mon Sep 17 00:00:00 2001 From: James Troup Date: Fri, 23 Oct 2026 02:59:59 +0100 Subject: [PATCH 3/4] chore: address more PR review feedback from Cubic --- packages/cubejs-docker/latest.Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-docker/latest.Dockerfile b/packages/cubejs-docker/latest.Dockerfile index f2770bbceb7ae..74353fd511025 100644 --- a/packages/cubejs-docker/latest.Dockerfile +++ b/packages/cubejs-docker/latest.Dockerfile @@ -33,14 +33,17 @@ RUN yarn install --prod \ && rm -rf /cube/node_modules/duckdb/src \ && yarn cache clean -# Stacklet: overlay locally compiled dist files over npm-installed packages. -# stacklet-patches/-dist/ dirs are populated by scripts/build-docker.sh. +# Stacklet: replace the dist/ of npm-installed packages with locally compiled +# ones. stacklet-patches/-dist/ dirs are populated by +# scripts/build-docker.sh. Each holds a package's full compiled output, so +# replacing rather than copying over stops files deleted from the source +# surviving from the npm build. COPY stacklet-patches/ /cube/stacklet-patches/ RUN find /cube/stacklet-patches -maxdepth 1 -name '*-dist' -type d | while read d; do \ pkg=$(basename "$d" -dist); \ target="/cube/node_modules/@cubejs-backend/$pkg/dist"; \ [ -d "$target" ] || { echo "stacklet-patches: $target not installed" >&2; exit 1; }; \ - cp -r "$d/." "$target/" || exit 1; \ + rm -rf "$target" && cp -r "$d" "$target" || exit 1; \ done && rm -rf /cube/stacklet-patches FROM node:22.22.0-bookworm-slim From 4708db73d9e5fc7195b6bbccb3d97dd200bfe554 Mon Sep 17 00:00:00 2001 From: James Troup Date: Fri, 23 Oct 2026 03:23:59 +0100 Subject: [PATCH 4/4] chore: switch stepsecurity into enforce mode --- .github/workflows/stacklet-image.yml | 38 +++++++++++++++++++++++++--- .github/workflows/stacklet-lint.yml | 14 +++++++++- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/.github/workflows/stacklet-image.yml b/.github/workflows/stacklet-image.yml index 1b30921e89eac..bd0cdbc15b38f 100644 --- a/.github/workflows/stacklet-image.yml +++ b/.github/workflows/stacklet-image.yml @@ -44,8 +44,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: - use-policy-store: true + egress-policy: block api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + allowed-endpoints: > + api.github.com:443 + github.com:443 + release-assets.githubusercontent.com:443 - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -91,8 +95,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: - use-policy-store: true + egress-policy: block api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + allowed-endpoints: > + api.ecr.${{ vars.AWS_REGION }}.amazonaws.com:443 + github.com:443 + sts.${{ vars.AWS_REGION }}.amazonaws.com:443 - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -158,11 +166,26 @@ jobs: permissions: contents: read steps: + # The image build reaches the npm, Docker Hub and Debian registries; + # the two env vars below keep yarn and the smoke test off Cypress's + # download servers and Cube's telemetry endpoint. - name: Harden Runner uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: - use-policy-store: true + egress-policy: block api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + allowed-endpoints: > + api.github.com:443 + auth.docker.io:443 + deb.debian.org:80 + github.com:443 + nodejs.org:443 + npm.duckdb.org:443 + production.cloudfront.docker.com:443 + registry-1.docker.io:443 + registry.npmjs.org:443 + registry.yarnpkg.com:443 + release-assets.githubusercontent.com:443 - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -180,6 +203,8 @@ jobs: - name: Install dependencies env: CUBESTORE_SKIP_POST_INSTALL: "true" + # A dev dependency's test runner; the build never uses it. + CYPRESS_INSTALL_BINARY: "0" run: yarn install --frozen-lockfile # Single-platform image with no attestations, so ECR stores a plain @@ -208,6 +233,7 @@ jobs: -e CUBEJS_API_SECRET=smoke-test \ -e CUBEJS_DB_TYPE=postgres \ -e CUBEJS_CACHE_AND_QUEUE_DRIVER=memory \ + -e CUBEJS_TELEMETRY=false \ "$IMAGE" probe="require('http').get('http://localhost:4000/livez',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))" ok=false @@ -262,8 +288,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: - use-policy-store: true + egress-policy: block api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + allowed-endpoints: > + ${{ vars.ECR_REGISTRY }}:443 + api.ecr.${{ vars.AWS_REGION }}.amazonaws.com:443 + sts.${{ vars.AWS_REGION }}.amazonaws.com:443 - name: Download image if: needs.build.result == 'success' diff --git a/.github/workflows/stacklet-lint.yml b/.github/workflows/stacklet-lint.yml index e7be83573eb96..416644cedf147 100644 --- a/.github/workflows/stacklet-lint.yml +++ b/.github/workflows/stacklet-lint.yml @@ -32,8 +32,20 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: - use-policy-store: true + egress-policy: block api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + # mise installs pre-commit; pre-commit builds actionlint (Go) and + # installs zizmor and shellcheck (PyPI). + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + mise-versions.jdx.dev:443 + mise.jdx.dev:443 + proxy.golang.org:443 + pypi.org:443 + release-assets.githubusercontent.com:443 + storage.googleapis.com:443 + sum.golang.org:443 - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1