diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 876188af..7f0b207a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,20 +1,30 @@ name: Docker image # The official self-host image, ghcr.io/cloudhal-dev/cloudhal (issue #143, docs/guides/self-host.md). -# Built from the npm release, never from the tree, so the image is exactly what npm users get: -# - release.yml calls this after a successful npm publish (`workflow_call` with the version): -# tags X.Y.Z, X.Y, latest and sha-; -# - Actions › Docker image › Run workflow with a version re-publishes or backfills it (X.Y.Z, X.Y -# and sha-; `latest` never moves on a dispatch, so a backfill cannot point it at an -# older release); with no version it builds `edge`; -# - nightly, `edge` is built from main's own packed tarballs (scripts/pack-packages.ts): tagged -# edge and sha- only, never latest; +# Built from the npm release, never from the tree, so the image is exactly what npm users get. +# Three jobs, so that no release tag moves before the gates pass (#166): +# - `build` builds linux/amd64 + linux/arm64 with an SBOM and buildx provenance, pushes it under +# the candidate tag `sha-` only, then runs scripts/docker-smoke.ts on the amd64 image +# (empty volume, health check, Studio's page, an anonymous `/mcp` call, the first owner through +# the API, a publish, `site-build` twice) and Trivy, which fails the job on a CRITICAL +# vulnerability with a fix; +# - `smoke-arm64` runs the same smoke test on the arm64 image, on GitHub's native arm64 runner +# (free for public repositories; under QEMU a Node process and an Astro build take far longer +# than the ten minutes that would be worth); +# - `tag` adds the release tags to the very digest that passed, with `imagetools create`, and then +# writes the GitHub build attestation for it. +# Which tags `tag` adds: +# - release.yml calls this after a successful npm publish (`workflow_call` with the version), and +# Actions › Docker image › Run workflow with a version re-publishes or backfills one. X.Y.Z +# always; X.Y when the version is the highest patch of that minor among the repository's v* +# tags; `latest` when it is the highest release of all. So a release started by hand still gets +# `latest`, and a backfill of an older patch moves nothing backwards. +# - with no version (a dispatch without one, and nightly) `edge` is built from main's own packed +# tarballs (scripts/pack-packages.ts) and tagged edge only, never latest; # - a pull request that touches the image builds it for linux/amd64 without pushing and runs the -# same smoke test and scan. -# Every push is linux/amd64 + linux/arm64 with an SBOM, buildx provenance and a GitHub build -# attestation; scripts/docker-smoke.ts then runs the amd64 image (empty volume, health check, the -# first owner through the API, a publish, `site-build`), and Trivy fails the job on a CRITICAL -# vulnerability with a fix. Versions before 0.6.3 have no `--init-if-empty`: the image needs it. +# amd64 smoke test and the scan. +# Versions before 0.6.4 cannot pass the gates: the smoke test expects the first owner's setup +# token (#165), which 0.6.4 brought, and the image needs `--init-if-empty` (0.6.3). on: workflow_call: @@ -49,15 +59,15 @@ env: IMAGE: ghcr.io/${{ github.repository_owner }}/cloudhal jobs: - image: - name: ${{ inputs.version && format('Image {0}', inputs.version) || 'Image edge' }} + build: + name: ${{ inputs.version && format('Build {0}', inputs.version) || 'Build edge' }} runs-on: ubuntu-latest timeout-minutes: 90 permissions: contents: read packages: write - id-token: write - attestations: write + outputs: + digest: ${{ steps.build.outputs.digest }} env: VERSION: ${{ inputs.version || '' }} PUSH: ${{ github.event_name != 'pull_request' }} @@ -68,6 +78,21 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24 + # The compose file's Caddyfile, in the Caddy the compose file runs (a few seconds). + - name: Validate the Caddyfile + run: | + docker run --rm \ + -e CLOUDHAL_BASE_URL=https://studio.example.com \ + -e CLOUDHAL_SITE_URL=https://www.example.com \ + -v "$PWD/deploy/docker/Caddyfile:/etc/caddy/Caddyfile:ro" \ + caddy:2.11 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + - name: Check the version + if: env.VERSION != '' + run: | + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::version must be x.y.z (got '$VERSION')" + exit 1 + fi # edge: the packages of this ref, built and packed the way the release job packs them - if: env.VERSION == '' @@ -83,17 +108,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # The candidate tag only; the release tags come from the `tag` job once the gates pass. - id: meta uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ${{ env.IMAGE }} flavor: latest=false - tags: | - type=semver,pattern={{version}},value=v${{ env.VERSION }},enable=${{ env.VERSION != '' }} - type=semver,pattern={{major}}.{{minor}},value=v${{ env.VERSION }},enable=${{ env.VERSION != '' }} - type=raw,value=latest,enable=${{ env.VERSION != '' && github.event_name != 'workflow_dispatch' }} - type=raw,value=edge,enable=${{ env.VERSION == '' }} - type=sha,prefix=sha- + tags: type=sha,prefix=sha- labels: | org.opencontainers.image.title=Cloudhal org.opencontainers.image.description=Agent-native site platform: Studio, API and MCP server, self-hosted @@ -118,12 +139,6 @@ jobs: # attestations need a registry push; a loaded local image takes none sbom: ${{ env.PUSH == 'true' }} provenance: ${{ env.PUSH == 'true' && 'mode=max' || 'false' }} - - if: env.PUSH == 'true' - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 - with: - subject-name: ${{ env.IMAGE }} - subject-digest: ${{ steps.build.outputs.digest }} - push-to-registry: true - id: ref run: | @@ -144,3 +159,95 @@ jobs: ignore-unfixed: true exit-code: "1" format: table + + smoke-arm64: + name: Smoke test (linux/arm64) + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: node scripts/docker-smoke.ts --image "$IMAGE@$DIGEST" + env: + DIGEST: ${{ needs.build.outputs.digest }} + + tag: + name: ${{ inputs.version && format('Tag {0}', inputs.version) || 'Tag edge' }} + needs: [build, smoke-arm64] + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + id-token: write + attestations: write + env: + VERSION: ${{ inputs.version || '' }} + DIGEST: ${{ needs.build.outputs.digest }} + steps: + - uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: tags + name: Which tags this digest gets + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if [ -z "$VERSION" ]; then + tags="edge" + else + # Every released version: the repository's v* tags (release.yml pushes one per release) + # plus this version itself, which has none yet when a release is finished by hand. + # `grep` always matches at least this version, so the pipelines never come up empty. + released=$( { gh api --paginate "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" --jq '.[].ref' | sed 's#^refs/tags/v##'; echo "$VERSION"; } \ + | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -uV) + minor="${VERSION%.*}" + highest=$(printf '%s\n' "$released" | tail -n 1) + highest_of_minor=$(printf '%s\n' "$released" | grep -E "^${minor//./\\.}\.[0-9]+$" | tail -n 1) + tags="$VERSION" + [ "$highest_of_minor" = "$VERSION" ] && tags="$tags $minor" + [ "$highest" = "$VERSION" ] && tags="$tags latest" + fi + echo "tags=$tags" >> "$GITHUB_OUTPUT" + echo "Tagging $IMAGE@$DIGEST as: $tags" | tee -a "$GITHUB_STEP_SUMMARY" + - id: push + name: Tag the digest that passed + env: + TAGS: ${{ steps.tags.outputs.tags }} + run: | + set -euo pipefail + args=() + for t in $TAGS; do args+=(--tag "$IMAGE:$t"); done + # One source and no annotations: the index is copied as it is, so the digest stays the + # one the smoke tests and Trivy saw. Read back and checked, and the attestation below + # names what the tags really point at. + docker buildx imagetools create "${args[@]}" "$IMAGE@$DIGEST" + first="${TAGS%% *}" + final=$(docker buildx imagetools inspect "$IMAGE:$first" --format '{{.Manifest.Digest}}') + echo "digest=$final" >> "$GITHUB_OUTPUT" + if [ "$final" != "$DIGEST" ]; then + echo "::warning::$IMAGE:$first is $final, not the tested $DIGEST" + fi + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bf74ce2..6496455f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ All publishable packages (`@cloudhal/*` and `create-cloudhal`) share one version - **cloudhal-template** (the Deploy to Cloudflare button): the workflow's `template` job (`.github/workflows/template-sync.yml`) opens `chore: Cloudhal X.Y.Z and the X.Y.Z starter theme` from `cloudhal/vX.Y.Z`, with `src/`, `public/`, `astro.config.mjs` and `tsconfig.json` from the published starter, `@cloudhal/*` at `^X.Y.Z` and a lockfile that installs exactly `X.Y.Z`. Merge it once its `cloudhal/template-check` status is green: the same run installs a copy of the synced template with `npm ci` and runs `cloudhal deploy --dry-run` with `WORKERS_CI=1`. To sync again, run Actions › Template sync › Run workflow. Without the `TEMPLATE_SYNC_APP_CLIENT_ID` variable and the `TEMPLATE_SYNC_APP_PRIVATE_KEY` secret (a GitHub App installed on cloudhal-template only, with Contents, Pull requests and Commit statuses write) the job only logs a notice. Then sync by hand from a checkout: clone cloudhal-template next to this repository, run `node scripts/template-sync.ts --template ../cloudhal-template --version X.Y.Z --summary pr-body.md` and then the same command with `--lock` in place of `--summary` (Node 24, no `pnpm install` needed), check a copy of the template (`npm ci`, then `WORKERS_CI=1 npx cloudhal deploy --dry-run`), commit on `cloudhal/vX.Y.Z`, push, open the PR with the title above and `pr-body.md` as its body, and post the `cloudhal/template-check` status on its head commit (`gh api -X POST repos/cloudhal-dev/cloudhal-template/statuses/ -f state=success -f context=cloudhal/template-check -f description='npm ci and cloudhal deploy --dry-run (WORKERS_CI=1) passed, run by hand'`). Dependabot's `@cloudhal/*` PR on the template is no substitute: it moves dependencies only, no theme, and it fails to install whenever the starter's own pins moved with the release (0.6.3 moved astro to 7.4.7 for the `^7.4.0` peer range, so Dependabot's `^0.6.1` PR hit ERESOLVE); close it as superseded by the sync PR. - **cloudhal.com**: Dependabot opens the `@cloudhal/*` PR the next morning (06:00 Europe/Istanbul). A patch release merges itself after its check; a minor bump waits for a person. - **try.cloudhal.com** (`cloudhal-dev/try`): by hand. Bump its exact `@cloudhal/*` pins, then `pnpm install` and `pnpm run deploy`. - - **The Docker image** (`ghcr.io/cloudhal-dev/cloudhal`, [self-host guide](docs/guides/self-host.md)): the workflow's `docker` job (`.github/workflows/docker.yml`) builds it from the npm release and pushes `X.Y.Z`, `X.Y` and `latest` for linux/amd64 and linux/arm64, with an SBOM, provenance and a smoke test. The image never carries a version of its own: one release, one version everywhere. To rebuild a version (a base-image fix) run Actions › Docker image › Run workflow with it; that re-tags `X.Y.Z` and `X.Y` and leaves `latest` alone. `edge` is built nightly from `main`. + - **The Docker image** (`ghcr.io/cloudhal-dev/cloudhal`, [self-host guide](docs/guides/self-host.md)): the workflow's `docker` job (`.github/workflows/docker.yml`) builds it from the npm release for linux/amd64 and linux/arm64 with an SBOM and provenance, pushes it under `sha-` only, runs the smoke test on both architectures and Trivy against that digest, and only then tags it: `X.Y.Z` always, `X.Y` when the version is the highest patch of that minor among the `v*` tags, `latest` when it is the highest release of all. The image never carries a version of its own: one release, one version everywhere. To rebuild a version (a base-image fix) run Actions › Docker image › Run workflow with it (0.6.4 or later: older versions fail the smoke test's setup-token check); the same rule decides its tags, so a release finished by hand still gets `latest` and a backfill of an older patch moves nothing. `edge` is built nightly from `main`. **If something fails:** use **Re-run failed jobs** on the failed run. The script skips packages npm already has and an existing tag or release, so it finishes a half-done release. A new run (Actions › Release › Run workflow) does not: once npm has every package, `plan` skips `publish`, so a missing tag or GitHub Release stays missing. To redo only the template PR, run Actions › Template sync › Run workflow. npm's trusted-publisher entry for each package names `release.yml` and the `npm-publish` environment: renaming either breaks publishing. diff --git a/deploy/docker/.env.example b/deploy/docker/.env.example index 4e334718..79adf905 100644 --- a/deploy/docker/.env.example +++ b/deploy/docker/.env.example @@ -1,20 +1,42 @@ -# Copy to .env next to compose.yml. No secrets live here: Better Auth's secret is created on the -# volume at first start (/data/.cloudhal/auth-secret), and invites are printed to the logs. +# Copy to .env next to compose.yml (docs/guides/self-host.md). `docker compose` reads it and passes +# the values to the container. Treat it as private — `chmod 600 .env`, never commit it — since it +# holds secrets once Build chat is on (CLOUDHAL_AI_TOKEN) or BETTER_AUTH_SECRET or +# CLOUDHAL_SETUP_TOKEN is set. To keep +# secrets out of a checkout, put them in a second file outside it and name both, with full paths +# (relative ones resolve against the current directory), in COMPOSE_ENV_FILES for every command +# that creates the container (`up`, `pull && up`, `run`), e.g. in the shell profile: +# export COMPOSE_ENV_FILES=/path/to/cloudhal/.env,/etc/cloudhal/secrets.env +# A one-off `--env-file` is not enough: the next `up` without it drops the secrets. The cron line's +# `docker compose exec` runs in the running container and needs neither. # The image tag. `latest` follows every release; pin the minor (`0.6`) in production. CLOUDHAL_TAG=latest -# Where Studio is reached. Without a reverse proxy: this machine. Behind one (the caddy profile): -# the public https origin, and CLOUDHAL_BEHIND_PROXY=1 so the proxy's X-Forwarded-* headers count. -CLOUDHAL_BASE_URL=http://localhost:8787 +# The volume that holds the installation (the project, its database, media, the built site). A +# stack created with the compose file of 0.6.3 has `_cloudhal-data` (`docker volume ls`; +# `cloudhal_cloudhal-data` after the guide's quick start): name it here to keep using it. A second +# Cloudhal stack on this host needs a volume name of its own (and its own port and base URL). +#CLOUDHAL_VOLUME=cloudhal-data + +# Where browsers open Studio: scheme, host and port, no path, e.g. http://localhost:9000 with +# CLOUDHAL_PORT=9000, http://192.168.1.20:8787 on the LAN, https://studio.example.org behind a +# proxy. Set, it is the only origin sign-in accepts (and the origin of invite links), so it must be +# exactly what the address bar shows: a wrong value leaves Studio loading but every sign-in +# answered 403. Empty: sign-in accepts whatever address the browser used, and the log warns. +# Required behind a reverse proxy (the caddy profile), with CLOUDHAL_BEHIND_PROXY=1 so the proxy's +# X-Forwarded-* headers count: Studio refuses to start with the proxy flag and no base URL (or, +# in this image, a localhost one). +CLOUDHAL_BASE_URL= CLOUDHAL_BEHIND_PROXY=0 # Which address and port of this machine publish the container's port 8787 (0.0.0.0 to expose it -# directly; keep 127.0.0.1 with the caddy profile). +# directly; keep 127.0.0.1 with the caddy profile). A new port goes into CLOUDHAL_BASE_URL too. CLOUDHAL_BIND=127.0.0.1 CLOUDHAL_PORT=8787 -# The public site, served by Caddy from `site-build`'s output (caddy profile only). -CLOUDHAL_SITE_URL=https://www.example.com +# The public site's origin (https://host, no path), once the site has a domain: Caddy serves +# `site-build`'s output there (caddy profile), and `site-build` writes the host into canonical +# links, the sitemap and feeds (cloudhal.json `domains.site`). Leave it empty until then. +CLOUDHAL_SITE_URL= # The first start on an empty volume creates a project from this starter with this site name. CLOUDHAL_INIT=blog @@ -24,6 +46,18 @@ CLOUDHAL_SITE_NAME=My site CLOUDHAL_AUTOMATIONS=on # Extra origins the API accepts browser requests from, comma-separated (normally empty). CLOUDHAL_TRUSTED_ORIGINS= -# Workers AI over REST for Build chat (RFC-0016): both or neither. -CLOUDHAL_AI_TOKEN= -CLOUDFLARE_ACCOUNT_ID= + +# Secrets, each passed to the container only when set here or in the shell. +# Better Auth's secret is created on the volume at first start (/data/.cloudhal/auth-secret) and +# kept there; set this only to supply your own (32 or more random characters). Never leave it empty. +#BETTER_AUTH_SECRET= +# The first owner's setup token. Empty: a token is generated at each start while there is no +# account, and the setup link (http://localhost:8787/login#setup=) is printed to the logs +# (`docker compose logs cloudhal`). To choose it yourself: a random value of 32 characters or +# more, only letters, digits and - . _ ~ (`openssl rand -hex 32`); the log then shows the link +# with in place of the value. +#CLOUDHAL_SETUP_TOKEN= +# Workers AI over REST for Build chat (RFC-0016): both or neither. The token (a Cloudflare API +# token with Workers AI read) stays in the container's memory; it is never written to the volume. +#CLOUDHAL_AI_TOKEN= +#CLOUDFLARE_ACCOUNT_ID= diff --git a/deploy/docker/Caddyfile b/deploy/docker/Caddyfile index 14ca2920..63cf0d0a 100644 --- a/deploy/docker/Caddyfile +++ b/deploy/docker/Caddyfile @@ -1,7 +1,8 @@ # Caddy for the compose file's `caddy` profile: automatic TLS for both hosts (the DNS of each must # point at this machine, ports 80 and 443 open). Studio and the API at CLOUDHAL_BASE_URL, proxied -# with X-Forwarded-Proto/Host (set CLOUDHAL_BEHIND_PROXY=1); the public site, as `site-build` -# wrote it, at CLOUDHAL_SITE_URL. +# with X-Forwarded-Proto/Host/For (set CLOUDHAL_BEHIND_PROXY=1); the public site at +# CLOUDHAL_SITE_URL, from /data/dist: the built site as `site-build` publishes it (a real folder +# with the 0.6.3 image; a symlink to /data/site/current, swapped atomically, from 0.6.4 on). {$CLOUDHAL_BASE_URL} { reverse_proxy cloudhal:8787 } @@ -9,9 +10,36 @@ {$CLOUDHAL_SITE_URL} { root * /srv/cloudhal/dist encode zstd gzip - try_files {path} {path}/ {path}.html - file_server + # The build's `_headers` file (the security headers of the Cloudflare deployment) is for hosts + # that read it; Caddy does not. Media files, Astro's assets and the renderer's scripts are named + # by content hash and never change under a path: immutable, but only when the file exists, so a + # miss is never cached. Add the security headers from `_headers` here if you want them. + @immutable { + path /media/* /_astro/* /_cloudhal/* + file + } + header @immutable Cache-Control "public, max-age=31536000, immutable" + # A page opened before the last build may still ask for that build's hashed assets: they come + # from /data/site/previous, the build before the current one (0.6.4 and later; immutable too). + @previous { + path /_astro/* /_cloudhal/* /media/* + not file + file { + root /srv/cloudhal/site/previous + } + } + handle @previous { + root * /srv/cloudhal/site/previous + header Cache-Control "public, max-age=31536000, immutable" + file_server + } + handle { + try_files {path} {path}/ {path}.html + file_server + } handle_errors { + # /404.html exists only when the site has a page at /404 (the starter ships none); until + # then a miss is Caddy's plain 404. @notfound expression {err.status_code} == 404 rewrite @notfound /404.html file_server diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 324c944a..66a0c285 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -3,12 +3,13 @@ # volume. Built from the npm release (`CLOUDHAL_VERSION=x.y.z`), or from packed tarballs in # `tarballs/` next to this file for the nightly `edge` tag (.github/workflows/docker.yml). # -# docker build --build-arg CLOUDHAL_VERSION=0.6.2 -t cloudhal:local deploy/docker +# docker build --build-arg CLOUDHAL_VERSION=0.6.4 -t cloudhal:local deploy/docker # docker run --rm -p 127.0.0.1:8787:8787 -v cloudhal-data:/data cloudhal:local # # No compiler is installed: better-sqlite3 comes as a prebuilt binary for linux/amd64 and -# linux/arm64. The public site is `cloudhal build` output (`site-build` in the container) that a -# web server such as the compose file's Caddy serves from the same volume. +# linux/arm64. The public site is `cloudhal build` output, published by `site-build` at /data/dist +# (a symlink to the newest build under /data/site, swapped atomically) for a web server such as the +# compose file's Caddy to serve from the same volume. FROM node:24-bookworm-slim AS build ARG CLOUDHAL_VERSION="" diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 092f7d1c..0f485f1c 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -3,17 +3,17 @@ The official self-host image, `ghcr.io/cloudhal-dev/cloudhal`, built from each npm release (issue #143). The guide is [docs/guides/self-host.md](../../docs/guides/self-host.md); this folder holds what it refers to: - `Dockerfile` — `node:24-bookworm-slim`, `@cloudhal/cli` installed with npm (`CLOUDHAL_VERSION`), the blog starter's dependencies next to it, non-root, `/data` volume, port 8787, a health check on `/api/health`. `install.mjs` is its build stage. -- `bin/entrypoint` — `cloudhal studio --init-if-empty` in `/data`; `bin/site-build` — the public site as static HTML into `/data/dist`. +- `bin/entrypoint` — `cloudhal studio --init-if-empty` in `/data`; `bin/site-build` — the public site as static HTML, built in a staging copy of the project and published at `/data/dist`, a symlink to the newest build under `/data/site` (swapped atomically, never a 404 — except for a moment on the first build after upgrading from 0.6.3, when the real folder moves aside to `/data/site/legacy-0.6.3`; the build before the current one is kept as `/data/site/previous`). Never run an Astro build (`cloudhal build`, `cloudhal deploy`) inside `/data` by hand: it empties the served build through the symlink. - `compose.yml`, `.env.example`, `Caddyfile` — Studio alone, or with Caddy (profile `caddy`) serving Studio and the built site with automatic TLS. Build it yourself from a release, or from this checkout's packages (`edge`): ```bash -docker build --build-arg CLOUDHAL_VERSION=0.6.3 -t cloudhal:local deploy/docker +docker build --build-arg CLOUDHAL_VERSION=0.6.4 -t cloudhal:local deploy/docker ``` ```bash node scripts/pack-packages.ts deploy/docker/tarballs && docker build -t cloudhal:local deploy/docker ``` -`node scripts/docker-smoke.ts --image cloudhal:local` runs what CI runs after every build (`.github/workflows/docker.yml`). +`node scripts/docker-smoke.ts --image cloudhal:local` runs what CI runs after every build (`.github/workflows/docker.yml`); `--platform linux/arm64` runs the other architecture where Docker can. diff --git a/deploy/docker/bin/entrypoint b/deploy/docker/bin/entrypoint index e81b920e..dbbcd418 100755 --- a/deploy/docker/bin/entrypoint +++ b/deploy/docker/bin/entrypoint @@ -4,7 +4,25 @@ # `docker run … cloudhal export` or `docker compose exec cloudhal site-build` work too. set -eu cd /data +# An empty BETTER_AUTH_SECRET (a blank value in .env) would be taken as the secret itself: treat it +# as unset, so the secret created on the volume stays in use. +if [ -z "${BETTER_AUTH_SECRET-unset}" ]; then unset BETTER_AUTH_SECRET; fi if [ "${1:-studio}" = "studio" ]; then + # Behind a proxy the public origin is required (`cloudhal studio` refuses to start without one). + # A localhost origin there is a leftover default, such as a .env from 0.6.3, and would only fail + # every sign-in with 403: refuse it as loudly. + proxy=$(printf '%s' "${CLOUDHAL_BEHIND_PROXY:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + case "$proxy" in + 1 | true | yes | on) + case "${CLOUDHAL_BASE_URL:-}" in + http://localhost | http://localhost:* | https://localhost | https://localhost:* | \ + http://127.0.0.1 | http://127.0.0.1:* | https://127.0.0.1 | https://127.0.0.1:*) + echo "entrypoint: CLOUDHAL_BEHIND_PROXY=1 needs CLOUDHAL_BASE_URL set to the public origin browsers open Studio at, not $CLOUDHAL_BASE_URL" >&2 + exit 1 + ;; + esac + ;; + esac [ $# -gt 0 ] && shift exec cloudhal studio --init-if-empty "$@" fi diff --git a/deploy/docker/bin/site-build b/deploy/docker/bin/site-build index 4bf97f7a..fe6d91b6 100755 --- a/deploy/docker/bin/site-build +++ b/deploy/docker/bin/site-build @@ -1,13 +1,149 @@ #!/bin/sh -# Builds the public site: static HTML from the published snapshot into /data/dist, which a web -# server (the compose file's Caddy) serves. The project's dependencies are the image's -# node_modules, linked into the project on first use, so no install and no network is needed. +# Builds the public site as static HTML and publishes it on the volume: # docker compose exec cloudhal site-build +# The served path is /data/dist, as in 0.6.3, so a web server (the compose file's Caddy, or nginx +# with root /data/dist) needs no change. It is now a symlink to /data/site/current, which points at +# the newest build in /data/site/; /data/site/previous points at the one before. +# +# `cloudhal build` always writes /dist and empties it first, so it never runs in /data: +# the project's own files are copied to /data/.site-build (its node_modules and data directory are +# symlinks, so the build reads the same store) and the site is built there. Only a finished build +# is published: it moves to /data/site/, then `current`, `previous` and `dist` are each +# replaced with one rename (`ln -sfn` + `mv -T`), which is atomic. Visitors see the previous build +# until the new one is complete, never a 404 or a half-written page — except for a moment on the +# first build after upgrading from 0.6.3, when the real /data/dist folder is moved aside and the +# symlink takes its place (two renames; bookworm's coreutils has no `mv --exchange`). A failed +# build changes nothing. The build before the current one is kept (pages opened from it still load +# their hashed assets, which the Caddyfile serves from `previous`); older ones are removed; the +# 0.6.3 folder is kept as `site/legacy-0.6.3` for one build. Never run an Astro build in /data by +# hand (`cloudhal build`, `cloudhal deploy`): it would empty the served build through the symlink. +# +# The snapshot is the published one, else the latest: before the first publish the site is built +# from the latest snapshot (RFC-0043's `cloudhal build --published` is not in this version). The +# project's dependencies are the image's node_modules, linked into the project on first use, so no +# install and no network is needed. set -eu cd /data if [ ! -f cloudhal.json ]; then echo "site-build: no project in /data yet; start the container once first" >&2 exit 1 fi +# One build at a time: a cron run and a hand-started one would otherwise share the staging folder. +exec 9>/tmp/site-build.lock +if command -v flock >/dev/null 2>&1 && ! flock -n 9; then + echo "site-build: another build is running; try again when it is done" >&2 + exit 1 +fi [ -e node_modules ] || ln -s /opt/cloudhal/node_modules node_modules -exec cloudhal build "$@" + +# Preflight, before anything is written. A real /data/dist folder is the 0.6.3 layout: it moves +# aside on publish. A mount point there (a host bind mount) cannot be moved: refuse, rather than +# fail after every build and leave a new one behind each time. +legacy="" +if [ -d dist ] && [ ! -L dist ]; then + if command -v mountpoint >/dev/null 2>&1; then + if mountpoint -q dist; then legacy=mount; fi + elif [ "$(stat -c %d dist)" != "$(stat -c %d .)" ]; then + legacy=mount + fi + if [ "$legacy" = mount ]; then + echo "site-build: /data/dist is a mount point; site-build publishes it as a symlink to /data/site/current. Remove that mount and serve /data/dist (or /data/site/current) from the volume instead" >&2 + exit 1 + fi + legacy=0.6.3 +fi +# Temporary links an interrupted run left behind (the prune below handles folders only). +find . -maxdepth 1 -type l -name 'dist.*' -exec rm -f {} + +if [ -d site ]; then + find site -maxdepth 1 -type l \( -name 'current.*' -o -name 'previous.*' \) -exec rm -f {} + +fi + +# The public origin for canonical links, the sitemap and feeds is the site's primary domain, kept +# in cloudhal.json as `domains.site` (`cloudhal domain set `). CLOUDHAL_SITE_URL — the +# origin Caddy serves — sets it when they differ; without either, the build's URLs say localhost. +# Setting the domain never stops the build: a value it cannot use is reported and skipped. +if [ -n "${CLOUDHAL_SITE_URL:-}" ]; then + host=$(node -e ' + let u; + try { u = new URL(process.argv[1]); } catch { u = undefined; } + if (!u || u.protocol !== "https:" || u.port || u.pathname !== "/" || u.search || u.hash) { + console.error("site-build: CLOUDHAL_SITE_URL must be an https origin without a path or port, such as https://www.example.org (got " + JSON.stringify(process.argv[1]) + "); building without it"); + process.exit(0); + } + // the rule for site domains (core-ts isHostname): labels, a dot, a letter TLD + if (!/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(u.hostname)) { + console.error("site-build: " + u.hostname + " cannot be a site domain (no dot, or an IP address); building without it"); + process.exit(0); + } + process.stdout.write(u.hostname); + ' "$CLOUDHAL_SITE_URL") + case "$host" in + "") ;; + example.com | *.example.com | example.net | *.example.net | example.org | *.example.org) + echo "site-build: CLOUDHAL_SITE_URL is the placeholder $CLOUDHAL_SITE_URL; set the site's real origin, or leave it empty" >&2 + ;; + *) + current=$(node -e ' + const cfg = JSON.parse(require("node:fs").readFileSync("cloudhal.json", "utf8")); + process.stdout.write(cfg.domains?.site ?? ""); + ') + if [ "$host" != "$current" ]; then + if cloudhal domain set "$host" >/dev/null; then + echo "site-build: public origin https://$host (cloudhal.json domains.site; the live site's primary domain)" + echo "site-build: a published snapshot that names another primary domain keeps it until the next publish" + else + echo "site-build: could not make $host the site's domain; building without it" >&2 + fi + fi + ;; + esac +fi + + +# Stage: the project's files, copied; node_modules and the data directory (cloudhal.json +# `dataDir`, default .cloudhal), linked; the build output, archives and caches, left out. +data_top=$(node -e ' + const cfg = JSON.parse(require("node:fs").readFileSync("cloudhal.json", "utf8")); + process.stdout.write(String(cfg.dataDir ?? ".cloudhal").replace(/^(\.\/)+/, "").split("/")[0]); +') +stage=.site-build +rm -rf "$stage" +trap 'rm -rf "/data/$stage"' EXIT +mkdir "$stage" +for entry in .[!.]* ..?* *; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + # (the data directory first: `*.cloudhal`, for archives, would match `.cloudhal` too) + case "$entry" in + # node_modules is itself a link (to /opt/cloudhal/node_modules): link its target, not a chain + node_modules) ln -s "$(readlink -f node_modules)" "$stage/node_modules" ;; + "$data_top") ln -s "/data/$entry" "$stage/$entry" ;; + "$stage" | site | dist | .astro | .cloudhal-init | ?*.cloudhal | *.tgz) ;; + *) cp -a "$entry" "$stage/" ;; + esac +done +(cd "$stage" && cloudhal build "$@") + +# Publish: the finished build → /data/site/, then each pointer with one rename. +id="$(date -u +%Y%m%dT%H%M%SZ).$$" +mkdir -p site +mv "$stage/dist" "site/$id" +prev=$(readlink site/current 2>/dev/null || true) +# The 0.6.3 folder becomes `previous`, so its pages' assets keep loading for one more build. +if [ "$legacy" = 0.6.3 ]; then prev="legacy-0.6.3"; fi +ln -sfn "$id" "site/current.$$" +mv -T "site/current.$$" site/current +if [ -n "$prev" ] && [ "$prev" != "$id" ]; then + ln -sfn "$prev" "site/previous.$$" + mv -T "site/previous.$$" site/previous +fi +# /data/dist, the stable served path. A 0.6.3 volume has a real folder there: it moves aside once +# (two renames, the one moment it is missing), and every later build only replaces the symlink. +if [ "$legacy" = 0.6.3 ]; then + rm -rf site/legacy-0.6.3 + mv dist site/legacy-0.6.3 +fi +ln -sfn site/current "dist.$$" +mv -T "dist.$$" dist +# Builds that neither `current` nor `previous` points at. +find site -mindepth 1 -maxdepth 1 -type d ! -name "$id" ! -name "${prev:-$id}" -exec rm -rf {} + +echo "site-build: /data/dist -> site/current -> $id" diff --git a/deploy/docker/compose.yml b/deploy/docker/compose.yml index c0596046..2d0b6238 100644 --- a/deploy/docker/compose.yml +++ b/deploy/docker/compose.yml @@ -12,14 +12,27 @@ services: ports: - "${CLOUDHAL_BIND:-127.0.0.1}:${CLOUDHAL_PORT:-8787}:8787" environment: - CLOUDHAL_BASE_URL: ${CLOUDHAL_BASE_URL:-http://localhost:8787} - CLOUDHAL_BEHIND_PROXY: ${CLOUDHAL_BEHIND_PROXY:-0} - CLOUDHAL_INIT: ${CLOUDHAL_INIT:-blog} - CLOUDHAL_SITE_NAME: ${CLOUDHAL_SITE_NAME:-My site} - CLOUDHAL_AUTOMATIONS: ${CLOUDHAL_AUTOMATIONS:-on} - CLOUDHAL_TRUSTED_ORIGINS: ${CLOUDHAL_TRUSTED_ORIGINS:-} - CLOUDHAL_AI_TOKEN: ${CLOUDHAL_AI_TOKEN:-} - CLOUDFLARE_ACCOUNT_ID: ${CLOUDFLARE_ACCOUNT_ID:-} + # The origin browsers open Studio at, which sign-in is tied to (.env.example). No default on + # purpose: empty, sign-in accepts the browser's own address and the log warns; behind a proxy + # (CLOUDHAL_BEHIND_PROXY=1) it is required, and Studio refuses to start without it. + - CLOUDHAL_BASE_URL=${CLOUDHAL_BASE_URL:-} + - CLOUDHAL_BEHIND_PROXY=${CLOUDHAL_BEHIND_PROXY:-0} + - CLOUDHAL_INIT=${CLOUDHAL_INIT:-blog} + - CLOUDHAL_SITE_NAME=${CLOUDHAL_SITE_NAME:-My site} + # The public site's origin: `site-build` writes its host into canonical links, the sitemap + # and feeds (cloudhal.json `domains.site`); with the caddy profile Caddy serves the site there. + - CLOUDHAL_SITE_URL=${CLOUDHAL_SITE_URL:-} + - CLOUDHAL_AUTOMATIONS=${CLOUDHAL_AUTOMATIONS:-on} + - CLOUDHAL_TRUSTED_ORIGINS=${CLOUDHAL_TRUSTED_ORIGINS:-} + # Secrets. A name without a value is passed through from .env or the shell only when it is + # set there, so an unset BETTER_AUTH_SECRET leaves the container with the secret it created + # on the volume (/data/.cloudhal/auth-secret). Never set it to an empty value. + - BETTER_AUTH_SECRET + # The first owner's setup token; empty: one is generated at each start while there is no + # account, and the setup link is printed to the logs. + - CLOUDHAL_SETUP_TOKEN=${CLOUDHAL_SETUP_TOKEN:-} + - CLOUDHAL_AI_TOKEN + - CLOUDFLARE_ACCOUNT_ID volumes: - cloudhal-data:/data @@ -34,16 +47,35 @@ services: - "443:443" - "443:443/udp" environment: - CLOUDHAL_BASE_URL: ${CLOUDHAL_BASE_URL:?set CLOUDHAL_BASE_URL in .env} - CLOUDHAL_SITE_URL: ${CLOUDHAL_SITE_URL:?set CLOUDHAL_SITE_URL in .env} + - CLOUDHAL_BASE_URL=${CLOUDHAL_BASE_URL:-} + - CLOUDHAL_SITE_URL=${CLOUDHAL_SITE_URL:-} + # Both origins are required for this profile. They are checked here, when Caddy starts, rather + # than with `:?` interpolation, which would fail `docker compose up` for everyone who never + # uses Caddy. + command: + - sh + - -c + - | + if [ -z "$$CLOUDHAL_BASE_URL" ] || [ -z "$$CLOUDHAL_SITE_URL" ]; then + echo "caddy: the caddy profile needs CLOUDHAL_BASE_URL and CLOUDHAL_SITE_URL in .env (https://… origins)" >&2 + exit 1 + fi + exec caddy run --config /etc/caddy/Caddyfile --adapter caddyfile volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - # the built site (`site-build` writes /data/dist), read-only + # the built site at /data/dist (from 0.6.4 a symlink to /data/site/current, the newest + # build; /data/site/previous is the one before), read-only here - cloudhal-data:/srv/cloudhal:ro - caddy-data:/data - caddy-config:/config volumes: + # The installation. Named outright, so that this stack, `docker run -v cloudhal-data:/data …` and + # the guide's backup commands share one volume whatever the compose project is called. A stack + # created with the compose file of 0.6.3 has `_cloudhal-data` (`docker volume ls`): set + # CLOUDHAL_VOLUME to that name in .env to keep it (the guide, "Backup and restore"). A second + # stack on the same host must set a CLOUDHAL_VOLUME of its own: one volume, one container. cloudhal-data: + name: ${CLOUDHAL_VOLUME:-cloudhal-data} caddy-data: caddy-config: diff --git a/docs/guides/self-host.md b/docs/guides/self-host.md index cec461e5..c625f57f 100644 --- a/docs/guides/self-host.md +++ b/docs/guides/self-host.md @@ -18,64 +18,97 @@ curl -fsSL https://raw.githubusercontent.com/cloudhal-dev/cloudhal/main/deploy/d docker compose up -d ``` -Open http://localhost:8787. The first start on the empty volume creates a project from the blog starter (`CLOUDHAL_INIT`, named after `CLOUDHAL_SITE_NAME`) and runs the migrations; the first account that signs up is the Owner, after which sign-up closes and everyone else joins by invite. Invites and password-reset links are printed to the container's logs (`docker compose logs cloudhal`), not sent: mail is a follow-up for self-hosting. +Studio answers at http://localhost:8787. The first start on the empty volume creates a project from the blog starter (`CLOUDHAL_INIT`, named after `CLOUDHAL_SITE_NAME`) and runs the migrations. It also prints a one-time setup link to the container's logs (`docker compose logs cloudhal`), `http://localhost:8787/login#setup=`. Open it to create the Owner account. Without it sign-up is refused, so nobody who reaches the port first can claim the installation. After that sign-up closes and everyone else joins by invite. To choose the token yourself (for example from a secrets manager), set `CLOUDHAL_SETUP_TOKEN` to a random value of 32 characters or more, using only letters, digits and `- . _ ~` (`openssl rand -hex 32`). The log then shows the link with `` in place of the value. Invites are links: Studio's Team page creates one, shows it once, and you pass it on yourself; a forgotten password is reset with a link an Administrator creates there. Nothing is mailed (see [limits](#limits)). + +**Version.** The setup link needs image 0.6.4 or later. With an older image (a `CLOUDHAL_TAG` before 0.6.4, or `latest` until 0.6.4 is released) there is no setup token and the first account that signs up becomes the Owner: keep Studio on this machine (`CLOUDHAL_BIND=127.0.0.1`), do not start the caddy profile on a public host, and create the Owner first. Agents connect to the MCP endpoint at `/mcp` with an agent key made in Studio, as for a Cloudflare deployment ([connect an agent](../agents/clients.md)). ## Beyond localhost: a public origin -Studio's sign-in cookies and the API's origin checks are tied to the URL browsers use, so once Studio is reached through a DNS name, tell the container: +Studio's sign-in cookies and the API's origin checks are tied to the URL browsers use. Out of the box `CLOUDHAL_BASE_URL` is empty: sign-in accepts whatever address the browser used, and the log prints a one-line warning naming `--base-url`. Once Studio has a fixed address, set `CLOUDHAL_BASE_URL` to exactly that origin — a DNS name, or this machine on another port or a LAN address (`http://192.168.1.20:8787` with `CLOUDHAL_BIND=0.0.0.0`). It is then the only origin sign-in accepts and the origin of invite and password-reset links; with it wrong, Studio loads but sign-up and sign-in answer 403. Without it, the printed setup link names the container's own port 8787: with another `CLOUDHAL_PORT`, open it on that port. Behind a reverse proxy it is required: `cloudhal studio` refuses to start with `CLOUDHAL_BEHIND_PROXY=1` and no `CLOUDHAL_BASE_URL`, and the image refuses a `localhost` one there too (a `.env` kept from 0.6.3 sets `http://localhost:8787`). | Variable | What it does | | --- | --- | -| `CLOUDHAL_BASE_URL` | The public origin, e.g. `https://studio.example.com`: Better Auth's base URL (its cookies become `Secure`), a trusted origin for the API, and the URL the log prints. Origin only, no path. | -| `CLOUDHAL_BEHIND_PROXY` | `1` when a reverse proxy terminates TLS in front of the container: each request's scheme and host are then taken from the proxy's `X-Forwarded-Proto` / `X-Forwarded-Host`. Set it only when the proxy is the only way in, since the headers are then trusted. | +| `CLOUDHAL_BASE_URL` | The origin browsers open Studio at, e.g. `https://studio.example.com`. Required behind a reverse proxy, and recommended whenever Studio is reached from another machine. It is Better Auth's base URL (its cookies become `Secure` for https), the only origin sign-in accepts besides `CLOUDHAL_TRUSTED_ORIGINS`, the origin of invite and password-reset links, and the URL the log prints. Without it, sign-in accepts whatever address the browser used, and the log warns. Origin only, no path. | +| `CLOUDHAL_BEHIND_PROXY` | `1` when a reverse proxy terminates TLS in front of the container: each request's scheme and host are then taken from the proxy's `X-Forwarded-Proto` / `X-Forwarded-Host`, and the client's address for rate limits from the last `X-Forwarded-For` hop. Requires `CLOUDHAL_BASE_URL`: Studio refuses to start without it. Set it only when the proxy is the only way in, since the headers are then trusted. | | `CLOUDHAL_HOST` | The address the server binds inside the container; the image sets `0.0.0.0`. | -With the compose file's **`caddy` profile** both are handled for you: set `CLOUDHAL_BASE_URL` (Studio) and `CLOUDHAL_SITE_URL` (the public site) in `.env`, `CLOUDHAL_BEHIND_PROXY=1`, keep `CLOUDHAL_BIND=127.0.0.1`, point both names at the server and run +With the compose file's **`caddy` profile** both are handled for you: set `CLOUDHAL_BASE_URL` (Studio) and `CLOUDHAL_SITE_URL` (the public site) in `.env`, both `https://` origins, `CLOUDHAL_BEHIND_PROXY=1`, keep `CLOUDHAL_BIND=127.0.0.1`, point both names at the server, download the Caddyfile next to `compose.yml` (without it Docker creates an empty `./Caddyfile/` folder and Caddy cannot start; if that folder already exists, `rmdir Caddyfile` first) and run + +```bash +curl -fsSLO https://raw.githubusercontent.com/cloudhal-dev/cloudhal/main/deploy/docker/Caddyfile +``` ```bash docker compose --profile caddy up -d ``` -Caddy obtains and renews certificates for both hosts, proxies the Studio host to the container with the forwarded headers, and serves the built site for the site host. +Caddy obtains and renews certificates for both hosts, proxies the Studio host to the container with the forwarded headers, and serves the built site for the site host. Without the profile the two variables are not needed; with it, Caddy refuses to start while either is empty and says so in `docker compose logs caddy`. -With a **reverse proxy of your own** (nginx, Traefik, a cloud load balancer): proxy the Studio host to port 8787, pass `X-Forwarded-Proto` and `X-Forwarded-Host`, set `CLOUDHAL_BASE_URL` and `CLOUDHAL_BEHIND_PROXY=1`, and serve `/data/dist` from the volume for the site host. +With a **reverse proxy of your own** (nginx, Traefik, a cloud load balancer): proxy the Studio host to port 8787, set `CLOUDHAL_BASE_URL` and `CLOUDHAL_BEHIND_PROXY=1`, and send `X-Forwarded-Proto`, `X-Forwarded-Host` and `X-Forwarded-For`. The proxy must overwrite or append `X-Forwarded-For` with the client's address, never pass on what the client sent: its last hop is the address rate limits count (Caddy does this by default). For nginx: + +```nginx +location / { + proxy_pass http://127.0.0.1:8787; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Build chat streams server-sent events + proxy_buffering off; +} +``` + +Serve `/data/dist` from the volume for the site host. It is the stable path: a real folder with the 0.6.3 image, and from 0.6.4 on a symlink to the newest build, which the server must follow (nginx and Caddy do by default). A build replaces the symlink atomically, so the server never sees it missing — except for a moment on the first build after upgrading from 0.6.3, when the real folder moves aside and the symlink takes its place. A mount point of your own at `/data/dist` cannot be moved: `site-build` refuses it, so serve the volume's `/data/dist` instead. ## Environment -All of these are read by `cloudhal studio` in the container (`.env.example` lists them with their defaults): +All of these are read by `cloudhal studio` in the container, except those marked *compose*, which the compose file reads (`.env.example` lists them with their defaults): | Variable | Default | Meaning | | --- | --- | --- | | `CLOUDHAL_TAG` (compose) | `latest` | The image tag; see [upgrades](#upgrades). | -| `CLOUDHAL_BASE_URL` | `http://localhost:8787` | The public origin, as above. | -| `CLOUDHAL_BEHIND_PROXY` | `0` | Trust `X-Forwarded-*`. | -| `CLOUDHAL_BIND`, `CLOUDHAL_PORT` (compose) | `127.0.0.1`, `8787` | Which address and port of the host publish the container's port 8787. | -| `CLOUDHAL_SITE_URL` (caddy profile) | — | The public site's origin, served by Caddy. | +| `CLOUDHAL_VOLUME` (compose) | `cloudhal-data` | The name of the volume that holds the installation; see [backup and restore](#backup-and-restore). A second Cloudhal stack on the same host needs a name of its own (and its own `CLOUDHAL_PORT` and `CLOUDHAL_BASE_URL`): two containers on one volume are not supported. | +| `CLOUDHAL_BASE_URL` | empty | The origin browsers open Studio at, as above. Required with `CLOUDHAL_BEHIND_PROXY=1`: Studio refuses to start without it. Empty: sign-in accepts any address the browser uses, and the log warns. | +| `CLOUDHAL_BEHIND_PROXY` | `0` | Trust `X-Forwarded-*`. Requires `CLOUDHAL_BASE_URL`; Studio refuses to start without it. | +| `CLOUDHAL_SETUP_TOKEN` | empty | The first owner's setup token. Empty: a token is generated at each start while there is no account, and printed to the logs. | +| `CLOUDHAL_BIND`, `CLOUDHAL_PORT` (compose) | `127.0.0.1`, `8787` | Which address and port of the host publish the container's port 8787. A new port goes into `CLOUDHAL_BASE_URL` too. | +| `CLOUDHAL_SITE_URL` | empty | The public site's `https://` origin: Caddy serves the site there (caddy profile), and `site-build` makes its host the site's domain ([the site's origin](#the-public-site)). | | `CLOUDHAL_INIT` | `blog` | The starter a first start on an empty volume scaffolds. | | `CLOUDHAL_SITE_NAME` | `My site` | The name of that first site. | | `CLOUDHAL_AUTOMATIONS` | `on` | `off` stops scheduled automations; releases and gates go on ([schedules](schedules.md)). | | `CLOUDHAL_TRUSTED_ORIGINS` | empty | Extra origins the API accepts browser requests from, comma-separated. | -| `CLOUDHAL_AI_TOKEN`, `CLOUDFLARE_ACCOUNT_ID` | empty | Workers AI over REST for Build chat (both or neither); the token is never stored. | +| `BETTER_AUTH_SECRET` | created on the volume | Better Auth's secret. Unset, the container creates one at first start and keeps it at `/data/.cloudhal/auth-secret`; set it only to supply your own, and never empty. | +| `CLOUDHAL_AI_TOKEN`, `CLOUDFLARE_ACCOUNT_ID` | empty | Workers AI over REST for Build chat (both or neither). The token is kept in the container's memory, never written to the volume. | -Better Auth's secret is created on the volume at first start (`/data/.cloudhal/auth-secret`); set `BETTER_AUTH_SECRET` to supply your own. +**Secrets.** `CLOUDHAL_AI_TOKEN`, `BETTER_AUTH_SECRET` and `CLOUDHAL_SETUP_TOKEN` are secrets, and `.env` holds them once they are set: keep the file private (`chmod 600 .env`) and out of version control. The compose file passes each of them to the container only when it is set, so if the compose folder is a checkout, put the secrets in a second file outside it and name both files in `COMPOSE_ENV_FILES` for every `docker compose` command that creates or recreates the container (`up`, `pull && up`, `run`), for example in the shell profile. Give full paths: relative ones resolve against the current directory, not the compose folder. + +```bash +export COMPOSE_ENV_FILES=/path/to/cloudhal/.env,/etc/cloudhal/secrets.env +``` + +A one-off `docker compose --env-file … up -d` is not enough: the next command without the flags (an upgrade) recreates the container without the secrets, which signs everyone out when `BETTER_AUTH_SECRET` was yours and turns Build chat off. ## The public site -`cloudhal studio` serves Studio and the API, not the site. The site is `cloudhal build` output, static HTML of the published snapshot, which the image writes to `/data/dist` with +`cloudhal studio` serves Studio and the API, not the site. The site is `cloudhal build` output: static HTML of the published snapshot — or of the latest snapshot before the first publish (a build that renders a status page instead until then is RFC-0043's `cloudhal build --published`, not in this version). The image builds and publishes it with ```bash docker compose exec cloudhal site-build ``` -and Caddy (or your web server) serves from the volume. Run it after each publish; a cron line on the host (`*/15 * * * * docker compose -f /path/to/compose.yml exec -T cloudhal site-build`) keeps the site at most that far behind. Drafts, previews and scheduled releases are visible in Studio at once; the public pages change when the next build runs. Rebuilding on publish from inside the container is a follow-up issue; serving the site live from the snapshot, as the Cloudflare deployment does, is an RFC of its own. +and Caddy (or your web server) serves `/data/dist` from the volume. `cloudhal build` always writes the project's `dist` folder and empties it first, so `site-build` never runs it in `/data`: it copies the project's files to `/data/.site-build` (its `node_modules` and data folder linked, so the build reads the same database) and builds there. The finished build moves to `/data/site/`; then `/data/site/current`, `/data/site/previous` and `/data/dist` — symlinks to the newest build, the one before it, and `current` — are each replaced with one rename (`ln -sfn` and `mv -T`, which is atomic). Visitors see the previous build until the new one is complete, never a 404 or a half-written page — except for a moment on the first build after upgrading from 0.6.3, when the real `/data/dist` folder moves to `/data/site/legacy-0.6.3` (kept as `previous` for one build) and the symlink takes its place: the container's coreutils cannot swap a folder for a symlink in one step. A failed build changes nothing. The build before the current one is kept, and the compose file's Caddyfile serves a hashed asset (`/_astro/*`, `/_cloudhal/*`, `/media/*`) from it when the current build no longer has it, so a page opened before the swap still loads its scripts and media; older builds are removed. Never run an Astro build inside `/data` by hand (`cloudhal build`, `cloudhal deploy`): it would write through the `dist` symlink and empty the served build. Run `site-build` after each publish; a cron line on the host (`*/15 * * * * cd /path/to/cloudhal && docker compose exec -T cloudhal site-build`) keeps the site at most that far behind (`exec` runs in the running container, so it needs no `COMPOSE_ENV_FILES`). Drafts and scheduled releases show in Studio at once (the editor previews drafts); the public pages change when the next build runs. Signed preview links to unpublished content on the public site belong to the Cloudflare deployment; the container has no preview secret. Rebuilding on publish from inside the container is a follow-up issue; serving the site live from the snapshot, as the Cloudflare deployment does, is an RFC of its own. + +**The site's origin.** Canonical links, the sitemap, feeds and structured data need the public origin, which a static build cannot learn from a request. `cloudhal build` takes it from the site's primary domain, kept in `cloudhal.json` as `domains.site` and set with `cloudhal domain set ` (`docker compose exec cloudhal cloudhal domain set www.example.com`). `site-build` does this for you from `CLOUDHAL_SITE_URL`, an `https://` origin without a path or port, whenever it is set and differs (a name without a dot, such as `localhost`, or an IP address is skipped with a warning, and the build goes on). The domain then becomes the site's primary domain on the live site too, replacing one chosen in Studio. A published snapshot that already names a primary domain keeps it in that build: the new one reaches the pages with the next publish, and `site-build` says so. Without any domain the build's URLs say `localhost`. + +**Headers and the 404 page.** The build writes a `_headers` file — the security headers the Cloudflare deployment sends, and a cache rule for `/media/*` when the site has media — for hosts that read it, such as Cloudflare. Caddy does not: the compose file's Caddyfile marks `/media/*`, `/_astro/*` and `/_cloudhal/*` immutable on its own (files named by content hash, and only when the file exists, so a miss is never cached), and the security headers can be copied from `/data/dist/_headers` into it. A miss is answered with `/404.html`, which exists only when the site has a page at `/404` (the starter ships none); until then it is the web server's plain 404. The project's dependencies for the build (Astro, the renderer, Tailwind) are the image's own `node_modules`, linked into the project on the first `site-build`: no install, no network, and an image upgrade upgrades them. ## Upgrades -The image carries the same version as the npm packages: `0.6.3` is `@cloudhal/cli@0.6.3`. Tags: `X.Y.Z`, `X.Y` (the newest patch of a minor), `latest` (the newest release) and `edge` (nightly from `main`, for trying what is coming; never for a site you care about). `latest` follows patch releases; **pin the minor in production** (`CLOUDHAL_TAG=0.6`), since in 0.x a minor bump is a breaking change ([releasing](https://github.com/cloudhal-dev/cloudhal/blob/main/CONTRIBUTING.md#releasing)). +The image carries the same version as the npm packages: `0.6.4` is `@cloudhal/cli@0.6.4`. What this guide describes — the setup token, `site-build`'s layout under `/data/site` — needs 0.6.4 or later. Tags: `X.Y.Z`, `X.Y` (the newest patch of a minor), `latest` (the newest release) and `edge` (nightly from `main`, for trying what is coming; never for a site you care about). A tag moves only after the release's image has passed the smoke test on both architectures and a vulnerability scan. `latest` follows patch releases; **pin the minor in production** (`CLOUDHAL_TAG=0.6`), since in 0.x a minor bump is a breaking change ([releasing](https://github.com/cloudhal-dev/cloudhal/blob/main/CONTRIBUTING.md#releasing)). ```bash docker compose pull && docker compose up -d @@ -83,9 +116,86 @@ docker compose pull && docker compose up -d Database migrations run when the container starts, as they do on every `cloudhal studio` start. Going back to an older image over a newer database is not supported: take a backup first. +**From the compose files of 0.6.3.** The compose file now names the volume `cloudhal-data` outright, where the earlier file left the name to compose (`_cloudhal-data`; [issue #166](https://github.com/cloudhal-dev/cloudhal/issues/166)). In this order: + +1. **Keep your volume.** Set `CLOUDHAL_VOLUME=_cloudhal-data` in `.env` (see [backup and restore](#backup-and-restore)). Without it `docker compose up` creates an empty `cloudhal-data` and starts a fresh site next to your data. +2. **Fix two values a 0.6.3 `.env` carries**, before anything starts: + - `CLOUDHAL_BASE_URL=http://localhost:8787`: behind a proxy, set the public origin. The new entrypoint refuses `CLOUDHAL_BEHIND_PROXY=1` with a localhost base URL, and the container would restart in a loop. + - `CLOUDHAL_SITE_URL=https://www.example.com`: set the site's origin, or leave it empty (`site-build` refuses to make an example domain the site's). +3. **Download the new files** next to `.env`: `compose.yml`, the `Caddyfile` for the caddy profile (an old one keeps working, but the previous-build fallback for hashed assets needs the new one; if `./Caddyfile/` is a folder, `rmdir Caddyfile` first), and `.env.example` to compare with your `.env` (saved as `.env.example`, never over `.env`): + +```bash +curl -fsSLO https://raw.githubusercontent.com/cloudhal-dev/cloudhal/main/deploy/docker/compose.yml +``` + +```bash +curl -fsSLO https://raw.githubusercontent.com/cloudhal-dev/cloudhal/main/deploy/docker/Caddyfile +``` + +```bash +curl -fsSLO https://raw.githubusercontent.com/cloudhal-dev/cloudhal/main/deploy/docker/.env.example +``` + +4. **Pull and check the image.** This check never touches the named volume: + +```bash +docker compose pull +``` + +```bash +docker run --rm --entrypoint grep "$(docker compose config --images | grep cloudhal-dev/cloudhal)" -c site/current /opt/cloudhal/bin/site-build +``` + + A number above 0 means 0.6.4 or later. A 0 means the image is still 0.6.3 (0.6.4 is not on your tag yet): the new files work with it too, `/data/dist` stays a real folder, and the layout changes with the first build of a 0.6.4 image. +5. **Start the stack** as usual (`docker compose up -d`, or `docker compose --profile caddy up -d`). + +The site stays at `/data/dist` throughout. The 0.6.3 `site-build` wrote a real folder there. The first build of 0.6.4 moves it to `/data/site/legacy-0.6.3`, kept as `previous` for one build, and puts the symlink in its place: `/data/dist` is missing only between those two renames. An old Caddyfile or an nginx root of `/data/dist` keeps working. + ## Backup and restore -The volume is the installation: stop the container and copy it (`docker run --rm -v cloudhal-data:/data -v "$PWD":/backup alpine tar czf /backup/cloudhal-data.tgz -C /data .`), or restore the same way into an empty volume. For one site rather than the installation, `cloudhal export` writes a portable `.cloudhal` archive of its graph, snapshots and media (never accounts, comments or secrets) and `cloudhal import` reads it back ([site archives](site-archives.md)): +The volume is the installation. The compose file names it `cloudhal-data`, as the Dockerfile's `docker run -v cloudhal-data:/data` example does, so one set of commands serves both; `docker volume ls` shows the volumes you have. A stack created with the compose file of 0.6.3 has `_cloudhal-data` instead (`cloudhal_cloudhal-data` after the quick start's `mkdir cloudhal`): set `CLOUDHAL_VOLUME` to that name in `.env` and use it in place of `cloudhal-data` below. Stop the container first, so that SQLite's write-ahead log is folded into the database file: + +```bash +docker compose stop cloudhal +``` + +```bash +docker run --rm -v cloudhal-data:/data -v "$PWD":/backup alpine tar czf /backup/cloudhal-data.tgz -C /data . +``` + +```bash +docker compose start cloudhal +``` + +`docker run -v :/data` creates an empty volume of that name when none exists, so a wrong name archives nothing and exits 0: look inside the archive (`tar tzf cloudhal-data.tgz | head`) before relying on it. To restore, check the archive, stop the stack, and unpack into a volume that is really empty — `docker volume create` on a name that exists does nothing and exits 0, and unpacking over a live installation leaves newer files (SQLite's `data.db-wal` among them) next to the restored ones: + +```bash +tar tzf cloudhal-data.tgz | head +``` + +```bash +docker compose down +``` + +```bash +docker volume rm cloudhal-data +``` + +```bash +docker volume create cloudhal-data +``` + +```bash +docker run --rm -v cloudhal-data:/data -v "$PWD":/backup alpine tar xzf /backup/cloudhal-data.tgz -C /data +``` + +```bash +docker compose up -d +``` + +To keep the current volume as it is, unpack into a new one instead (`docker volume create cloudhal-data-restored`, the same `tar xzf` with that name) and set `CLOUDHAL_VOLUME=cloudhal-data-restored` in `.env` before `docker compose up -d`. + +For one site rather than the installation, `cloudhal export` writes a portable `.cloudhal` archive of its graph, snapshots and media (never accounts, comments or secrets) and `cloudhal import` reads it back ([site archives](site-archives.md)): ```bash docker compose exec cloudhal cloudhal export --out /data/backup.cloudhal @@ -101,8 +211,9 @@ docker compose cp cloudhal:/data/backup.cloudhal . ## Limits -- **Mail** is printed to the logs; there is no SMTP sender yet. +- **Mail.** There is no mail sender in the container: invites and password resets are links made on Studio's Team page and passed on by hand. With `mail` in `cloudhal.json` (`"mail": { "from": "studio@example.com" }`) the container prints every mail it would send to its logs instead (`docker compose logs cloudhal`) and Studio offers delivery by email too — good for trying the flows, not for people who expect mail. An SMTP sender is a follow-up. - **`render_page`** (the agents' screenshot tool) needs Chrome, which the image does not carry: the tool reports itself unavailable. A `-chrome` variant is a possible follow-up. +- **`--no-auth`** is refused unless `--host` is loopback and `--behind-proxy` is off, so it cannot be used in the container, which binds `0.0.0.0`. - **One process.** `cloudhal studio` runs the scheduler's tick, the comments job and analytics rollups itself, on the project's SQLite file; several containers on one volume are not supported. - **The site is static** between builds, as above. - **Deploying to Cloudflare** from the container is not set up (the Cloudflare adapter is not installed); use the CLI on your machine for that. @@ -110,7 +221,7 @@ docker compose cp cloudhal:/data/backup.cloudhal . ## Building the image yourself ```bash -docker build --build-arg CLOUDHAL_VERSION=0.6.3 -t cloudhal:local deploy/docker +docker build --build-arg CLOUDHAL_VERSION=0.6.4 -t cloudhal:local deploy/docker ``` -builds a release version from npm; `node scripts/pack-packages.ts deploy/docker/tarballs` followed by the same build without the argument builds `edge` from a checkout. `node scripts/docker-smoke.ts --image cloudhal:local` runs what CI runs: an empty volume, the health check, the first owner through the API, a publish and `site-build`. +builds a release version from npm (0.6.4 or later: the smoke test expects the setup token); `node scripts/pack-packages.ts deploy/docker/tarballs` followed by the same build without the argument builds `edge` from a checkout. `node scripts/docker-smoke.ts --image cloudhal:local` runs what CI runs: an empty volume, the health check, Studio's page, the first owner through the setup token (refused without it), Studio's page, an anonymous `/mcp` call, a publish and `site-build` twice (the second build swaps the served tree). `--platform linux/arm64` runs the other architecture's image where Docker can (Apple silicon, or QEMU). diff --git a/scripts/docker-smoke.ts b/scripts/docker-smoke.ts index c3272c16..22dd2ee8 100644 --- a/scripts/docker-smoke.ts +++ b/scripts/docker-smoke.ts @@ -1,10 +1,12 @@ /** * What a self-hoster gets (docs/guides/self-host.md, #143): runs the image with an empty volume, - * waits for its HEALTHCHECK, then — through the API, as Studio would — creates the first owner, - * lists the scaffolded site, publishes it, builds the public site with `site-build` and reads the - * built home page. Run by .github/workflows/docker.yml after every build, and by hand: + * waits for its HEALTHCHECK, then — through the API, as Studio would — checks that the first owner + * can sign up only with the setup token (#165), that Studio's page is served and that `/mcp` + * refuses an anonymous call, lists the scaffolded site, publishes it, builds the public site with + * `site-build` twice (the second build swaps the served tree in place, #166) and reads the built + * home page. Run by .github/workflows/docker.yml after every build, and by hand: * - * node scripts/docker-smoke.ts [--image ] [--port ] [--keep] + * node scripts/docker-smoke.ts [--image ] [--platform ] [--port ] [--keep] */ import { execFileSync, spawnSync } from "node:child_process"; @@ -13,6 +15,7 @@ const arg = (name: string): string | undefined => { return i === -1 ? undefined : process.argv[i + 1]; }; const image = arg("image") ?? "cloudhal:local"; +const platform = arg("platform"); const port = Number(arg("port") ?? 18787); const keep = process.argv.includes("--keep"); const name = `cloudhal-smoke-${process.pid}`; @@ -20,6 +23,10 @@ const origin = `http://localhost:${port}`; // #165: the first account needs the setup token; the container gets one so the test need not read // the printed link from its logs. const setupToken = `smoke-setup-token-${crypto.randomUUID()}`; +/** The built home page, as the starter renders the site name. */ +const HOME_H1 = /]*>Smoke Site<\/h1>/; +/** The served path: a symlink to /data/site/current, the newest build (#166). */ +const SITE = "/data/dist"; const docker = (args: string[], input?: string): string => execFileSync("docker", args, { encoding: "utf8", ...(input ? { input } : {}) }).trim(); @@ -66,11 +73,21 @@ function expect(ok: boolean, what: string, detail?: unknown): void { if (!ok) throw new Error(`${what}${detail === undefined ? "" : `: ${JSON.stringify(detail)}`}`); } +/** Runs `site-build` in the container; returns the build `current` points at afterwards. */ +function siteBuild(which: string): string { + const r = spawnSync("docker", ["exec", name, "site-build"], { stdio: "inherit" }); + expect(r.status === 0, `site-build (${which}) exited ${r.status}`); + const served = docker(["exec", name, "readlink", SITE]); + expect(served === "site/current", `${SITE} links to site/current`, served); + return docker(["exec", name, "readlink", "/data/site/current"]); +} + docker([ "run", "-d", "--name", name, + ...(platform ? ["--platform", platform] : []), "-p", `127.0.0.1:${port}:8787`, "-e", @@ -112,7 +129,26 @@ try { "the first account is the Owner", me.body, ); - step("the first owner signed up through the API"); + step("the first owner signed up with the setup token, not without it"); + + // Studio's index.html (the SPA fallback), not an API 404 and not an empty page. + const home = await fetch(`${origin}/`); + const homeHtml = await home.text(); + expect(home.status === 200, "GET /", home.status); + expect( + (home.headers.get("content-type") ?? "").includes("text/html") && + homeHtml.includes("Cloudhal Studio"), + "GET / is Studio's index.html", + { type: home.headers.get("content-type"), start: homeHtml.slice(0, 120) }, + ); + step("GET / serves Studio"); + + // The MCP endpoint is public but never anonymous (ADR-0011). + const mcp = await api("POST", "/mcp", { + body: { jsonrpc: "2.0", id: 1, method: "initialize", params: {} }, + }); + expect(mcp.status === 401, "POST /mcp without credentials is refused with 401", mcp.status); + step("POST /mcp without credentials answers 401"); const sites = await api("GET", "/api/sites", { cookie }); const site = (sites.body as { id: string; name: string }[])[0]; @@ -125,10 +161,44 @@ try { expect(publish.status === 200 || publish.status === 201, "publish", publish.body); step(`site ${site.id} published`); - spawnSync("docker", ["exec", name, "site-build"], { stdio: "inherit" }); - const html = docker(["exec", name, "cat", "/data/dist/index.html"]); - expect(/]*>Smoke Site<\/h1>/.test(html), "the built home page shows the site name"); - step("site-build wrote the public site to /data/dist"); + // A volume from 0.6.3 has a real /data/dist folder: the first build moves it aside, keeps it as + // site/previous, and puts the symlink in its place (#166). + docker(["exec", name, "sh", "-c", "mkdir -p /data/dist && touch /data/dist/marker"]); + const first = siteBuild("first"); + const resolved = docker(["exec", name, "readlink", "-f", SITE]); + expect(resolved.startsWith("/data/site/"), `${SITE} resolves under /data/site/`, resolved); + const legacy = spawnSync("docker", ["exec", name, "test", "-e", "/data/site/previous/marker"]); + expect(legacy.status === 0, "site/previous is the moved 0.6.3 folder, marker included"); + step("the first site-build moved a 0.6.3 /data/dist aside as site/previous"); + expect( + HOME_H1.test(docker(["exec", name, "cat", `${SITE}/index.html`])), + "the built home page shows the site name", + ); + step(`site-build published the site: ${SITE} -> site/current -> ${first}`); + + // A second build swaps the served tree in one rename and keeps the one before it, whose pages + // may still load their assets (#166). + const second = siteBuild("second"); + expect(second !== first, "the second build is a new tree", { first, second }); + const kept = docker([ + "exec", + name, + "sh", + "-c", + "ls -1 /data/site | grep -v -x -e current -e previous", + ]); + expect( + kept === [first, second].sort().join("\n"), + "the newest build and the one before it are kept", + kept, + ); + const previous = docker(["exec", name, "readlink", "/data/site/previous"]); + expect(previous === first, "site/previous links to the first build", previous); + expect( + HOME_H1.test(docker(["exec", name, "cat", `${SITE}/index.html`])), + "the home page is served after the swap", + ); + step("a second site-build swapped the served tree and kept the previous one"); console.log("\nDocker smoke test passed."); } catch (e) { console.error(`\n✗ ${e instanceof Error ? e.message : String(e)}\n--- container logs ---`);