From f6d34130956f14e1bdfacf1a958901b3731d812e Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 1 Nov 2026 00:04:00 +0000 Subject: [PATCH 01/33] Rename CI image labels to Ubuntu 26.04 (#10934) Rename the `ubuntu24-*-pre-v2` image labels to `ubuntu26-*-pre-v2`, to match the AMIs that vortex-data/.github-private#39 moves to Ubuntu 26.04. The release binary fallback runner moves to `ubuntu-26.04`. GPU jobs stay on `ubuntu24-gpu-x64` (no 26.04 GPU image upstream). --------- Signed-off-by: Robert Kruszewski --- .github/runs-on.yml | 4 ++-- .github/workflows/ci.yml | 32 ++++++++++++------------- .github/workflows/compat-gen-upload.yml | 4 ++-- .github/workflows/compat-validation.yml | 2 +- .github/workflows/develop-bench.yml | 4 ++-- .github/workflows/musl.yml | 2 +- .github/workflows/package.yml | 4 ++-- .github/workflows/pr-bench-runner.yml | 4 ++-- .github/workflows/publish-dry-runs.yml | 8 +++---- .github/workflows/release-binaries.yml | 8 +++---- .github/workflows/rust-instrumented.yml | 10 ++++---- .github/workflows/s3-measurements.yml | 2 +- .github/workflows/sql-bench-matrix.yml | 4 ++-- vortex-ffi/test/CMakeLists.txt | 8 +++++++ 14 files changed, 52 insertions(+), 44 deletions(-) diff --git a/.github/runs-on.yml b/.github/runs-on.yml index b946f8031ec..714b9d15ad4 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -1,12 +1,12 @@ _extends: .github-private images: - ubuntu24-full-x64-pre-v2: + ubuntu26-full-x64-pre-v2: platform: "linux" arch: "x64" name: "vortex-ci-v2-x64-*" owner: "245040174862" - ubuntu24-full-arm64-pre-v2: + ubuntu26-full-arm64-pre-v2: platform: "linux" arch: "arm64" name: "vortex-ci-v2-arm64-*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d01053937..1ed35f27f74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: if: github.event.pull_request.draft != true runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=lightweight-checks', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-small/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=lightweight-checks', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 10 steps: @@ -111,7 +111,7 @@ jobs: if: github.event.pull_request.draft != true runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-test', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-large/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=python-test', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 env: @@ -264,7 +264,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-docs/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-docs/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -293,7 +293,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner={1}/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) + && format('runs-on={0}/region=us-east-1/runner={1}/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) || 'ubuntu-latest' }} strategy: fail-fast: false @@ -338,7 +338,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -358,7 +358,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-msrv', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-msrv', github.run_id) || 'ubuntu-latest' }} env: RUSTFLAGS: "-A warnings" @@ -382,7 +382,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-large/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -451,7 +451,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -528,7 +528,7 @@ jobs: github.event.pull_request.draft != true && !cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') && github.repository == 'vortex-data/vortex' timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-ci-test/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64 + runs-on: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-ci-test/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64 steps: - parallel: - uses: runs-on/action@v2 @@ -572,17 +572,17 @@ jobs: - name: avx2 runner: amd64-ci-medium family: c6a.2xlarge - image: ubuntu24-full-x64-pre-v2 + image: ubuntu26-full-x64-pre-v2 expect: "sse2,popcnt,bmi1,bmi2,avx,avx2" - name: avx512 runner: amd64-ci-medium family: c6i.2xlarge - image: ubuntu24-full-x64-pre-v2 + image: ubuntu26-full-x64-pre-v2 expect: "sse2,popcnt,bmi1,bmi2,avx,avx2,avx512f,avx512bw,avx512vbmi2,avx512vpopcntdq" - name: sve runner: arm64-medium family: c8g.2xlarge - image: ubuntu24-full-arm64-pre-v2 + image: ubuntu26-full-arm64-pre-v2 expect: "neon,sve,sve2,sve2-bitperm" runs-on: >- runs-on=${{ github.run_id }}/region=us-east-1/runner=${{ matrix.config.runner }}/family=${{ matrix.config.family }}/spot=price-capacity-optimized/image=${{ matrix.config.image }}/extras=s3-cache/tag=rust-test-cpu-features-${{ matrix.config.name }} @@ -612,7 +612,7 @@ jobs: if: github.event.pull_request.draft != true runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=java', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=java', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -662,7 +662,7 @@ jobs: if: github.event.pull_request.draft != true && !cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-sql-logic-test/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=sql-logic-test', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-sql-logic-test/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=sql-logic-test', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -691,7 +691,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=c-cxx-generated', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=c-cxx-generated', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -833,7 +833,7 @@ jobs: if: github.event.pull_request.draft != true && !cancelled() runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=integration-smoke', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=integration-smoke', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: diff --git a/.github/workflows/compat-gen-upload.yml b/.github/workflows/compat-gen-upload.yml index 63346a7dac5..e927a7e7b97 100644 --- a/.github/workflows/compat-gen-upload.yml +++ b/.github/workflows/compat-gen-upload.yml @@ -25,7 +25,7 @@ jobs: dry-run: runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=compat-gen-dry-run', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=compat-gen-dry-run', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 permissions: @@ -78,7 +78,7 @@ jobs: if: inputs.confirm_upload == 'yes' runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=compat-gen-upload', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=compat-gen-upload', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 environment: compat-upload diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index 355043c3625..56ff88d1f84 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -25,7 +25,7 @@ jobs: compat-test: runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=compat-validation', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=compat-validation', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index f02f456435a..be17f19bad0 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -18,7 +18,7 @@ jobs: # forever. With the long `family` list and the longest benchmark id, this label is close. runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) || 'ubuntu-latest' }} strategy: fail-fast: false @@ -68,7 +68,7 @@ jobs: timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag={1}{2}', github.run_id, matrix.benchmark.id, matrix.benchmark.variant_id) + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag={1}{2}', github.run_id, matrix.benchmark.id, matrix.benchmark.variant_id) || 'ubuntu-latest' }} strategy: fail-fast: false diff --git a/.github/workflows/musl.yml b/.github/workflows/musl.yml index f005e07e558..722bebc304d 100644 --- a/.github/workflows/musl.yml +++ b/.github/workflows/musl.yml @@ -28,7 +28,7 @@ jobs: timeout-minutes: 90 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-build-musl', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-large/image=ubuntu26-full-x64-pre-v2/tag=rust-build-musl', github.run_id) || 'ubuntu-latest' }} container: image: ghcr.io/rust-lang/rust:1-alpine3.24 diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index c12b12b8d8e..a6e0b8f1b78 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -182,9 +182,9 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=prepare-java-linux-amd64 + runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=prepare-java-linux-amd64 - target: aarch64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=prepare-java-linux-arm64 + runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-medium/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag=prepare-java-linux-arm64 runs-on: ${{ matrix.runner }} steps: - uses: runs-on/action@v2 diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 1689f8e7fa0..4e7f73b2af2 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -48,7 +48,7 @@ jobs: # forever. With the long `family` list and the longest benchmark id, this label is close. runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu24-full-arm64-pre-v2/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu26-full-arm64-pre-v2/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -93,7 +93,7 @@ jobs: timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu24-full-arm64-pre-v2/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu26-full-arm64-pre-v2/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: # `extras=s3-cache` points ACTIONS_RESULTS_URL at the RunsOn artifact proxy, but only diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index 8e12b1e97fb..1c8f5733241 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -28,7 +28,7 @@ jobs: if: github.event.pull_request.draft != true runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-wheel-build', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=python-wheel-build', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -82,9 +82,9 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=check-java-publish-build-amd64 + runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=check-java-publish-build-amd64 - target: aarch64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=check-java-publish-build-arm64 + runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-medium/image=ubuntu26-full-arm64-pre-v2/extras=s3-cache/tag=check-java-publish-build-arm64 runs-on: ${{ matrix.runner }} steps: - parallel: @@ -123,7 +123,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-publish-dry-run', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-xsmall/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-publish-dry-run', github.run_id) || 'ubuntu-latest' }} steps: - parallel: diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 21f60f2255d..c3713473a3f 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -31,12 +31,12 @@ jobs: # cargo-zigbuild with glibc 2.31 pinning, matching the Python wheel # and JNI builds. - target: aarch64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=release-vx-aarch64-linux - fallback_runner: ubuntu-24.04 + runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=release-vx-aarch64-linux + fallback_runner: ubuntu-26.04 archive: tgz - target: x86_64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=release-vx-amd64-linux - fallback_runner: ubuntu-24.04 + runner: runs-on=${{ github.run_id }}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=release-vx-amd64-linux + fallback_runner: ubuntu-26.04 archive: tgz steps: - uses: runs-on/action@v2 diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index a90110cb7d4..db04cd4ca5d 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -36,7 +36,7 @@ jobs: - suite: tests runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-large/volume=200gb/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite) + && format('runs-on={0}/region=us-east-1/runner=amd64-large/volume=200gb/image=ubuntu26-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite) || 'ubuntu-latest' }} env: RUSTFLAGS: "-Cinstrument-coverage -A warnings" @@ -113,7 +113,7 @@ jobs: id-token: write runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -158,7 +158,7 @@ jobs: name: "Rust tests (${{ matrix.sanitizer }})" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-test-sanitizer', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/extras=s3-cache/tag=rust-test-sanitizer', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 env: @@ -238,7 +238,7 @@ jobs: RUST_LIB_BACKTRACE: 0 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) || 'ubuntu-latest' }} steps: - parallel: @@ -287,7 +287,7 @@ jobs: if: github.event.pull_request.draft != true runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id) + && format('runs-on={0}/region=us-east-1/runner=amd64-medium/image=ubuntu26-full-x64-pre-v2/tag=rust-miri', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 env: diff --git a/.github/workflows/s3-measurements.yml b/.github/workflows/s3-measurements.yml index f52b0c0d857..5e6156b2540 100644 --- a/.github/workflows/s3-measurements.yml +++ b/.github/workflows/s3-measurements.yml @@ -29,7 +29,7 @@ concurrency: jobs: collect: timeout-minutes: 780 - runs-on: runs-on=${{ github.run_id }}/runner=bench-dedicated/family=c8gd.16xlarge/spot=false/image=ubuntu24-full-arm64-pre-v2/tag=s3-measure/extras=s3-cache + runs-on: runs-on=${{ github.run_id }}/runner=bench-dedicated/family=c8gd.16xlarge/spot=false/image=ubuntu26-full-arm64-pre-v2/tag=s3-measure/extras=s3-cache env: AWS_REGION: us-east-1 REMOTE_ROOT: s3://vortex-ci-benchmark-datasets/s3-measurements/${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index f95a6c1ff3d..7697aaa9af6 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -49,7 +49,7 @@ jobs: FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu24-full-arm64-pre-v2/tag=build{1}', github.run_id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+c8gn.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge+x8g.8xlarge+i8g.8xlarge/image=ubuntu26-full-arm64-pre-v2/tag=build{1}', github.run_id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -112,7 +112,7 @@ jobs: runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family={1}/image=ubuntu24-full-arm64-pre-v2/tag={2}{3}', github.run_id, inputs.machine_type, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family={1}/image=ubuntu26-full-arm64-pre-v2/tag={2}{3}', github.run_id, inputs.machine_type, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 diff --git a/vortex-ffi/test/CMakeLists.txt b/vortex-ffi/test/CMakeLists.txt index 5daf85f5244..4ddfa7b6f11 100644 --- a/vortex-ffi/test/CMakeLists.txt +++ b/vortex-ffi/test/CMakeLists.txt @@ -24,6 +24,14 @@ target_compile_options(vortex_ffi_test PRIVATE -Wall -Wextra -Wpedantic) if(VORTEX_WARNINGS_AS_ERRORS) target_compile_options(vortex_ffi_test PRIVATE -Werror) endif() + +# Catch2 test macros expand to __COUNTER__, which newer clang flags as a C2y extension under +# -Wpedantic. Check the positive flag: GCC accepts unknown -Wno-* flags without a diagnostic. +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag(-Wc2y-extensions VORTEX_HAS_WC2Y_EXTENSIONS) +if(VORTEX_HAS_WC2Y_EXTENSIONS) + target_compile_options(vortex_ffi_test PRIVATE -Wno-c2y-extensions) +endif() target_link_libraries(vortex_ffi_test PRIVATE Vortex::ffi Catch2::Catch2WithMain From 65c992d92f582ff3eb55df1028117e94c321af42 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 1 Nov 2026 00:33:31 +0000 Subject: [PATCH 02/33] Point the Ubuntu 26.04 CI image labels at the Ubuntu 26.04 AMIs (#11075) #10934 renamed the Linux CI image labels to `ubuntu26-full-{x64,arm64}-pre-v2`, but the `name` patterns in `.github/runs-on.yml` still match the old AMIs: | label | pattern before | AMI that CI boots | |---|---|---| | `ubuntu26-full-x64-pre-v2` | `vortex-ci-v2-x64-*` | `vortex-ci-v2-x64-20261030180912` (old cloud-init image) | | `ubuntu26-full-arm64-pre-v2` | `vortex-ci-v2-arm64-*` | old cloud-init image | The AMI build in `.github-private` now publishes `vortex-ci-v2-ubuntu26-{x64,arm64}-*`. This PR changes the patterns to match those names. Signed-off-by: Robert Kruszewski --- .github/runs-on.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 714b9d15ad4..819f2a91a52 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -4,12 +4,12 @@ images: ubuntu26-full-x64-pre-v2: platform: "linux" arch: "x64" - name: "vortex-ci-v2-x64-*" + name: "vortex-ci-v2-ubuntu26-x64-*" owner: "245040174862" ubuntu26-full-arm64-pre-v2: platform: "linux" arch: "arm64" - name: "vortex-ci-v2-arm64-*" + name: "vortex-ci-v2-ubuntu26-arm64-*" owner: "245040174862" windows25-full-x64-pre: platform: "windows" From 9085458de99841b9974a3adb5e5dfa2b62585de2 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 2 Nov 2026 10:19:08 +0000 Subject: [PATCH 03/33] Run the remaining benchmarks on the V2 scan (#11061) Follows #11055 (FineWeb). This moves every remaining benchmark except StatPopGen and PolarSignals onto the planner-driven V2 scan. After this PR, those two are the only benchmarks still on V1. Signed-off-by: Joe Isaacs --- bench-orchestrator/bench_orchestrator/config.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index cc8b31bc2a9..904ca5683f0 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -95,19 +95,19 @@ class Benchmark(Enum): # Whether each benchmark runs its Vortex scans on the planner-driven V2 scan (`True`) or on V1. # Every benchmark is listed so its scan version is explicit. BENCHMARK_SCAN_V2: dict[Benchmark, bool] = { - Benchmark.APPIAN: False, + Benchmark.APPIAN: True, Benchmark.TPCH: True, Benchmark.TPCDS: True, Benchmark.CLICKBENCH: True, - Benchmark.CLICKBENCH_SORTED: False, + Benchmark.CLICKBENCH_SORTED: True, Benchmark.FINEWEB: True, - Benchmark.GHARCHIVE: False, - Benchmark.JOB: False, + Benchmark.GHARCHIVE: True, + Benchmark.JOB: True, Benchmark.POLARSIGNALS: False, - Benchmark.PUBLIC_BI: False, + Benchmark.PUBLIC_BI: True, Benchmark.STATPOPGEN: False, - Benchmark.SPATIALBENCH: False, - Benchmark.VORTEX_QUERIES: False, + Benchmark.SPATIALBENCH: True, + Benchmark.VORTEX_QUERIES: True, } From 085c87148431f3a9d90c278e9f2f990a940da887 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 2 Nov 2026 10:43:10 +0000 Subject: [PATCH 04/33] Better READMEs and user docs (#10960) - Remove "Future work" sections from docs - Move docs on how to integrate DataFusion as a crate into "Developer docs" - Make language bindings docs smaller, update C++ API implementation. - Add more SQL examples on Duckdb user guide page. - Make vortex-ffi and vortex-sqllogictest READMEs less verbose. --------- Signed-off-by: Mikhail Kot --- .../integrations/datafusion.md | 73 +++--- docs/developer-guide/language-bindings.md | 28 +-- docs/user-guide/datafusion.md | 87 +------- docs/user-guide/duckdb.md | 19 +- docs/user-guide/index.md | 14 +- docs/user-guide/work-in-progress.md | 19 -- vortex-duckdb/README.md | 4 +- vortex-ffi/CMakeLists.txt | 20 +- vortex-ffi/README.md | 72 +++--- vortex-sqllogictest/README.md | 211 +++--------------- 10 files changed, 148 insertions(+), 399 deletions(-) delete mode 100644 docs/user-guide/work-in-progress.md diff --git a/docs/developer-guide/integrations/datafusion.md b/docs/developer-guide/integrations/datafusion.md index 7a21575cc68..eb85dcbc1ee 100644 --- a/docs/developer-guide/integrations/datafusion.md +++ b/docs/developer-guide/integrations/datafusion.md @@ -1,8 +1,53 @@ # DataFusion -The `vortex-datafusion` crate integrates Vortex as a native file format in Apache DataFusion. -It registers a `FileFormat` and `FileSource` so that DataFusion's query planner can discover, -partition, and scan Vortex files using the same machinery it uses for Parquet and CSV. +Vortex provides a +[`vortex-datafusion`](https://crates.io/crates/vortex-datafusion) crate for +`.vortex` file support in DataFusion. + +## Setup and registration + +Add the dependency: + +```toml +[dependencies] +vortex-datafusion = "" +``` + +Register the Vortex format with a `SessionContext`: + +:::{literalinclude} ../../../vortex-datafusion/src/persistent/tests.rs +:language: rust +:dedent: +:start-after: [setup] +:end-before: [setup] +::: + +Create an external table and query it: + +:::{literalinclude} ../../../vortex-datafusion/src/persistent/tests.rs +:language: rust +:dedent: +:start-after: [create] +:end-before: [create] +::: + +You can also register a `ListingTable` directly: + +:::{literalinclude} ../../../vortex-datafusion/examples/vortex_table.rs +:language: rust +:dedent: +:start-after: [register] +:end-before: [register] +::: + +Write query results to Vortex using `INSERT INTO`: + +:::{literalinclude} ../../../vortex-datafusion/src/persistent/tests.rs +:language: rust +:dedent: +:start-after: [write] +:end-before: [write] +::: ## Registration @@ -25,18 +70,6 @@ Layout readers are cached across partitions using a shared concurrent map keyed This avoids redundant footer parsing when the same file is accessed by multiple partitions or repeated queries. -## Threading Model - -DataFusion runs on Tokio, and the Vortex integration operates entirely within that async -context. The Vortex session is configured with `with_tokio()` to capture the current Tokio -runtime handle. All I/O -- file opens, segment reads, object store fetches -- is dispatched as -Tokio tasks and scheduled across Tokio's multi-threaded executor. - -DataFusion's physical executor manages parallelism by assigning partitions to its own task pool. -Each partition opens its files and drives a `ScanBuilder` that returns an async stream of -record batches. Multiple partitions execute concurrently, with DataFusion controlling the degree -of parallelism. - ## Filter and Projection Pushdown The integration converts DataFusion physical expressions into Vortex expressions using an @@ -65,13 +98,3 @@ schema. Vortex arrays produced by the scan are converted to Arrow `RecordBatch`es for consumption by DataFusion. Batches are sliced to respect DataFusion's configured batch size preference. - -## Future Work - -The current integration builds directly on the `ScanBuilder` and layout reader APIs. Future work -will migrate it to use the [Scan API](/concepts/scanning) `Source` trait, which will simplify -the integration by providing a standard interface for file discovery, partitioning, and pushdown -that is shared across all engine integrations. - -Better support for dynamic expressions would enable use-cases like top-k queries, where the scan's -filter expression is updated during execution as the query engine discovers tighter bounds. diff --git a/docs/developer-guide/language-bindings.md b/docs/developer-guide/language-bindings.md index 8905364910b..8c305a3f96b 100644 --- a/docs/developer-guide/language-bindings.md +++ b/docs/developer-guide/language-bindings.md @@ -67,14 +67,13 @@ layout plugins, and extension DTypes. ## Per-Language Status -| Language | Current Tier | Target Tier | Technology | Notes | -|---------------|--------------|-------------|------------|-----------------------------------------------| -| Rust | 3 | 3 | Native | Future: stable plugin API via C ABI | -| Python | ~2 | 3 | PyO3 | Already has native expressions + array access | -| C | ~1 | 2 | cbindgen | Foundation ABI for all non-Rust bindings | -| C++ | ~1 | 2 | cxx -> C | Migrate from cxx to wrap C API | -| Java (JNI) | ~1 | 1 | JNI | Broad JDK compatibility, Arrow-based | -| Java (Panama) | — | 2 | Panama FFI | Direct C ABI access, requires JDK 22+ | +| Language | Current Tier | Technology | +|---------------|--------------|------------| +| Rust | 3 | Native | +| Python | ~2 | PyO3 | +| C | ~1 | cbindgen | +| C++ | ~1 | C API | +| Java (JNI) | ~1 | JNI | ### Rust @@ -96,21 +95,8 @@ evolves with the project and should be statically linked. In the future, a subse be flagged as stable for use via dynamic linking. The target is Tier 2, which requires array tree inspection, compute execution, and stabilized error handling and memory ownership conventions. -### C++ - -C++ bindings currently use [cxx](https://cxx.rs) for Rust interop. The plan is to migrate to -wrapping the C API directly, providing RAII wrappers and CMake integration. Target is Tier 2. - ### Java (JNI) Java JNI bindings provide Tier ~1 capabilities today (Arrow I/O and basic scan). JNI will remain at Tier 1 for broad JDK compatibility. This is the current integration point for Spark and Trino connectors. - -### Java (Panama) - -A new binding layer using Java's Foreign Function & Memory API (Panama) to call the C API -directly. Panama enables native array access without JNI overhead, targeting Tier 2. Requires -JDK 22+. Trino already supports JDK 22 and can adopt Panama immediately. Spark targets older LTS -releases and will not support Panama for some time, so the JNI path remains essential for Spark -integration. diff --git a/docs/user-guide/datafusion.md b/docs/user-guide/datafusion.md index 0be982575b6..ca3ae86bda3 100644 --- a/docs/user-guide/datafusion.md +++ b/docs/user-guide/datafusion.md @@ -1,83 +1,12 @@ # DataFusion -The `vortex-datafusion` crate integrates Vortex as a native DataFusion `FileFormat`, supporting -both reads and writes with filter, projection, and limit pushdown. +> If you're a developer willing to integrate Vortex and Datafusion, see +> [`vortex-datafusion` developer +> guide](../developer-guide/integrations/datafusion) instead. -## Setup +Vortex provides a +[DataFusion CLI](https://github.com/vortex-data/vortex-datafusion-cli) which +allows you to query `.vortex` files via DataFusion. -Add the dependency: - -```toml -[dependencies] -vortex-datafusion = "" -``` - -Register the Vortex format with a `SessionContext`: - -:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs -:language: rust -:dedent: -:start-after: [setup] -:end-before: [setup] -::: - -## Reading Vortex Files - -### SQL - -Create an external table and query it: - -:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs -:language: rust -:dedent: -:start-after: [create] -:end-before: [create] -::: - -### Rust API - -You can also register a `ListingTable` directly: - -:::{literalinclude} ../../vortex-datafusion/examples/vortex_table.rs -:language: rust -:dedent: -:start-after: [register] -:end-before: [register] -::: - -## Writing Vortex Files - -Write query results to Vortex using `INSERT INTO`: - -:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs -:language: rust -:dedent: -:start-after: [write] -:end-before: [write] -::: - -Partitioned writes are supported — DataFusion automatically creates subdirectories for each -partition value. - -## Querying - -Filters and projections are pushed down into the Vortex scan: - -:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs -:language: rust -:dedent: -:start-after: [query] -:end-before: [query] -::: - -### Pushdown Support - -The integration pushes the following operations into the Vortex scan: - -- **Projections** — only referenced columns are read and decompressed. -- **Filters** — comparison (`=`, `<`, `>`), logical (`AND`, `OR`, `NOT`), `IN`, `LIKE`, - `IS NULL`, and cast expressions are evaluated during the scan. Unsupported filters fall back - to DataFusion post-scan evaluation. -- **Limits** — applied at the scan level when no filter is present. -- **File pruning** — files are eliminated without being opened based on partition values and - file-level column statistics (min/max). +You can also use [`vx`](../getting-started/query) +binary with `--query` argument. diff --git a/docs/user-guide/duckdb.md b/docs/user-guide/duckdb.md index fa2fc1fee3f..82c367a3e84 100644 --- a/docs/user-guide/duckdb.md +++ b/docs/user-guide/duckdb.md @@ -13,14 +13,31 @@ INSTALL vortex; LOAD vortex; ```sql SELECT * FROM 'data.vortex'; -# this syntax supports arguments like hive_partitioning=true + +# alternative syntax SELECT * FROM read_vortex('data.vortex'); + +# globbing is also supported +SELECT * FROM 'foo/*.vortex'; + +# you can pass lists of files +SELECT * FROM read_vortex(['1.vortex', '2.vortex']); + +# hive_partitioning is supported. +# assuming folder structure data/year=2000/1.vortex, data/year=2001/2.vortex +SELECT * FROM 'data/*/*.vortex'; + +# alternative syntax supports hive_partitioning, filename, and allow_empty +# parameters +SELECT * FROM read_vortex(['1.vortex', '2.vortex'], filename = true); ``` ## Writing files ```sql COPY (SELECT * FROM my_table) TO 'output.vortex'; +# you can also get file statistics +COPY (SELECT * FROM my_table) TO 'output.vortex' (FORMAT vortex, RETURN_STATS) ``` Make sure to call `LOAD vortex` before writing any files. Duckdb writes CSV diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 27cca7a194d..08747fc3f20 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -1,6 +1,17 @@ # User Guides -Using Vortex with query engines, table formats, and data tools. +Vortex has an integration with +[DuckDB](https://duckdb.org), +[Apache Datafusion](https://datafusion.apache.org), +[Pandas](https://pandas.pydata.org), +[Polars](https://pola.rs), +[Ray](https://ray.io), +and [Apache Spark](https://spark.apache.org). +There is Python support via [vortex-python](./vortex-python.md), +including [pyarrow](https://arrow.apache.org/docs/python/index.html). +Work is in progress for +[Iceberg](https://lists.apache.org/thread/pb66o8gk8nqc6yormfld7pkt6rdfkvh7) +and [cuDF](https://github.com/NVIDIA/cudf/pull/24289). ```{toctree} --- @@ -15,5 +26,4 @@ pyarrow ray spark vortex-python -work-in-progress ``` diff --git a/docs/user-guide/work-in-progress.md b/docs/user-guide/work-in-progress.md deleted file mode 100644 index 8c8be097676..00000000000 --- a/docs/user-guide/work-in-progress.md +++ /dev/null @@ -1,19 +0,0 @@ -# Work In Progress - -The following integrations are under active development. - -## Iceberg - -Apache Iceberg table format support, enabling Vortex as a storage format for Iceberg tables. - -## Substrait - -Substrait plan consumption for portable query execution over Vortex data. - -## cuDF - -RAPIDS cuDF integration for GPU-accelerated DataFrames backed by Vortex arrays. - -## Trino - -Trino connector for querying Vortex tables via JNI. diff --git a/vortex-duckdb/README.md b/vortex-duckdb/README.md index 3efda0af58a..bee3eb06c66 100644 --- a/vortex-duckdb/README.md +++ b/vortex-duckdb/README.md @@ -37,8 +37,8 @@ want to test a full setup, the release commit hash. 5. Open a PR in duckdb-vortex, check the tests and merge it. 6. Ensure the PR has been backported to the current release branch, i.e. for - duckdb 1.5.* the release branch is - [v1.5-variegata](https://github.com/vortex-data/duckdb-vortex/tree/v1.5-variegata). + duckdb 2.0.* the release branch is + [v2.0-cyanoptera](https://github.com/vortex-data/duckdb-vortex/tree/v2.0-cyanoptera). This should happen automatically. 7. Notify DuckDB maintainers in the Discord `#spiraldb` channel of the new commit and ask to update the extension version. This will likely take some diff --git a/vortex-ffi/CMakeLists.txt b/vortex-ffi/CMakeLists.txt index 29b1cc2ee41..02aaeaf39fc 100644 --- a/vortex-ffi/CMakeLists.txt +++ b/vortex-ffi/CMakeLists.txt @@ -1,11 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# Entry point for the Rust FFI libraries. It exports Vortex::ffi_static and -# Vortex::ffi_shared for C consumers; lang/cpp layers the C++ wrapper on top. -# It supports standalone configuration and add_subdirectory() from a parent. - -# CMake 3.25 is required for block() and FetchContent SYSTEM. cmake_minimum_required(VERSION 3.25) project(VortexFFI DESCRIPTION "Vortex C FFI" LANGUAGES C CXX) @@ -13,18 +8,12 @@ project(VortexFFI DESCRIPTION "Vortex C FFI" LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# Standalone entry points own their defaults; otherwise inherit Vortex's marker. if(PROJECT_IS_TOP_LEVEL) set(_vortex_top_level ON) elseif(NOT DEFINED _vortex_top_level) set(_vortex_top_level OFF) endif() -# Vortex does not use C++ modules, so it does not require compiler-specific -# dependency scanners such as clang-scan-deps. -set(CMAKE_CXX_SCAN_FOR_MODULES OFF) - -# A standalone configure writes compile_commands.json for editor tooling. if(PROJECT_IS_TOP_LEVEL) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) endif() @@ -39,19 +28,14 @@ option(VORTEX_BUILD_EXAMPLES "Build Vortex examples" OFF) option(VORTEX_ENABLE_CUDA "Build the CUDA-enabled Vortex FFI" OFF) set(VORTEX_CARGO_PROFILE "" CACHE STRING "Override the Cargo profile selected from CMAKE_BUILD_TYPE") -# Sanitizers instrument the Rust archive, its C dependencies, and consumers of -# either FFI target; Configure.cmake enforces the toolchain constraints. set(VORTEX_SANITIZER "" CACHE STRING - "Comma-separated sanitizers for Vortex and its consumers: asan, lsan, ubsan, tsan") + "Comma-separated sanitizer list: asan, lsan, ubsan, tsan") option(VORTEX_SANITIZE_RUST_STD "Rebuild Rust's standard library with sanitizer instrumentation" OFF) -option(VORTEX_WARNINGS_AS_ERRORS - "Treat warnings in Vortex-owned sources as errors" - "${_vortex_top_level}") +option(VORTEX_WARNINGS_AS_ERRORS "Treat warnings as errors" "${_vortex_top_level}") -# Define the Cargo build and both FFI library targets. include(cmake/Configure.cmake) add_library(Vortex::ffi_static ALIAS vortex_ffi_static) diff --git a/vortex-ffi/README.md b/vortex-ffi/README.md index 3209cf02f42..b524a30e119 100644 --- a/vortex-ffi/README.md +++ b/vortex-ffi/README.md @@ -1,29 +1,28 @@ # Vortex C bindings -## Use from CMake - -Add a complete Vortex checkout. CMake builds the Rust archive for you: +Building requires CMake 3.25+ and a C11 compiler. ```cmake add_subdirectory(path/to/vortex vortex) -target_link_libraries(my_target PRIVATE Vortex::ffi) +target_link_libraries(my_target PRIVATE Vortex::ffi_static) +target_link_libraries(my_target PRIVATE Vortex::ffi_shared) ``` -The target supplies the library and headers, selecting static or shared linkage according -to `BUILD_SHARED_LIBS` (`OFF` by default). Set the option before adding Vortex. To select -linkage explicitly, use `Vortex::ffi_static` or `Vortex::ffi_shared`; both remain available -regardless of the option. You can also add `vortex-ffi` directly. See the -[CMake build guide](../lang/cpp/README.md) for requirements, options, and deployment limits. - -### Examples and tests +Note this builds Vortex Rust side for you. -Run from the repository root. Tests use Catch2 and require a C++ compiler. +Tests require a C++-20 compiler. To run tests, ```sh -cmake -S vortex-ffi -B build/ffi \ - -DVORTEX_BUILD_EXAMPLES=ON -DVORTEX_BUILD_TESTS=ON +cmake -S vortex-ffi -B build/ffi -DVORTEX_BUILD_TESTS=ON cmake --build build/ffi --parallel ctest --test-dir build/ffi --output-on-failure +``` + +To run examples, + +```sh +cmake -S vortex-ffi -B build/ffi -DVORTEX_BUILD_EXAMPLES=ON +cmake --build build/ffi --parallel ./build/ffi/examples/write_sample sample.vortex ./build/ffi/examples/dtype sample.vortex @@ -31,29 +30,12 @@ ctest --test-dir build/ffi --output-on-failure ./build/ffi/examples/scan_to_arrow sample.vortex ``` -## Runtime threading - -By default, calling threads drive the shared runtime with no Vortex workers. - -`vx_runtime_set_worker_threads(n)` configures a library-local pool of background workers that -drive the shared runtime. The pool is shared by all FFI sessions in that library instance and -persists across calls for the lifetime of the library. Passing zero signals the workers to stop -and restores caller-driven execution. Leave it at zero if your application already provides concurrency. +To regenerate `vortex.h` after making some changes, run `cargo +nightly build -p +vortex-ffi`. -## Update the C header +## Running tests with sanitizers -Regenerate `vortex-ffi/cinclude/vortex.h`: - -```sh -cargo +nightly build -p vortex-ffi -``` - -## Sanitizer tests - -### Rust and C/C++ together - -Requires nightly Rust and upstream LLVM Clang. See the -[sanitizer guide](../lang/cpp/README.md#sanitizers) for scope and alternatives. +C tests with sanitizer also require clang. To run them, ```sh rustup toolchain install nightly --component rust-src @@ -70,22 +52,20 @@ ctest --test-dir build/ffi-asan --output-on-failure For ThreadSanitizer, use `-DVORTEX_SANITIZER=tsan` and set `TSAN_OPTIONS=suppressions=/absolute/path/to/vortex/vortex-ffi/tsan_suppressions.txt`. -Pipe `build/ffi-asan/test/vortex_ffi_test` output through `rustfilt -i-` to demangle Rust symbols. - -### Rust only - -Use nightly with `rust-src` installed. This example uses the native Linux x86_64 target: +To run only Rust-side tests with sanitizers on x86_64, ```sh +# -Zsanitizer=memory and -Zsanitizer=thread are other options RUSTFLAGS="-Zsanitizer=address -Cunsafe-allow-abi-mismatch=sanitizer" \ cargo +nightly test -p vortex-ffi -Zbuild-std \ --target x86_64-unknown-linux-gnu --tests -- --no-capture ``` -Substitute your native target. Use `-Zsanitizer=memory` for MemorySanitizer or -`-Zsanitizer=thread` for ThreadSanitizer, with the suppression file above. +## Driving the runtime -- `-Zbuild-std` instruments std to avoid memory/thread sanitizer false positives. -- `-Cunsafe-allow-abi-mismatch=sanitizer` allows deliberately uninstrumented dependencies. -- `--tests` excludes doctests, which do not inherit `RUSTFLAGS`. -- Prefer `cargo test` over nextest for leak detection. Use `llvm-symbolizer` for stack traces. +Vortex is an async library which means it requires an underlying runtime, a +thread scheduler. By default, Vortex uses host threads to drive the runtime, +i.e. calling a Vortex function like `vx_data_source_scan` on multiple threads +drives the runtime on the caller threads. You can create an explicit thread pool +inside Vortex by calling `vx_runtime_set_worker_threads(n)`. After this call, +Vortex will drive the runtime on the internal thread pool and not host threads. diff --git a/vortex-sqllogictest/README.md b/vortex-sqllogictest/README.md index 14dfe5041cf..6e4590143d8 100644 --- a/vortex-sqllogictest/README.md +++ b/vortex-sqllogictest/README.md @@ -1,206 +1,45 @@ # vortex-sqllogictest -This crate uses [`sqllogictest-rs`](https://github.com/risinglightdb/sqllogictest-rs) to run -`.slt`-based tests against both DataFusion and DuckDB, each preconfigured to read Vortex files. - -Every `.slt` file is turned into one independent test per engine, driven by `sqllogictest`'s -`libtest-mimic` harness. Tests run in parallel; within a single file the records execute -sequentially for one engine. Each test is named `slt::::`, for example -`slt::datafusion::integers.slt` or `slt::duckdb::duckdb/explain.slt`. +This crate tests correctness of our DuckDB and DataFusion integrations. It runs +[.slt](https://github.com/risinglightdb/sqllogictest-rs) tests with our queries +as well as 0.1 scale of TPC-H, TPC-DS, and 1 shard of partitioned ClickBench. +For last three query sets, tests also verify each query's `EXPLAIN` result. ## Running tests -Some tests use TPC-H and TPC-DS data at scale factor 0.1 and one shard of the partitioned -ClickBench dataset (about one million rows of `hits`). Generate them first, then run the suite -with `cargo nextest`: - -```shell -./vortex-sqllogictest/slt/generate_data.sh -cargo nextest run -p vortex-sqllogictest -# the built-in cargo test harness also works: +```sh +# For running our own queries cargo test -p vortex-sqllogictest --test sqllogictests -``` - -`generate_data.sh` accepts dataset names to generate only some of the fixtures, for example -`./vortex-sqllogictest/slt/generate_data.sh tpch`, `... tpcds` or `... clickbench`. Run it with -`--help` to list the datasets. It runs each dataset's own script, `slt/tpch/generate_data.sh`, -`slt/tpcds/generate_data.sh` or `slt/clickbench/generate_data.sh`, which can also be run directly. -TPC-H is generated by `vortex-bench`'s `data-gen` binary, the same generator the benchmarks use, -and TPC-DS by `tpcgen-cli` through `uvx`. - -The generated Vortex and Parquet data lives under `slt/tpch/data/`, `slt/tpcds/data/` and -`slt/clickbench/data/` (git-ignored). Both formats are required; regenerate older fixtures if they -only contain Vortex files. If either format is missing, that suite's tests are reported as -**ignored**, so the rest of the suite still runs. These `.slt` files load their tables through -paths relative to the crate root, so run the tests via `cargo nextest`/`cargo test`, which set the -working directory accordingly. - -TPC-H scripts live under `slt/tpch/datafusion/` and `slt/tpch/duckdb/`, TPC-DS scripts under -`slt/tpcds/datafusion/` and `slt/tpcds/duckdb/`, ClickBench scripts under -`slt/clickbench/datafusion/` and `slt/clickbench/duckdb/`. Each engine has its own -`create.slt.no` and `drop.slt.no`. The query results, `results/q*.slt.no` (`q1` to `q22` for -TPC-H, `q1` to `q99` for TPC-DS, `q0` to `q42` for ClickBench, matching the upstream numbering), -are shared by both engines in `slt/tpch/results/` and `slt/tpcds/results/`. ClickBench keeps one -`results/` directory for each engine. Each engine's `tpch.slt`/`tpcds.slt`/`clickbench.slt` runs -the queries against Vortex and asserts EXPLAIN output from the matching `plans/q*.slt.no`. Its -`parquet.slt` runs the same queries against the original Parquet fixtures and checks the same -expected results. - -DataFusion truncates decimal division and `avg` to the scale of the result type, while DuckDB -calculates them as `DOUBLE`. Thus the DuckDB validator accepts a value with more fractional digits -when it truncates to the expected value. Use `onlyif datafusion`/`onlyif duckdb` records when the -engines differ more than that, as TPC-DS q66 does. -DataFusion uses external tables; DuckDB uses views over files. The `FILE_FORMAT` substitution -variable selects the format in each engine's table setup. - -ClickBench plans explain the upstream queries unchanged. Where an upstream query leaves the order -of tied rows unspecified, its result record adds tie-breaking `ORDER BY` columns so both formats -and repeated runs produce the same rows. The ClickBench and TPC-DS generators also run -`slt/clickbench/duckdb/parity.slt` and `slt/tpcds/duckdb/parity.slt` right after converting their -fixtures; they read both formats through DuckDB and fail if the Parquet and Vortex data differ. -After completing ClickBench or TPC-DS DataFusion plans, replace the byte ranges in `file_groups` -with ``, as the TPC-H plans do: - -```shell -sed -i -E 's/\.vortex:[0-9]+\.\.[0-9]+/.vortex:/g' vortex-sqllogictest/slt/{clickbench,tpcds}/datafusion/plans/*.slt.no -``` -Because the harness is `libtest-mimic`-based, the standard test flags work, including -`cargo nextest`, filtering, and listing: +# For running TPC-H, TPC-DS, or ClickBench you need to generate data first +./slt/generate_data.sh +# You can generate data for a specific dataset +./slt/generate_data.sh tpcds +# Then you can run the tests +cargo test -p vortex-sqllogictest --test sqllogictests -```shell -# Run only DuckDB tests: +# You can run only DuckDB tests... cargo nextest run -p vortex-sqllogictest -E 'test(/slt::duckdb::/)' -# Run a single file on both engines (substring filter): +# ... or a single test on both DuckDB and DataFusion cargo nextest run -p vortex-sqllogictest -E 'test(strings)' -# List every generated test without running: + +# You can also list all tests cargo nextest list -p vortex-sqllogictest ``` -## Scratch directory and `${WORK_DIR}` - -Tests reference a per-test working directory through the `${WORK_DIR}` substitution variable. The -runner sets `WORK_DIR` to a constant, git-ignored scratch directory **inside this crate** — -`scratch//` — rather than an OS tempdir. The path is deterministic (named after the test, -not random), so it is easy to inspect, and each test gets its own directory so concurrent tests -never collide. The directory is recreated empty before each test and removed afterwards — whether -the test passed, failed, or panicked (cleanup errors are logged, not fatal). - -Query output is passed through a normalization step that rewrites the scratch path back to the -`${WORK_DIR}` token. This keeps expected output stable across machines and runs, and is what lets -`--complete` (below) write portable expected values instead of machine-specific paths. - -## Selecting which engine runs a test - -There are two complementary mechanisms: - -- **Per-file, by directory.** A file under a `datafusion/` directory runs **only** on DataFusion; - a file under a `duckdb/` directory runs **only** on DuckDB. Anything else runs on **both**. This - is how engine-specific features (e.g. DuckDB `EXPLAIN` plans) are kept isolated. -- **Per-record, by label.** Use `onlyif