Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions duckdb-vortex-partitioned/install
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,23 @@
set -e

# Build duckdb-vortex from source. Idempotent.
if [ -x duckdb-vortex/build/release/duckdb ]; then
DUCKDB=duckdb-vortex/build/release/duckdb
if "$DUCKDB" -c "SELECT vortex_version()" >/dev/null 2>&1; then
exit 0
fi

# Vortex 0.90.0 on DuckDB v2.0.0. DuckDB's own `INSTALL vortex` ships an
# older Vortex, so build the extension statically into the DuckDB CLI.
DUCKDB_VORTEX_REF=59b149bf38d354951dd078183852ca9ed11cb8d4

# vcpkg fails the build with `unable to read $HOME` when cloud-init runs
# this script with HOME unset. cloud-init.sh.in already exports it, but
# stamp HOME locally too so the build works regardless of which
# cloud-init template generated the wrapper.
export HOME="${HOME:-/root}"

sudo apt-get update -y
sudo apt-get install -y ninja-build cmake build-essential make ccache pip clang pkg-config
sudo apt-get install -y ninja-build cmake build-essential make ccache pip clang libclang-dev pkg-config zip unzip

if ! command -v cargo >/dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
Expand All @@ -25,14 +30,25 @@ fi
export CC=clang
export CXX=clang++

# On many-core hosts ninja and cargo together run hundreds of compiler
# jobs, which exceeds the default 1024 open-file limit (os error 24).
ulimit -n "$(ulimit -Hn)" 2>/dev/null || true

if [ ! -d duckdb-vortex ]; then
git clone https://github.com/vortex-data/duckdb-vortex --recursive
git clone https://github.com/vortex-data/duckdb-vortex
fi

(
cd duckdb-vortex
git fetch --tags
git checkout v0.44.0
git fetch origin "$DUCKDB_VORTEX_REF"
git checkout "$DUCKDB_VORTEX_REF"
git submodule update --init --recursive
GEN=ninja NATIVE_ARCH=1 LTO=thin make
# Since DuckDB 2.0 the shell links only the extensions it is told to.
# The extension uses no C++20 modules, and scanning for them needs
# clang-scan-deps, which Ubuntu's clang package does not ship.
EXT_FLAGS='-DSTATICALLY_LINK_EXTENSIONS="core_functions parquet json icu vortex" -DCMAKE_CXX_SCAN_FOR_MODULES=OFF' \
GEN=ninja NATIVE_ARCH=1 LTO=thin make
)

# Fail here rather than at load time if vortex did not make it into the shell.
"$DUCKDB" -c "SELECT vortex_version()"
4 changes: 2 additions & 2 deletions duckdb-vortex-partitioned/load
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ DUCKDB="$(pwd)/duckdb-vortex/build/release/duckdb"
# Convert each parquet partition to a vortex file in parallel.
seq 0 99 | xargs -P"$(nproc)" -I{} bash -c '
if [ ! -f "hits_{}.vortex" ]; then
"'"$DUCKDB"'" -c "
"'"$DUCKDB"'" -no-agent -c "
COPY (
SELECT *
REPLACE (
Expand All @@ -22,7 +22,7 @@ seq 0 99 | xargs -P"$(nproc)" -I{} bash -c '

# Build a persistent VIEW over the resulting vortex files.
rm -f hits.db
"$DUCKDB" hits.db -c "CREATE VIEW hits AS SELECT * FROM read_vortex('hits_*.vortex')"
"$DUCKDB" -no-agent hits.db -c "CREATE VIEW hits AS SELECT * FROM read_vortex('hits_*.vortex')"

# Free the source parquet files.
rm -f hits_*.parquet
Expand Down
3 changes: 2 additions & 1 deletion duckdb-vortex-partitioned/query
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ set -e
DUCKDB="$(pwd)/duckdb-vortex/build/release/duckdb"
query=$(cat)

out=$("$DUCKDB" hits.db -c ".timer on" -c "$query" 2>/tmp/duckdb.err.$$) && status=0 || status=$?
# -no-agent: keep the plain CLI output; see ../duckdb-vortex/query.
out=$("$DUCKDB" -no-agent hits.db -c ".timer on" -c "$query" 2>/tmp/duckdb.err.$$) && status=0 || status=$?

if [ "$status" -ne 0 ]; then
cat /tmp/duckdb.err.$$ >&2
Expand Down
59 changes: 59 additions & 0 deletions duckdb-vortex-partitioned/results/20261102/c7a.metal-48xl.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{
"system": "DuckDB (Vortex, partitioned)",
"date": "2026-11-02",
"machine": "c7a.metal-48xl",
"cluster_size": 1,
"proprietary": "no",
"hardware": "cpu",
"tuned": "no",
"tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"],
"load_time": 87,
"data_size": 12251457760,
"concurrent_qps": null,
"concurrent_error_ratio": null,
"result": [
[0.18, 0.048, 0.043],
[0.217, 0.048, 0.045],
[0.22, 0.073, 0.07],
[0.584, 0.117, 0.112],
[0.61, 0.261, 0.249],
[0.558, 0.36, 0.364],
[0.196, 0.051, 0.049],
[0.231, 0.064, 0.058],
[0.943, 0.301, 0.311],
[1.22, 0.377, 0.345],
[0.621, 0.159, 0.158],
[0.708, 0.172, 0.156],
[0.568, 0.39, 0.378],
[2.007, 0.394, 0.372],
[0.681, 0.303, 0.322],
[0.628, 0.267, 0.275],
[1.945, 0.509, 0.485],
[1.943, 0.505, 0.467],
[3.012, 0.621, 0.617],
[0.28, 0.098, 0.098],
[8.56, 0.531, 0.488],
[13.066, 0.741, 0.758],
[16.711, 0.624, 0.631],
[2.12, 0.244, 0.256],
[0.22, 0.077, 0.083],
[0.514, 0.18, 0.18],
[0.212, 0.073, 0.077],
[11.186, 0.515, 0.566],
[9.255, 1.35, 1.476],
[0.225, 0.115, 0.114],
[1.911, 0.269, 0.27],
[4.921, 0.426, 0.395],
[3.694, 0.837, 0.865],
[11.287, 1.247, 1.305],
[11.261, 1.339, 1.444],
[0.333, 0.221, 0.241],
[0.2, 0.068, 0.067],
[0.178, 0.049, 0.053],
[0.266, 0.06, 0.062],
[0.295, 0.084, 0.093],
[0.256, 0.05, 0.053],
[0.247, 0.055, 0.056],
[0.244, 0.057, 0.058]
]
}
59 changes: 59 additions & 0 deletions duckdb-vortex-partitioned/results/20261102/c8g.4xlarge.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{
"system": "DuckDB (Vortex, partitioned)",
"date": "2026-11-02",
"machine": "c8g.4xlarge",
"cluster_size": 1,
"proprietary": "no",
"hardware": "cpu",
"tuned": "no",
"tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"],
"load_time": 919,
"data_size": 12251458440,
"concurrent_qps": null,
"concurrent_error_ratio": null,
"result": [
[0.119, 0.016, 0.016],
[0.198, 0.023, 0.025],
[0.324, 0.03, 0.032],
[0.638, 0.076, 0.077],
[1.033, 0.186, 0.188],
[0.927, 0.193, 0.195],
[0.164, 0.018, 0.019],
[0.209, 0.028, 0.029],
[1.019, 0.23, 0.23],
[1.374, 0.317, 0.331],
[0.722, 0.094, 0.089],
[1.057, 0.103, 0.098],
[1.177, 0.207, 0.21],
[2.48, 0.376, 0.382],
[0.957, 0.25, 0.252],
[0.799, 0.242, 0.232],
[2.357, 0.48, 0.481],
[2.23, 0.381, 0.382],
[3.508, 0.818, 0.834],
[0.36, 0.039, 0.041],
[10.311, 0.335, 0.341],
[14.233, 0.448, 0.445],
[16.877, 0.744, 0.736],
[2.044, 0.141, 0.137],
[0.168, 0.033, 0.034],
[1.238, 0.081, 0.077],
[0.253, 0.035, 0.036],
[12.26, 0.278, 0.281],
[10.006, 2.418, 2.476],
[0.193, 0.041, 0.041],
[2.239, 0.185, 0.19],
[5.418, 0.254, 0.254],
[3.969, 0.872, 0.867],
[12.116, 0.981, 0.979],
[12.12, 1.09, 1.086],
[0.429, 0.203, 0.196],
[0.228, 0.036, 0.032],
[0.168, 0.019, 0.019],
[0.224, 0.032, 0.031],
[0.284, 0.058, 0.057],
[0.181, 0.018, 0.02],
[0.182, 0.018, 0.019],
[0.172, 0.016, 0.016]
]
}
59 changes: 59 additions & 0 deletions duckdb-vortex-partitioned/results/20261102/c8g.metal-48xl.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{
"system": "DuckDB (Vortex, partitioned)",
"date": "2026-11-02",
"machine": "c8g.metal-48xl",
"cluster_size": 1,
"proprietary": "no",
"hardware": "cpu",
"tuned": "no",
"tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"],
"load_time": 82,
"data_size": 12251458440,
"concurrent_qps": null,
"concurrent_error_ratio": null,
"result": [
[0.114, 0.034, 0.036],
[0.137, 0.037, 0.047],
[0.156, 0.061, 0.057],
[0.529, 0.109, 0.114],
[0.71, 0.225, 0.241],
[0.494, 0.346, 0.318],
[0.116, 0.032, 0.034],
[0.15, 0.052, 0.051],
[0.892, 0.268, 0.271],
[1.205, 0.318, 0.303],
[0.596, 0.131, 0.125],
[0.75, 0.135, 0.14],
[0.587, 0.318, 0.328],
[1.963, 0.344, 0.329],
[0.655, 0.269, 0.281],
[0.566, 0.266, 0.26],
[1.913, 0.444, 0.415],
[1.925, 0.433, 0.458],
[2.968, 0.584, 0.542],
[0.259, 0.085, 0.073],
[8.795, 0.547, 0.466],
[13.254, 0.679, 0.677],
[16.617, 0.522, 0.5],
[2.066, 0.221, 0.23],
[0.211, 0.073, 0.068],
[0.491, 0.144, 0.144],
[0.195, 0.068, 0.069],
[11.166, 0.481, 0.493],
[9.277, 1.857, 1.811],
[0.199, 0.086, 0.09],
[1.903, 0.231, 0.227],
[4.885, 0.355, 0.351],
[3.656, 0.874, 0.816],
[11.271, 1.198, 1.22],
[11.208, 1.281, 1.24],
[0.442, 0.207, 0.197],
[0.184, 0.052, 0.051],
[0.156, 0.035, 0.042],
[0.179, 0.048, 0.055],
[0.204, 0.067, 0.065],
[0.169, 0.042, 0.038],
[0.166, 0.037, 0.036],
[0.161, 0.043, 0.043]
]
}
3 changes: 2 additions & 1 deletion duckdb-vortex/check
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

duckdb -c "SELECT 1" >/dev/null
DUCKDB="$(pwd)/duckdb-vortex/build/release/duckdb"
"$DUCKDB" -c "SELECT 1" >/dev/null
59 changes: 47 additions & 12 deletions duckdb-vortex/install
Original file line number Diff line number Diff line change
@@ -1,19 +1,54 @@
#!/bin/bash
set -e

# duckdb writes its extension cache under $HOME/.duckdb/extensions. With
# HOME unset (cloud-init occasionally runs scripts that way) the
# `INSTALL vortex` below ends up writing to /.duckdb and later sibling
# scripts (load/query) can't find the extension. The previous
# `export HOME=${HOME:=~}` was inside the install-duckdb branch, so
# re-runs with duckdb already on PATH still hit the bug. Pin HOME
# unconditionally at the top.
# Build duckdb-vortex from source. Idempotent.
DUCKDB=duckdb-vortex/build/release/duckdb
if "$DUCKDB" -c "SELECT vortex_version()" >/dev/null 2>&1; then
exit 0
fi

# Vortex 0.90.0 on DuckDB v2.0.0. DuckDB's own `INSTALL vortex` ships an
# older Vortex, so build the extension statically into the DuckDB CLI.
DUCKDB_VORTEX_REF=59b149bf38d354951dd078183852ca9ed11cb8d4

# vcpkg fails the build with `unable to read $HOME` when cloud-init runs
# this script with HOME unset. cloud-init.sh.in already exports it, but
# stamp HOME locally too so the build works regardless of which
# cloud-init template generated the wrapper.
export HOME="${HOME:-/root}"

if ! command -v duckdb >/dev/null 2>&1; then
curl https://install.duckdb.org | sh
sudo ln -sf "$HOME/.duckdb/cli/latest/duckdb" /usr/local/bin/duckdb
sudo apt-get update -y
sudo apt-get install -y ninja-build cmake build-essential make ccache pip clang libclang-dev pkg-config zip unzip

if ! command -v cargo >/dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
| sh -s -- -y --default-toolchain stable --no-modify-path
fi
# shellcheck disable=SC1091
[ -f "$HOME/.cargo/env" ] && source "$HOME/.cargo/env"

export CC=clang
export CXX=clang++

# On many-core hosts ninja and cargo together run hundreds of compiler
# jobs, which exceeds the default 1024 open-file limit (os error 24).
ulimit -n "$(ulimit -Hn)" 2>/dev/null || true

if [ ! -d duckdb-vortex ]; then
git clone https://github.com/vortex-data/duckdb-vortex
fi

(
cd duckdb-vortex
git fetch origin "$DUCKDB_VORTEX_REF"
git checkout "$DUCKDB_VORTEX_REF"
git submodule update --init --recursive
# Since DuckDB 2.0 the shell links only the extensions it is told to.
# The extension uses no C++20 modules, and scanning for them needs
# clang-scan-deps, which Ubuntu's clang package does not ship.
EXT_FLAGS='-DSTATICALLY_LINK_EXTENSIONS="core_functions parquet json icu vortex" -DCMAKE_CXX_SCAN_FOR_MODULES=OFF' \
GEN=ninja NATIVE_ARCH=1 LTO=thin make
)

# Install the vortex extension (idempotent — safe to re-run).
duckdb -c "INSTALL vortex;"
# Fail here rather than at load time if vortex did not make it into the shell.
"$DUCKDB" -c "SELECT vortex_version()"
6 changes: 4 additions & 2 deletions duckdb-vortex/load
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
#!/bin/bash
set -e

DUCKDB="$(pwd)/duckdb-vortex/build/release/duckdb"

# Convert source parquet to Vortex format.
# -no-agent: keep the plain CLI output; see ./query.
duckdb -no-agent -c "LOAD vortex; COPY 'hits.parquet' TO 'hits.vortex' (FORMAT vortex);"
"$DUCKDB" -no-agent -c "COPY 'hits.parquet' TO 'hits.vortex' (FORMAT vortex);"

# Create the persistent VIEW (over hits.vortex) in hits.db.
rm -f hits.db
duckdb -no-agent hits.db -f create.sql
"$DUCKDB" -no-agent hits.db -f create.sql

# Source parquet no longer needed.
rm -f hits.parquet
Expand Down
7 changes: 4 additions & 3 deletions duckdb-vortex/query
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
#!/bin/bash
# Reads a SQL query from stdin, runs it via duckdb against hits.db (a VIEW
# over hits.vortex). The vortex extension is loaded for each invocation.
# Reads a SQL query from stdin, runs it via the custom-built duckdb-vortex
# binary against hits.db (a VIEW over hits.vortex).
# Stdout: query result. Stderr: runtime in fractional seconds on the last
# line. Exit non-zero on error.
set -e

DUCKDB="$(pwd)/duckdb-vortex/build/release/duckdb"
query=$(cat)

# -no-agent: the 2.0 CLI turns on "agent mode" when it detects an AI coding
# agent in the environment (AI_AGENT, CLAUDECODE, CURSOR_AGENT, ...) and
# stdout is not a terminal: Markdown tables, truncated results, JSON errors
# and a progress bar. Keep the plain output regardless of who runs this.
out=$(duckdb -no-agent hits.db -c "LOAD vortex;" -c ".timer on" -c "$query" 2>/tmp/duckdb.err.$$) && status=0 || status=$?
out=$("$DUCKDB" -no-agent hits.db -c ".timer on" -c "$query" 2>/tmp/duckdb.err.$$) && status=0 || status=$?

if [ "$status" -ne 0 ]; then
cat /tmp/duckdb.err.$$ >&2
Expand Down
Loading
Loading