From 1eb612208774bf6ff33636b8c5212fccb22fc072 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 11 Nov 2026 03:51:33 -0500 Subject: [PATCH] Show TPC-H SF10 on the homepage's results table The table gains a TPC-H row, after ClickBench: the geometric mean time of the 22 TPC-H queries at scale factor 10 on DataFusion, read from bench.vortex.dev like the other rows. The FAQ and llms.txt read the same seven panels, so the separate list for the FAQ's TPC-H item goes. Signed-off-by: Will Manning --- src/lib/bench-summary.ts | 63 +-- src/lib/faq.ts | 21 +- src/lib/llms.ts | 4 + tests/smoke.spec.ts | 7 +- tests/unit/bench-summary.test.ts | 95 ++-- tests/unit/faq.test.ts | 12 +- .../fixtures/bench-group-tpch-sf10-nvme.json | 425 ++++++++++++++++++ tests/unit/llms.test.ts | 11 +- tests/unit/results-table.test.tsx | 4 +- 9 files changed, 521 insertions(+), 121 deletions(-) create mode 100644 tests/unit/fixtures/bench-group-tpch-sf10-nvme.json diff --git a/src/lib/bench-summary.ts b/src/lib/bench-summary.ts index 43b55f94..8889aac8 100644 --- a/src/lib/bench-summary.ts +++ b/src/lib/bench-summary.ts @@ -64,11 +64,7 @@ export type Panel = { source?: { note: string }; }; -/** - * The results table's panels, the panels /faq adds to them - * (`morePanelsFrom`), and when bench answered. - */ -export type BenchSummary = { panels: Panel[]; more: Panel[]; readAt: Date }; +export type BenchSummary = { panels: Panel[]; readAt: Date }; // The systems the table can compare, by bench's format name, in the order the // rows are listed (the table sorts them best first). Each panel chooses the @@ -125,9 +121,10 @@ type Source = { }; }; -// The six panels, in the table's order: five from bench and, beside -// ClickBench, the GPU result bench does not publish yet, kept by hand in -// gpu-result.ts with its source. +// The seven panels, in the table's order: six from bench and, after the two +// query benchmarks, the GPU result bench does not publish yet, kept by hand +// in gpu-result.ts with its source. ClickBench and TPC-H run on DataFusion, +// the one engine that runs all three formats. const SOURCES: (Source | Panel)[] = [ { title: "Scan throughput", @@ -168,6 +165,19 @@ const SOURCES: (Source | Panel)[] = [ systems: WITH_LANCE, times: { series: (f) => `datafusion:${f}`, bucket: (chart) => chart } }, + { + title: "TPC-H SF10 on DataFusion", + short: "TPC-H", + unit: "ms", + kind: "speed", + better: "lower", + group: "TPC-H (NVMe) (SF=10)", + rankings: (s) => s.rankings, + name: (f) => `datafusion:${f}`, + score: "score", + systems: WITH_LANCE, + times: { series: (f) => `datafusion:${f}`, bucket: (chart) => chart } + }, GPU_PANEL, { title: "Random access, NVMe, hot", @@ -319,24 +329,6 @@ function timeRows( }); } -// The panels /faq adds to the table's: TPC-H at scale factor 10 on -// DataFusion, the one engine that runs all three formats. A panel shows -// bench's scores against the fastest, so it needs no charts. -const MORE_SOURCES: Source[] = [ - { - title: "TPC-H SF10 on DataFusion", - short: "TPC-H", - unit: "× fastest", - kind: "speed", - better: "lower", - group: "TPC-H (NVMe) (SF=10)", - rankings: (s) => s.rankings, - name: (f) => `datafusion:${f}`, - score: "score", - systems: WITH_LANCE - } -]; - /** * One bench panel from its group in bench's `/api/groups` payload and, for * a time panel, the group's charts. A shown system bench did not report @@ -397,10 +389,10 @@ function panelFor( } /** - * The results table's six panels: five from bench (its `/api/groups` + * The results table's seven panels: six from bench (its `/api/groups` * payload's `groups` array and, for a time panel, `charts`, its group's * `/api/group` payload by group name) and the hand-kept GPU panel. A bench - * panel that `panelFor` cannot build gives `null`: the table shows six + * panel that `panelFor` cannot build gives `null`: the table shows seven * panels or none. */ export function panelsFrom( @@ -421,16 +413,6 @@ export function panelsFrom( return panels; } -/** - * The panels /faq adds to the table's (`MORE_SOURCES`), from the same - * `/api/groups` payload. One that bench does not give is left out, so the - * table's panels never depend on these. - */ -export function morePanelsFrom(groups: unknown): Panel[] { - if (!Array.isArray(groups)) return []; - return MORE_SOURCES.flatMap((source) => panelFor(source, groups, {}) ?? []); -} - /** * The homepage's benchmark figures, read from bench.vortex.dev at build time * and again when the page rebuilds, at most once a day (ISR). A figure is never typed into the page. @@ -439,7 +421,7 @@ export function morePanelsFrom(groups: unknown): Panel[] { * bench answered the first read (its Date header), so a cached answer keeps * its own date. A read that fails or gets a 5xx or 429 is tried again, up to * three times. Any failure after that (a timeout, an error status, a payload - * without all six panels) gives `null`, with a warning that says why, and + * without all seven panels) gives `null`, with a warning that says why, and * the page then leaves the table out (but see `summaryForPage`). A time * panel whose charts are missing from the window shows bench's scores. * `fetchImpl` and `io` are for tests. @@ -514,11 +496,10 @@ export async function getBenchSummary( ) ); const panels = panelsFrom(groups, charts); - if (!panels) return fail("bench's groups do not give all six panels"); + if (!panels) return fail("bench's groups do not give all seven panels"); const date = new Date(answer.date ?? ""); return { panels, - more: morePanelsFrom(groups), readAt: Number.isNaN(date.getTime()) ? new Date() : date }; } catch (e) { diff --git a/src/lib/faq.ts b/src/lib/faq.ts index 0bde443d..ac0e22f6 100644 --- a/src/lib/faq.ts +++ b/src/lib/faq.ts @@ -49,19 +49,6 @@ export type Faq = { // The table formats in the Ecosystem band, for the answer on Lance. const TABLE_FORMATS = ["Apache Iceberg", "Apache Hudi", "Apache Paimon"]; -/** - * The panels the answers compare: the table's, with the panels /faq adds - * (`more`) after ClickBench, so the query benchmarks sit together. - */ -function comparedPanels( - panels: readonly Panel[], - more: readonly Panel[] -): Panel[] { - const at = panels.findIndex((p) => p.title === "ClickBench on DataFusion"); - const cut = at === -1 ? panels.length : at + 1; - return [...panels.slice(0, cut), ...more, ...panels.slice(cut)]; -} - /** * Vortex against Lance in a panel, in the words the table uses against * Parquet ("14× faster", "48% smaller"), or "Lance not measured". `null` @@ -100,15 +87,11 @@ export function faqs({ sparkVersion, ecosystem }: { - bench: { - panels: readonly Panel[]; - more?: readonly Panel[]; - readAt: Date; - } | null; + bench: { panels: readonly Panel[]; readAt: Date } | null; sparkVersion: string | null; ecosystem: readonly EcosystemEntry[]; }): Faq[] { - const compared = bench ? comparedPanels(bench.panels, bench.more ?? []) : []; + const compared = bench?.panels ?? []; const systems = resultsSystems(compared); const verdict = (panel: Panel) => resultsRow(panel, systems).verdict; const hasGpu = compared.some((p) => p.title === GPU_PANEL.title); diff --git a/src/lib/llms.ts b/src/lib/llms.ts index 4b5f5490..bd1a6e2a 100644 --- a/src/lib/llms.ts +++ b/src/lib/llms.ts @@ -50,6 +50,10 @@ function about(panel: Panel): string { return panel.unit === "ms" ? "the geometric mean time of the ClickBench queries on DataFusion, reading files on local NVMe, so lower is better" : "the ClickBench queries on DataFusion, reading files on local NVMe: the geometric mean of each query's time divided by the fastest format's, so lower is better"; + case "TPC-H SF10 on DataFusion": + return panel.unit === "ms" + ? "the geometric mean time of the 22 TPC-H queries at scale factor 10 on DataFusion, reading files on local NVMe, so lower is better" + : "the TPC-H queries at scale factor 10 on DataFusion, reading files on local NVMe: the geometric mean of each query's time divided by the fastest format's, so lower is better"; case "Random access, NVMe, hot": return panel.unit === "ms" ? "the geometric mean time to take rows from each of the random access benchmark's datasets on local NVMe, reusing an open file, so lower is better" diff --git a/tests/smoke.spec.ts b/tests/smoke.spec.ts index 09c17756..664e3847 100644 --- a/tests/smoke.spec.ts +++ b/tests/smoke.spec.ts @@ -182,7 +182,7 @@ test.describe("home", () => { }); test.describe("home bands", () => { - test("the results table shows six benchmarks with a source link, or is absent", async ({ + test("the results table shows seven benchmarks with a source link, or is absent", async ({ page }) => { await page.goto("/"); @@ -194,8 +194,8 @@ test.describe("home bands", () => { return; } await expect(table).toBeVisible(); - await expect(table.locator("tbody tr")).toHaveCount(6); - await expect(table.locator("tbody td.is-ours")).toHaveCount(6); + await expect(table.locator("tbody tr")).toHaveCount(7); + await expect(table.locator("tbody td.is-ours")).toHaveCount(7); await expect(table).toContainText(/read \d+ \w{3} \d{4}/); const source = table.locator('a[href="https://bench.vortex.dev"]'); await expect(source).toBeVisible(); @@ -226,6 +226,7 @@ test.describe("home bands", () => { "Scan", "Write", "ClickBench", + "TPC-H", "GPU, cuDF", "Random access", "File size" diff --git a/tests/unit/bench-summary.test.ts b/tests/unit/bench-summary.test.ts index f462722d..33a45681 100644 --- a/tests/unit/bench-summary.test.ts +++ b/tests/unit/bench-summary.test.ts @@ -5,7 +5,6 @@ import { formatReadDate, getBenchSummary, groupTimes, - morePanelsFrom, type Panel, panelsFrom, type Row, @@ -34,9 +33,11 @@ const PAYLOAD = fixture("bench-groups.json") as { // bench.vortex.dev/api/group/{slug}?n=25 for the two time panels' groups, read // at the same time, cut down to the series the card reads (and one it does // not) and their last five commits. Random access's hot series were read on -// 30 Oct 2026, from the same two commits as the rest. +// 30 Oct 2026, from the same two commits as the rest. TPC-H (NVMe) (SF=10)'s +// were read on 11 Nov 2026, with its group summary. const CHARTS = { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }; @@ -72,6 +73,12 @@ const CLICKBENCH_MS = { Parquet: 65.621614465414, Lance: 578.6306006236966 }; +// TPC-H SF10's, the same way (Python, 11 Nov 2026), over its 22 queries. +const TPCH_MS = { + Vortex: 60.50866557993905, + Parquet: 143.30314256082335, + Lance: 846.7997612370476 +}; const RANDOM_ACCESS_MS = { Vortex: 0.7290522397517741, Parquet: 70.61301694616571, @@ -79,11 +86,11 @@ const RANDOM_ACCESS_MS = { Arrow: 3.7780608403173668 }; -// The six panels from the fixtures. The ratio panels carry bench's published +// The seven panels from the fixtures. The ratio panels carry bench's published // geomean ratios, unrounded. The time panels carry each system's geomean // time in ms (checked below to within 1e-9 of the values above), with its -// coverage in queries or datasets. The hand-kept GPU panel sits beside -// ClickBench. Each bench panel has a row for every system the card knows and +// coverage in queries or datasets. The hand-kept GPU panel follows the two +// query benchmarks. Each bench panel has a row for every system the card knows and // names the ones it shows: Lance everywhere, Arrow only in random access and // file size, where bench runs it. Titles join their parts with commas, so a // line breaks cleanly. @@ -133,6 +140,15 @@ const PANELS: Panel[] = [ rows: timeRows(CLICKBENCH_MS, 43), systems: LANCE }, + { + title: "TPC-H SF10 on DataFusion", + short: "TPC-H", + unit: "ms", + kind: "speed", + better: "lower", + rows: timeRows(TPCH_MS, 22), + systems: LANCE + }, GPU_PANEL, { title: "Random access, NVMe, hot", @@ -172,7 +188,7 @@ const near = (panels: Panel[] | null | undefined) => })); describe("panelsFrom", () => { - it("reads the six panels from bench's group summaries and charts, in order", () => { + it("reads the seven panels from bench's group summaries and charts, in order", () => { assert.deepEqual(near(panelsFrom(groups(), charts())), near(PANELS)); }); @@ -191,7 +207,8 @@ describe("panelsFrom", () => { }; for (const [i, g, key, name] of [ [2, "Clickbench", "rankings", (f: string) => `datafusion:${f}`], - [4, "Random Access (NVMe)", "hotRankings", (f: string) => f] + [3, "TPC-H (NVMe) (SF=10)", "rankings", (f: string) => `datafusion:${f}`], + [5, "Random Access (NVMe)", "hotRankings", (f: string) => f] ] as const) { const rows = panels?.[i]?.rows ?? []; const ms = (s: string) => rows.find((r) => r.system === s)?.score ?? NaN; @@ -222,7 +239,7 @@ describe("panelsFrom", () => { const gs = groups(); const hot = group(gs, "Random Access (NVMe)").summary; hot.hotRankings = (hot.hotRankings ?? []).filter((r) => r.name !== "lance"); - assert.deepEqual(panelsFrom(gs, charts())?.[4]?.rows[2], { + assert.deepEqual(panelsFrom(gs, charts())?.[5]?.rows[2], { system: "Lance", score: null, measured: 4, @@ -272,14 +289,14 @@ describe("panelsFrom", () => { name: c.name })); assert.equal( - panelsFrom(gs, cs)?.[4]?.unit, + panelsFrom(gs, cs)?.[5]?.unit, "ms", "a full window keeps its times" ); const dropped = charts(); dropped["Random Access (NVMe)"].charts = dropped["Random Access (NVMe)"].charts.slice(1); - assert.equal(panelsFrom(gs, dropped)?.[4]?.unit, "× fastest"); + assert.equal(panelsFrom(gs, dropped)?.[5]?.unit, "× fastest"); }); // geomean over a partial set, so the panel shows bench's scores instead. it("falls back to bench's scores when the window holds only some of the queries", () => { @@ -296,16 +313,17 @@ describe("panelsFrom", () => { it("falls back to bench's scores when a time panel's charts are missing or not times", () => { const check = (panels: ReturnType, i: number) => { const panel = panels?.[i]; - assert.ok(panel, "the card keeps all six panels"); + assert.ok(panel, "the card keeps all seven panels"); assert.equal(panel.unit, "× fastest"); const vortex = panel.rows.find((r) => r.system === "Vortex"); assert.ok(vortex?.score && vortex.score >= 1, "Vortex shows its score"); }; check(panelsFrom(groups()), 2); - check(panelsFrom(groups()), 4); + check(panelsFrom(groups()), 3); + check(panelsFrom(groups()), 5); const cs = charts(); for (const c of cs["Random Access (NVMe)"].charts) c.unit_kind = "bytes"; - check(panelsFrom(groups(), cs), 4); + check(panelsFrom(groups(), cs), 5); assert.equal( panelsFrom(groups(), cs)?.[2]?.unit, "ms", @@ -329,18 +347,19 @@ describe("panelsFrom", () => { ); assert.ok(parquet); parquet.ratio = "1"; - assert.deepEqual(panelsFrom(gs, charts())?.[5]?.rows[1], { + assert.deepEqual(panelsFrom(gs, charts())?.[6]?.rows[1], { system: "Parquet", score: null }); }); - it("gives null when a group is missing, so the card never shows five panels", () => { + it("gives null when a group is missing, so the card never shows part of the table", () => { for (const name of [ "Compression", "Compression Size", "Random Access (NVMe)", - "Clickbench" + "Clickbench", + "TPC-H (NVMe) (SF=10)" ]) { const gs = groups().filter((g) => g.name !== name); assert.equal(panelsFrom(gs, charts()), null, name); @@ -459,7 +478,11 @@ const bench = (url: string) => { if (url === "https://bench.vortex.dev/api/groups") return json(groupsBody, 200, headers); - for (const name of ["Clickbench", "Random Access (NVMe)"] as const) + for (const name of [ + "Clickbench", + "TPC-H (NVMe) (SF=10)", + "Random Access (NVMe)" + ] as const) if (url === `https://bench.vortex.dev/api/group/${slugOf(name)}?n=25`) return json(CHARTS[name]); return json({ error: "not_found" }, 404); @@ -468,31 +491,6 @@ const bench = // No waits between tries and no warnings, for the tests that do not look at them. const QUIET = { wait: async () => {}, warn: () => {} }; -describe("morePanelsFrom", () => { - it("gives TPC-H SF10 on DataFusion from bench's scores", () => { - const [tpch, ...rest] = morePanelsFrom(PAYLOAD.groups); - assert.equal(rest.length, 0); - assert.equal(tpch?.title, "TPC-H SF10 on DataFusion"); - assert.equal(tpch?.unit, "× fastest"); - const score = (system: string) => - tpch?.rows.find((r) => r.system === system)?.score; - // The live scores of 11 Nov 2026: DataFusion on each format. - assert.ok(Math.abs((score("Vortex") ?? 0) - 1.7528) < 1e-3); - assert.ok(Math.abs((score("Parquet") ?? 0) - 4.1513) < 1e-3); - assert.ok(Math.abs((score("Lance") ?? 0) - 24.5304) < 1e-3); - }); - - it("leaves out a panel bench does not give, and never fails", () => { - const groups = PAYLOAD.groups.filter( - (g) => g.name !== "TPC-H (NVMe) (SF=10)" - ); - assert.deepEqual(morePanelsFrom(groups), []); - assert.deepEqual(morePanelsFrom(null), []); - // The table's panels do not depend on it. - assert.equal(panelsFrom(groups)?.length, PANELS.length); - }); -}); - describe("getBenchSummary", () => { it("reads bench's groups and the time panels' charts, dated by bench's answer", async () => { const { fake, calls } = stub( @@ -500,13 +498,13 @@ describe("getBenchSummary", () => { ); const summary = await getBenchSummary(fake, QUIET); assert.deepEqual(near(summary?.panels), near(PANELS)); - assert.deepEqual(summary?.more, morePanelsFrom(PAYLOAD.groups)); assert.equal(summary?.readAt.toISOString(), "2026-10-29T21:23:34.000Z"); assert.deepEqual( calls.map((c) => c.url), [ "https://bench.vortex.dev/api/groups", `https://bench.vortex.dev/api/group/${slugOf("Clickbench")}?n=25`, + `https://bench.vortex.dev/api/group/${slugOf("TPC-H (NVMe) (SF=10)")}?n=25`, `https://bench.vortex.dev/api/group/${slugOf("Random Access (NVMe)")}?n=25` ] ); @@ -554,9 +552,10 @@ describe("getBenchSummary", () => { }); for (const fake of [failing.fake, throwing.fake]) { const u = await units(fake); - assert.equal(u?.length, 6, "the card keeps all six panels"); + assert.equal(u?.length, 7, "the card keeps all seven panels"); assert.equal(u?.[2], "× fastest"); - assert.equal(u?.[4], "× fastest"); + assert.equal(u?.[3], "× fastest"); + assert.equal(u?.[5], "× fastest"); } }); @@ -614,7 +613,7 @@ describe("getBenchSummary", () => { wait: async (ms) => void waits.push(ms), warn: (m) => void warnings.push(m) }); - assert.equal(summary?.panels.length, 6, "the third try gives the card"); + assert.equal(summary?.panels.length, 7, "the third try gives the card"); assert.equal(calls.filter((c) => c.url.endsWith("/api/groups")).length, 3); assert.deepEqual(waits, [500, 1000]); assert.equal(warnings.length, 2); @@ -635,7 +634,7 @@ describe("getBenchSummary", () => { wait: async () => {}, warn: (m) => void warnings.push(m) }); - assert.equal(summary?.panels.length, 6); + assert.equal(summary?.panels.length, 7); assert.equal(calls.filter((c) => c.url.endsWith("/api/groups")).length, 2); assert.match(warnings[0] ?? "", /SyntaxError.*try 1 of 3/); }); @@ -713,7 +712,7 @@ describe("routeConfig", () => { }); describe("summaryForPage", () => { - const summary = { panels: PANELS, more: [], readAt: new Date(0) }; + const summary = { panels: PANELS, readAt: new Date(0) }; it("passes a summary through", () => { assert.equal( diff --git a/tests/unit/faq.test.ts b/tests/unit/faq.test.ts index 106486f5..c226b011 100644 --- a/tests/unit/faq.test.ts +++ b/tests/unit/faq.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { describe, it } from "node:test"; -import { morePanelsFrom, panelsFrom } from "@/lib/bench-summary"; +import { panelsFrom } from "@/lib/bench-summary"; import { defined } from "@/lib/defined"; import { answerText, faqJsonLd, faqMarkdown, faqs } from "@/lib/faq"; import { GPU_DETAIL } from "@/lib/gpu-result"; @@ -15,10 +15,10 @@ const fixture = (name: string) => const PANELS = defined( panelsFrom(fixture("bench-groups.json").groups, { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }) ); -const MORE = morePanelsFrom(fixture("bench-groups.json").groups); const ECOSYSTEM = [ { name: "Acme", href: "https://acme.example" }, { name: "Apache Thing", href: "https://thing.example" }, @@ -53,7 +53,7 @@ describe("faqs", () => { assert.equal(parquet.list?.length, PANELS.length); assert.equal(parquet.list?.[0]?.label, "Scan throughput"); // The GPU row is not bench's, and the note after the list says whose. - assert.match(defined(parquet.list?.[3]?.text), /not from bench/); + assert.match(defined(parquet.list?.[4]?.text), /not from bench/); assert.equal(parquet.after?.[0], GPU_DETAIL); }); @@ -178,7 +178,7 @@ describe("faqs", () => { describe("Vortex against Lance, and TPC-H", () => { const list = faqs({ - bench: { panels: PANELS, more: MORE, readAt: READ }, + bench: { panels: PANELS, readAt: READ }, sparkVersion: null, ecosystem: ECOSYSTEM }); @@ -188,7 +188,7 @@ describe("Vortex against Lance, and TPC-H", () => { const at = labels?.indexOf("ClickBench on DataFusion") ?? -1; assert.ok(at >= 0); assert.equal(labels?.[at + 1], "TPC-H SF10 on DataFusion"); - assert.equal(labels?.length, PANELS.length + MORE.length); + assert.equal(labels?.length, PANELS.length); }); it("gives bench's verdicts against Lance, without the GPU result", () => { @@ -226,7 +226,7 @@ describe("Vortex against Lance, and TPC-H", () => { ); const lance = byId( faqs({ - bench: { panels, more: MORE, readAt: READ }, + bench: { panels, readAt: READ }, sparkVersion: null, ecosystem: ECOSYSTEM }), diff --git a/tests/unit/fixtures/bench-group-tpch-sf10-nvme.json b/tests/unit/fixtures/bench-group-tpch-sf10-nvme.json new file mode 100644 index 00000000..1db4395c --- /dev/null +++ b/tests/unit/fixtures/bench-group-tpch-sf10-nvme.json @@ -0,0 +1,425 @@ +{ + "name": "TPC-H (NVMe) (SF=10)", + "charts": [ + { + "name": "Q1", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxfQ", + "display_name": "tpch sf=10 Q1 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 948211142, 946309712, 945476790, 950109401, 949298738 + ], + "datafusion:parquet": [ + 87590775, 87479163, 87201711, 87754445, 87446147 + ], + "datafusion:vortex-file-compressed": [ + 91939751, 90751357, 91794474, 90882596, 91668592 + ], + "duckdb:parquet": [67642047, 67840767, 68072635, 67454233, 67799692] + } + }, + { + "name": "Q2", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyfQ", + "display_name": "tpch sf=10 Q2 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 325468862, 325610921, 328398805, 327267658, 328482079 + ], + "datafusion:parquet": [ + 173001227, 175200822, 173665320, 173381519, 173132245 + ], + "datafusion:vortex-file-compressed": [ + 64749044, 64358184, 65424879, 65985132, 64157437 + ], + "duckdb:parquet": [87453718, 86997836, 87578713, 87180253, 87272399] + } + }, + { + "name": "Q3", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjozfQ", + "display_name": "tpch sf=10 Q3 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1411430437, 1409388169, 1407097625, 1414220087, 1411639294 + ], + "datafusion:parquet": [ + 135649557, 135293400, 135558336, 134782889, 134804820 + ], + "datafusion:vortex-file-compressed": [ + 64050691, 65382830, 63710306, 63695061, 63657237 + ], + "duckdb:parquet": [ + 110567483, 110753947, 111719608, 109983956, 110545515 + ] + } + }, + { + "name": "Q4", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo0fQ", + "display_name": "tpch sf=10 Q4 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 695421961, 695067243, 694326652, 697055193, 697051899 + ], + "datafusion:parquet": [ + 57202538, 57054007, 57577536, 57207812, 57289061 + ], + "datafusion:vortex-file-compressed": [ + 34283004, 34846774, 34882784, 34668351, 34162712 + ], + "duckdb:parquet": [76733018, 76915000, 78743355, 76702096, 76936356] + } + }, + { + "name": "Q5", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo1fQ", + "display_name": "tpch sf=10 Q5 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1596080999, 1601701359, 1608356042, 1613509389, 1612466734 + ], + "datafusion:parquet": [ + 167683320, 167079408, 167744958, 167521409, 167714923 + ], + "datafusion:vortex-file-compressed": [ + 118037852, 119736703, 119020421, 118432923, 120157165 + ], + "duckdb:parquet": [ + 124407206, 124844237, 124478575, 124357441, 122898948 + ] + } + }, + { + "name": "Q6", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo2fQ", + "display_name": "tpch sf=10 Q6 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 692133649, 695416944, 692953352, 688203270, 691791011 + ], + "datafusion:parquet": [ + 36560516, 36618152, 36545996, 36466591, 36435117 + ], + "datafusion:vortex-file-compressed": [ + 9697022, 9572409, 9401868, 9410264, 9274230 + ], + "duckdb:parquet": [35792629, 35730042, 35325890, 35878982, 35797151] + } + }, + { + "name": "Q7", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo3fQ", + "display_name": "tpch sf=10 Q7 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1144480237, 1148337279, 1144749823, 1152967420, 1149568207 + ], + "datafusion:parquet": [ + 215148399, 216042322, 217405649, 216482379, 215000845 + ], + "datafusion:vortex-file-compressed": [ + 99114538, 97744817, 98425578, 97366359, 96655771 + ], + "duckdb:parquet": [ + 117687174, 117024456, 120768731, 118863439, 117515950 + ] + } + }, + { + "name": "Q8", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo4fQ", + "display_name": "tpch sf=10 Q8 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1802590691, 1801024858, 1812809695, 1811432715, 1803238744 + ], + "datafusion:parquet": [ + 218050936, 218193790, 217839688, 218218522, 218370744 + ], + "datafusion:vortex-file-compressed": [ + 143681291, 143213373, 142744673, 141487147, 142357582 + ], + "duckdb:parquet": [ + 148112126, 148716748, 148689359, 147423156, 147783655 + ] + } + }, + { + "name": "Q9", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo5fQ", + "display_name": "tpch sf=10 Q9 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 2330970908, 2324447702, 2335361710, 2328712740, 2332323197 + ], + "datafusion:parquet": [ + 310282771, 309796004, 310341156, 309566267, 308626137 + ], + "datafusion:vortex-file-compressed": [ + 187258459, 185612738, 185741301, 184786697, 186327450 + ], + "duckdb:parquet": [ + 237150963, 236740278, 237484738, 236763466, 236623990 + ] + } + }, + { + "name": "Q10", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxMH0", + "display_name": "tpch sf=10 Q10 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1375552441, 1371995942, 1376372509, 1372663617, 1369853390 + ], + "datafusion:parquet": [ + 311558361, 311780691, 313553301, 312510795, 311384572 + ], + "datafusion:vortex-file-compressed": [ + 70349202, 70279009, 72061446, 70842590, 70422309 + ], + "duckdb:parquet": [ + 315915097, 311936978, 313520469, 314231421, 313917651 + ] + } + }, + { + "name": "Q11", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxMX0", + "display_name": "tpch sf=10 Q11 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 306870434, 308174608, 308416604, 309559021, 308141885 + ], + "datafusion:parquet": [ + 126818447, 128074912, 128307377, 127115284, 127111318 + ], + "datafusion:vortex-file-compressed": [ + 45342712, 45612543, 45930139, 45447034, 45940037 + ], + "duckdb:parquet": [43807706, 43592664, 43677931, 43609854, 43662905] + } + }, + { + "name": "Q12", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxMn0", + "display_name": "tpch sf=10 Q12 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1808707516, 1812153786, 1806159854, 1805362030, 1809536704 + ], + "datafusion:parquet": [ + 82484423, 82584828, 82364790, 82646717, 82958453 + ], + "datafusion:vortex-file-compressed": [ + 35295049, 35826007, 35314120, 35260424, 35795803 + ], + "duckdb:parquet": [71842260, 71201480, 71018144, 70838812, 70400241] + } + }, + { + "name": "Q13", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxM30", + "display_name": "tpch sf=10 Q13 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1439196425, 1436911374, 1437951598, 1438515807, 1438391539 + ], + "datafusion:parquet": [ + 210666696, 211213193, 211072492, 210560923, 210637606 + ], + "datafusion:vortex-file-compressed": [ + 48856530, 48407672, 49584256, 49049902, 50223057 + ], + "duckdb:parquet": [ + 293498245, 292624997, 294018449, 293948861, 291009422 + ] + } + }, + { + "name": "Q14", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxNH0", + "display_name": "tpch sf=10 Q14 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 259944197, 259281176, 259774022, 258786668, 258909276 + ], + "datafusion:parquet": [ + 83865637, 85404641, 86406150, 86550088, 87085322 + ], + "datafusion:vortex-file-compressed": [ + 20813907, 20824769, 20659509, 20717499, 20473150 + ], + "duckdb:parquet": [ + 101002991, 101089869, 102804064, 101216362, 100304314 + ] + } + }, + { + "name": "Q15", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxNX0", + "display_name": "tpch sf=10 Q15 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 546640887, 540551102, 545745314, 547927148, 544767525 + ], + "datafusion:parquet": [ + 106399009, 107755461, 106583407, 104275468, 106605982 + ], + "datafusion:vortex-file-compressed": [ + 33819085, 33824513, 34137690, 33112722, 33123464 + ], + "duckdb:parquet": [57913875, 57951643, 59733786, 58034774, 57560768] + } + }, + { + "name": "Q16", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxNn0", + "display_name": "tpch sf=10 Q16 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 194034768, 194447028, 195005258, 194846573, 196029253 + ], + "datafusion:parquet": [ + 95410082, 95115969, 94518003, 94912151, 95351671 + ], + "datafusion:vortex-file-compressed": [ + 37111541, 36953983, 37020151, 36764337, 36651435 + ], + "duckdb:parquet": [100674942, 100051125, 100839649, 100437198, 99948268] + } + }, + { + "name": "Q17", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxN30", + "display_name": "tpch sf=10 Q17 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1326854613, 1333766859, 1331206683, 1335413967, 1336674736 + ], + "datafusion:parquet": [ + 217225687, 213988397, 215796077, 215246776, 216689697 + ], + "datafusion:vortex-file-compressed": [ + 182349051, 178043973, 178145666, 178579225, 177137907 + ], + "duckdb:parquet": [82764740, 81663301, 83844693, 82798057, 84697233] + } + }, + { + "name": "Q18", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxOH0", + "display_name": "tpch sf=10 Q18 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1440649652, 1447565213, 1457970951, 1453512110, 1453180343 + ], + "datafusion:parquet": [ + 267223597, 269449039, 268909981, 269683567, 269806321 + ], + "datafusion:vortex-file-compressed": [ + 208477602, 206819944, 206825599, 209891546, 205584619 + ], + "duckdb:parquet": [ + 160233436, 156207522, 160111090, 145561039, 158546132 + ] + } + }, + { + "name": "Q19", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxOX0", + "display_name": "tpch sf=10 Q19 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1291069493, 1290659659, 1285501663, 1277919355, 1277806242 + ], + "datafusion:parquet": [ + 108524663, 109301510, 109216745, 108063994, 108285967 + ], + "datafusion:vortex-file-compressed": [ + 28141283, 28254835, 27634644, 28033767, 28559250 + ], + "duckdb:parquet": [98397556, 99043844, 97268315, 97841079, 98916638] + } + }, + { + "name": "Q20", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyMH0", + "display_name": "tpch sf=10 Q20 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 503763831, 504950036, 505145380, 508262948, 504450435 + ], + "datafusion:parquet": [ + 185529402, 185329106, 185575719, 186207395, 186134779 + ], + "datafusion:vortex-file-compressed": [ + 60140392, 61384303, 60699908, 60466586, 60820058 + ], + "duckdb:parquet": [ + 147049224, 146755400, 147628594, 146875965, 147543765 + ] + } + }, + { + "name": "Q21", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyMX0", + "display_name": "tpch sf=10 Q21 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1832732880, 1833507460, 1835857805, 1839811977, 1834144425 + ], + "datafusion:parquet": [ + 242714876, 242244157, 245362052, 245593952, 244867534 + ], + "datafusion:vortex-file-compressed": [ + 179957786, 183142490, 182840804, 181406322, 183253187 + ], + "duckdb:parquet": [ + 256489495, 253369561, 253195089, 255978191, 257690252 + ] + } + }, + { + "name": "Q22", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyMn0", + "display_name": "tpch sf=10 Q22 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 224384137, 224452763, 223847181, 225566381, 223264068 + ], + "datafusion:parquet": [ + 149828009, 149997964, 149129297, 149601609, 149767211 + ], + "datafusion:vortex-file-compressed": [ + 25806272, 25906261, 25316392, 25608014, 25882764 + ], + "duckdb:parquet": [ + 168308776, 168238576, 168707850, 168478030, 168086527 + ] + } + } + ] +} diff --git a/tests/unit/llms.test.ts b/tests/unit/llms.test.ts index 21eb714d..a1d85658 100644 --- a/tests/unit/llms.test.ts +++ b/tests/unit/llms.test.ts @@ -18,6 +18,7 @@ const fixture = (name: string) => ) as Record; const PANELS = panelsFrom(fixture("bench-groups.json").groups, { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }); const READ = new Date("2026-11-02T12:00:00Z"); @@ -29,13 +30,17 @@ describe("resultsSentences", () => { it("says what each row measures, its figures, and the verdict", () => { assert.ok(PANELS); const sentences = resultsSentences(PANELS); - assert.equal(sentences.length, 6); + assert.equal(sentences.length, 7); for (const s of sentences) assert.match(s, /Vortex against Parquet: .+\.$/); assert.match(sentences[0] ?? "", /^Scan, how fast each format scans/); - assert.match(sentences[3] ?? "", /^GPU, cuDF, query time on a GPU in cuDF/); - // Lance cannot run in cuDF, and the sentence says why, not a figure. assert.match( sentences[3] ?? "", + /^TPC-H, the geometric mean time of the 22/ + ); + assert.match(sentences[4] ?? "", /^GPU, cuDF, query time on a GPU in cuDF/); + // Lance cannot run in cuDF, and the sentence says why, not a figure. + assert.match( + sentences[4] ?? "", /Lance not available, no cuDF integration/ ); }); diff --git a/tests/unit/results-table.test.tsx b/tests/unit/results-table.test.tsx index 9645af4a..7b7e8ea9 100644 --- a/tests/unit/results-table.test.tsx +++ b/tests/unit/results-table.test.tsx @@ -13,6 +13,7 @@ const fixture = (name: string) => ) as Record; const PANELS = panelsFrom(fixture("bench-groups.json").groups, { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }); const READ = new Date("2026-11-02T12:00:00Z"); @@ -41,6 +42,7 @@ describe("ResultsTable", () => { "Scan", "Write", "ClickBench", + "TPC-H", "GPU, cuDF", "Random access", "File size" @@ -49,7 +51,7 @@ describe("ResultsTable", () => { html.includes(`${name}`), name ); - assert.equal(html.match(/