diff --git a/src/lib/bench-summary.ts b/src/lib/bench-summary.ts index 43b55f9..8889aac 100644 --- a/src/lib/bench-summary.ts +++ b/src/lib/bench-summary.ts @@ -64,11 +64,7 @@ export type Panel = { source?: { note: string }; }; -/** - * The results table's panels, the panels /faq adds to them - * (`morePanelsFrom`), and when bench answered. - */ -export type BenchSummary = { panels: Panel[]; more: Panel[]; readAt: Date }; +export type BenchSummary = { panels: Panel[]; readAt: Date }; // The systems the table can compare, by bench's format name, in the order the // rows are listed (the table sorts them best first). Each panel chooses the @@ -125,9 +121,10 @@ type Source = { }; }; -// The six panels, in the table's order: five from bench and, beside -// ClickBench, the GPU result bench does not publish yet, kept by hand in -// gpu-result.ts with its source. +// The seven panels, in the table's order: six from bench and, after the two +// query benchmarks, the GPU result bench does not publish yet, kept by hand +// in gpu-result.ts with its source. ClickBench and TPC-H run on DataFusion, +// the one engine that runs all three formats. const SOURCES: (Source | Panel)[] = [ { title: "Scan throughput", @@ -168,6 +165,19 @@ const SOURCES: (Source | Panel)[] = [ systems: WITH_LANCE, times: { series: (f) => `datafusion:${f}`, bucket: (chart) => chart } }, + { + title: "TPC-H SF10 on DataFusion", + short: "TPC-H", + unit: "ms", + kind: "speed", + better: "lower", + group: "TPC-H (NVMe) (SF=10)", + rankings: (s) => s.rankings, + name: (f) => `datafusion:${f}`, + score: "score", + systems: WITH_LANCE, + times: { series: (f) => `datafusion:${f}`, bucket: (chart) => chart } + }, GPU_PANEL, { title: "Random access, NVMe, hot", @@ -319,24 +329,6 @@ function timeRows( }); } -// The panels /faq adds to the table's: TPC-H at scale factor 10 on -// DataFusion, the one engine that runs all three formats. A panel shows -// bench's scores against the fastest, so it needs no charts. -const MORE_SOURCES: Source[] = [ - { - title: "TPC-H SF10 on DataFusion", - short: "TPC-H", - unit: "× fastest", - kind: "speed", - better: "lower", - group: "TPC-H (NVMe) (SF=10)", - rankings: (s) => s.rankings, - name: (f) => `datafusion:${f}`, - score: "score", - systems: WITH_LANCE - } -]; - /** * One bench panel from its group in bench's `/api/groups` payload and, for * a time panel, the group's charts. A shown system bench did not report @@ -397,10 +389,10 @@ function panelFor( } /** - * The results table's six panels: five from bench (its `/api/groups` + * The results table's seven panels: six from bench (its `/api/groups` * payload's `groups` array and, for a time panel, `charts`, its group's * `/api/group` payload by group name) and the hand-kept GPU panel. A bench - * panel that `panelFor` cannot build gives `null`: the table shows six + * panel that `panelFor` cannot build gives `null`: the table shows seven * panels or none. */ export function panelsFrom( @@ -421,16 +413,6 @@ export function panelsFrom( return panels; } -/** - * The panels /faq adds to the table's (`MORE_SOURCES`), from the same - * `/api/groups` payload. One that bench does not give is left out, so the - * table's panels never depend on these. - */ -export function morePanelsFrom(groups: unknown): Panel[] { - if (!Array.isArray(groups)) return []; - return MORE_SOURCES.flatMap((source) => panelFor(source, groups, {}) ?? []); -} - /** * The homepage's benchmark figures, read from bench.vortex.dev at build time * and again when the page rebuilds, at most once a day (ISR). A figure is never typed into the page. @@ -439,7 +421,7 @@ export function morePanelsFrom(groups: unknown): Panel[] { * bench answered the first read (its Date header), so a cached answer keeps * its own date. A read that fails or gets a 5xx or 429 is tried again, up to * three times. Any failure after that (a timeout, an error status, a payload - * without all six panels) gives `null`, with a warning that says why, and + * without all seven panels) gives `null`, with a warning that says why, and * the page then leaves the table out (but see `summaryForPage`). A time * panel whose charts are missing from the window shows bench's scores. * `fetchImpl` and `io` are for tests. @@ -514,11 +496,10 @@ export async function getBenchSummary( ) ); const panels = panelsFrom(groups, charts); - if (!panels) return fail("bench's groups do not give all six panels"); + if (!panels) return fail("bench's groups do not give all seven panels"); const date = new Date(answer.date ?? ""); return { panels, - more: morePanelsFrom(groups), readAt: Number.isNaN(date.getTime()) ? new Date() : date }; } catch (e) { diff --git a/src/lib/faq.ts b/src/lib/faq.ts index 0bde443..ac0e22f 100644 --- a/src/lib/faq.ts +++ b/src/lib/faq.ts @@ -49,19 +49,6 @@ export type Faq = { // The table formats in the Ecosystem band, for the answer on Lance. const TABLE_FORMATS = ["Apache Iceberg", "Apache Hudi", "Apache Paimon"]; -/** - * The panels the answers compare: the table's, with the panels /faq adds - * (`more`) after ClickBench, so the query benchmarks sit together. - */ -function comparedPanels( - panels: readonly Panel[], - more: readonly Panel[] -): Panel[] { - const at = panels.findIndex((p) => p.title === "ClickBench on DataFusion"); - const cut = at === -1 ? panels.length : at + 1; - return [...panels.slice(0, cut), ...more, ...panels.slice(cut)]; -} - /** * Vortex against Lance in a panel, in the words the table uses against * Parquet ("14× faster", "48% smaller"), or "Lance not measured". `null` @@ -100,15 +87,11 @@ export function faqs({ sparkVersion, ecosystem }: { - bench: { - panels: readonly Panel[]; - more?: readonly Panel[]; - readAt: Date; - } | null; + bench: { panels: readonly Panel[]; readAt: Date } | null; sparkVersion: string | null; ecosystem: readonly EcosystemEntry[]; }): Faq[] { - const compared = bench ? comparedPanels(bench.panels, bench.more ?? []) : []; + const compared = bench?.panels ?? []; const systems = resultsSystems(compared); const verdict = (panel: Panel) => resultsRow(panel, systems).verdict; const hasGpu = compared.some((p) => p.title === GPU_PANEL.title); diff --git a/src/lib/llms.ts b/src/lib/llms.ts index 4b5f549..bd1a6e2 100644 --- a/src/lib/llms.ts +++ b/src/lib/llms.ts @@ -50,6 +50,10 @@ function about(panel: Panel): string { return panel.unit === "ms" ? "the geometric mean time of the ClickBench queries on DataFusion, reading files on local NVMe, so lower is better" : "the ClickBench queries on DataFusion, reading files on local NVMe: the geometric mean of each query's time divided by the fastest format's, so lower is better"; + case "TPC-H SF10 on DataFusion": + return panel.unit === "ms" + ? "the geometric mean time of the 22 TPC-H queries at scale factor 10 on DataFusion, reading files on local NVMe, so lower is better" + : "the TPC-H queries at scale factor 10 on DataFusion, reading files on local NVMe: the geometric mean of each query's time divided by the fastest format's, so lower is better"; case "Random access, NVMe, hot": return panel.unit === "ms" ? "the geometric mean time to take rows from each of the random access benchmark's datasets on local NVMe, reusing an open file, so lower is better" diff --git a/tests/smoke.spec.ts b/tests/smoke.spec.ts index 09c1775..664e384 100644 --- a/tests/smoke.spec.ts +++ b/tests/smoke.spec.ts @@ -182,7 +182,7 @@ test.describe("home", () => { }); test.describe("home bands", () => { - test("the results table shows six benchmarks with a source link, or is absent", async ({ + test("the results table shows seven benchmarks with a source link, or is absent", async ({ page }) => { await page.goto("/"); @@ -194,8 +194,8 @@ test.describe("home bands", () => { return; } await expect(table).toBeVisible(); - await expect(table.locator("tbody tr")).toHaveCount(6); - await expect(table.locator("tbody td.is-ours")).toHaveCount(6); + await expect(table.locator("tbody tr")).toHaveCount(7); + await expect(table.locator("tbody td.is-ours")).toHaveCount(7); await expect(table).toContainText(/read \d+ \w{3} \d{4}/); const source = table.locator('a[href="https://bench.vortex.dev"]'); await expect(source).toBeVisible(); @@ -226,6 +226,7 @@ test.describe("home bands", () => { "Scan", "Write", "ClickBench", + "TPC-H", "GPU, cuDF", "Random access", "File size" diff --git a/tests/unit/bench-summary.test.ts b/tests/unit/bench-summary.test.ts index f462722..33a4568 100644 --- a/tests/unit/bench-summary.test.ts +++ b/tests/unit/bench-summary.test.ts @@ -5,7 +5,6 @@ import { formatReadDate, getBenchSummary, groupTimes, - morePanelsFrom, type Panel, panelsFrom, type Row, @@ -34,9 +33,11 @@ const PAYLOAD = fixture("bench-groups.json") as { // bench.vortex.dev/api/group/{slug}?n=25 for the two time panels' groups, read // at the same time, cut down to the series the card reads (and one it does // not) and their last five commits. Random access's hot series were read on -// 30 Oct 2026, from the same two commits as the rest. +// 30 Oct 2026, from the same two commits as the rest. TPC-H (NVMe) (SF=10)'s +// were read on 11 Nov 2026, with its group summary. const CHARTS = { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }; @@ -72,6 +73,12 @@ const CLICKBENCH_MS = { Parquet: 65.621614465414, Lance: 578.6306006236966 }; +// TPC-H SF10's, the same way (Python, 11 Nov 2026), over its 22 queries. +const TPCH_MS = { + Vortex: 60.50866557993905, + Parquet: 143.30314256082335, + Lance: 846.7997612370476 +}; const RANDOM_ACCESS_MS = { Vortex: 0.7290522397517741, Parquet: 70.61301694616571, @@ -79,11 +86,11 @@ const RANDOM_ACCESS_MS = { Arrow: 3.7780608403173668 }; -// The six panels from the fixtures. The ratio panels carry bench's published +// The seven panels from the fixtures. The ratio panels carry bench's published // geomean ratios, unrounded. The time panels carry each system's geomean // time in ms (checked below to within 1e-9 of the values above), with its -// coverage in queries or datasets. The hand-kept GPU panel sits beside -// ClickBench. Each bench panel has a row for every system the card knows and +// coverage in queries or datasets. The hand-kept GPU panel follows the two +// query benchmarks. Each bench panel has a row for every system the card knows and // names the ones it shows: Lance everywhere, Arrow only in random access and // file size, where bench runs it. Titles join their parts with commas, so a // line breaks cleanly. @@ -133,6 +140,15 @@ const PANELS: Panel[] = [ rows: timeRows(CLICKBENCH_MS, 43), systems: LANCE }, + { + title: "TPC-H SF10 on DataFusion", + short: "TPC-H", + unit: "ms", + kind: "speed", + better: "lower", + rows: timeRows(TPCH_MS, 22), + systems: LANCE + }, GPU_PANEL, { title: "Random access, NVMe, hot", @@ -172,7 +188,7 @@ const near = (panels: Panel[] | null | undefined) => })); describe("panelsFrom", () => { - it("reads the six panels from bench's group summaries and charts, in order", () => { + it("reads the seven panels from bench's group summaries and charts, in order", () => { assert.deepEqual(near(panelsFrom(groups(), charts())), near(PANELS)); }); @@ -191,7 +207,8 @@ describe("panelsFrom", () => { }; for (const [i, g, key, name] of [ [2, "Clickbench", "rankings", (f: string) => `datafusion:${f}`], - [4, "Random Access (NVMe)", "hotRankings", (f: string) => f] + [3, "TPC-H (NVMe) (SF=10)", "rankings", (f: string) => `datafusion:${f}`], + [5, "Random Access (NVMe)", "hotRankings", (f: string) => f] ] as const) { const rows = panels?.[i]?.rows ?? []; const ms = (s: string) => rows.find((r) => r.system === s)?.score ?? NaN; @@ -222,7 +239,7 @@ describe("panelsFrom", () => { const gs = groups(); const hot = group(gs, "Random Access (NVMe)").summary; hot.hotRankings = (hot.hotRankings ?? []).filter((r) => r.name !== "lance"); - assert.deepEqual(panelsFrom(gs, charts())?.[4]?.rows[2], { + assert.deepEqual(panelsFrom(gs, charts())?.[5]?.rows[2], { system: "Lance", score: null, measured: 4, @@ -272,14 +289,14 @@ describe("panelsFrom", () => { name: c.name })); assert.equal( - panelsFrom(gs, cs)?.[4]?.unit, + panelsFrom(gs, cs)?.[5]?.unit, "ms", "a full window keeps its times" ); const dropped = charts(); dropped["Random Access (NVMe)"].charts = dropped["Random Access (NVMe)"].charts.slice(1); - assert.equal(panelsFrom(gs, dropped)?.[4]?.unit, "× fastest"); + assert.equal(panelsFrom(gs, dropped)?.[5]?.unit, "× fastest"); }); // geomean over a partial set, so the panel shows bench's scores instead. it("falls back to bench's scores when the window holds only some of the queries", () => { @@ -296,16 +313,17 @@ describe("panelsFrom", () => { it("falls back to bench's scores when a time panel's charts are missing or not times", () => { const check = (panels: ReturnType, i: number) => { const panel = panels?.[i]; - assert.ok(panel, "the card keeps all six panels"); + assert.ok(panel, "the card keeps all seven panels"); assert.equal(panel.unit, "× fastest"); const vortex = panel.rows.find((r) => r.system === "Vortex"); assert.ok(vortex?.score && vortex.score >= 1, "Vortex shows its score"); }; check(panelsFrom(groups()), 2); - check(panelsFrom(groups()), 4); + check(panelsFrom(groups()), 3); + check(panelsFrom(groups()), 5); const cs = charts(); for (const c of cs["Random Access (NVMe)"].charts) c.unit_kind = "bytes"; - check(panelsFrom(groups(), cs), 4); + check(panelsFrom(groups(), cs), 5); assert.equal( panelsFrom(groups(), cs)?.[2]?.unit, "ms", @@ -329,18 +347,19 @@ describe("panelsFrom", () => { ); assert.ok(parquet); parquet.ratio = "1"; - assert.deepEqual(panelsFrom(gs, charts())?.[5]?.rows[1], { + assert.deepEqual(panelsFrom(gs, charts())?.[6]?.rows[1], { system: "Parquet", score: null }); }); - it("gives null when a group is missing, so the card never shows five panels", () => { + it("gives null when a group is missing, so the card never shows part of the table", () => { for (const name of [ "Compression", "Compression Size", "Random Access (NVMe)", - "Clickbench" + "Clickbench", + "TPC-H (NVMe) (SF=10)" ]) { const gs = groups().filter((g) => g.name !== name); assert.equal(panelsFrom(gs, charts()), null, name); @@ -459,7 +478,11 @@ const bench = (url: string) => { if (url === "https://bench.vortex.dev/api/groups") return json(groupsBody, 200, headers); - for (const name of ["Clickbench", "Random Access (NVMe)"] as const) + for (const name of [ + "Clickbench", + "TPC-H (NVMe) (SF=10)", + "Random Access (NVMe)" + ] as const) if (url === `https://bench.vortex.dev/api/group/${slugOf(name)}?n=25`) return json(CHARTS[name]); return json({ error: "not_found" }, 404); @@ -468,31 +491,6 @@ const bench = // No waits between tries and no warnings, for the tests that do not look at them. const QUIET = { wait: async () => {}, warn: () => {} }; -describe("morePanelsFrom", () => { - it("gives TPC-H SF10 on DataFusion from bench's scores", () => { - const [tpch, ...rest] = morePanelsFrom(PAYLOAD.groups); - assert.equal(rest.length, 0); - assert.equal(tpch?.title, "TPC-H SF10 on DataFusion"); - assert.equal(tpch?.unit, "× fastest"); - const score = (system: string) => - tpch?.rows.find((r) => r.system === system)?.score; - // The live scores of 11 Nov 2026: DataFusion on each format. - assert.ok(Math.abs((score("Vortex") ?? 0) - 1.7528) < 1e-3); - assert.ok(Math.abs((score("Parquet") ?? 0) - 4.1513) < 1e-3); - assert.ok(Math.abs((score("Lance") ?? 0) - 24.5304) < 1e-3); - }); - - it("leaves out a panel bench does not give, and never fails", () => { - const groups = PAYLOAD.groups.filter( - (g) => g.name !== "TPC-H (NVMe) (SF=10)" - ); - assert.deepEqual(morePanelsFrom(groups), []); - assert.deepEqual(morePanelsFrom(null), []); - // The table's panels do not depend on it. - assert.equal(panelsFrom(groups)?.length, PANELS.length); - }); -}); - describe("getBenchSummary", () => { it("reads bench's groups and the time panels' charts, dated by bench's answer", async () => { const { fake, calls } = stub( @@ -500,13 +498,13 @@ describe("getBenchSummary", () => { ); const summary = await getBenchSummary(fake, QUIET); assert.deepEqual(near(summary?.panels), near(PANELS)); - assert.deepEqual(summary?.more, morePanelsFrom(PAYLOAD.groups)); assert.equal(summary?.readAt.toISOString(), "2026-10-29T21:23:34.000Z"); assert.deepEqual( calls.map((c) => c.url), [ "https://bench.vortex.dev/api/groups", `https://bench.vortex.dev/api/group/${slugOf("Clickbench")}?n=25`, + `https://bench.vortex.dev/api/group/${slugOf("TPC-H (NVMe) (SF=10)")}?n=25`, `https://bench.vortex.dev/api/group/${slugOf("Random Access (NVMe)")}?n=25` ] ); @@ -554,9 +552,10 @@ describe("getBenchSummary", () => { }); for (const fake of [failing.fake, throwing.fake]) { const u = await units(fake); - assert.equal(u?.length, 6, "the card keeps all six panels"); + assert.equal(u?.length, 7, "the card keeps all seven panels"); assert.equal(u?.[2], "× fastest"); - assert.equal(u?.[4], "× fastest"); + assert.equal(u?.[3], "× fastest"); + assert.equal(u?.[5], "× fastest"); } }); @@ -614,7 +613,7 @@ describe("getBenchSummary", () => { wait: async (ms) => void waits.push(ms), warn: (m) => void warnings.push(m) }); - assert.equal(summary?.panels.length, 6, "the third try gives the card"); + assert.equal(summary?.panels.length, 7, "the third try gives the card"); assert.equal(calls.filter((c) => c.url.endsWith("/api/groups")).length, 3); assert.deepEqual(waits, [500, 1000]); assert.equal(warnings.length, 2); @@ -635,7 +634,7 @@ describe("getBenchSummary", () => { wait: async () => {}, warn: (m) => void warnings.push(m) }); - assert.equal(summary?.panels.length, 6); + assert.equal(summary?.panels.length, 7); assert.equal(calls.filter((c) => c.url.endsWith("/api/groups")).length, 2); assert.match(warnings[0] ?? "", /SyntaxError.*try 1 of 3/); }); @@ -713,7 +712,7 @@ describe("routeConfig", () => { }); describe("summaryForPage", () => { - const summary = { panels: PANELS, more: [], readAt: new Date(0) }; + const summary = { panels: PANELS, readAt: new Date(0) }; it("passes a summary through", () => { assert.equal( diff --git a/tests/unit/faq.test.ts b/tests/unit/faq.test.ts index 106486f..c226b01 100644 --- a/tests/unit/faq.test.ts +++ b/tests/unit/faq.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { describe, it } from "node:test"; -import { morePanelsFrom, panelsFrom } from "@/lib/bench-summary"; +import { panelsFrom } from "@/lib/bench-summary"; import { defined } from "@/lib/defined"; import { answerText, faqJsonLd, faqMarkdown, faqs } from "@/lib/faq"; import { GPU_DETAIL } from "@/lib/gpu-result"; @@ -15,10 +15,10 @@ const fixture = (name: string) => const PANELS = defined( panelsFrom(fixture("bench-groups.json").groups, { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }) ); -const MORE = morePanelsFrom(fixture("bench-groups.json").groups); const ECOSYSTEM = [ { name: "Acme", href: "https://acme.example" }, { name: "Apache Thing", href: "https://thing.example" }, @@ -53,7 +53,7 @@ describe("faqs", () => { assert.equal(parquet.list?.length, PANELS.length); assert.equal(parquet.list?.[0]?.label, "Scan throughput"); // The GPU row is not bench's, and the note after the list says whose. - assert.match(defined(parquet.list?.[3]?.text), /not from bench/); + assert.match(defined(parquet.list?.[4]?.text), /not from bench/); assert.equal(parquet.after?.[0], GPU_DETAIL); }); @@ -178,7 +178,7 @@ describe("faqs", () => { describe("Vortex against Lance, and TPC-H", () => { const list = faqs({ - bench: { panels: PANELS, more: MORE, readAt: READ }, + bench: { panels: PANELS, readAt: READ }, sparkVersion: null, ecosystem: ECOSYSTEM }); @@ -188,7 +188,7 @@ describe("Vortex against Lance, and TPC-H", () => { const at = labels?.indexOf("ClickBench on DataFusion") ?? -1; assert.ok(at >= 0); assert.equal(labels?.[at + 1], "TPC-H SF10 on DataFusion"); - assert.equal(labels?.length, PANELS.length + MORE.length); + assert.equal(labels?.length, PANELS.length); }); it("gives bench's verdicts against Lance, without the GPU result", () => { @@ -226,7 +226,7 @@ describe("Vortex against Lance, and TPC-H", () => { ); const lance = byId( faqs({ - bench: { panels, more: MORE, readAt: READ }, + bench: { panels, readAt: READ }, sparkVersion: null, ecosystem: ECOSYSTEM }), diff --git a/tests/unit/fixtures/bench-group-tpch-sf10-nvme.json b/tests/unit/fixtures/bench-group-tpch-sf10-nvme.json new file mode 100644 index 0000000..1db4395 --- /dev/null +++ b/tests/unit/fixtures/bench-group-tpch-sf10-nvme.json @@ -0,0 +1,425 @@ +{ + "name": "TPC-H (NVMe) (SF=10)", + "charts": [ + { + "name": "Q1", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxfQ", + "display_name": "tpch sf=10 Q1 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 948211142, 946309712, 945476790, 950109401, 949298738 + ], + "datafusion:parquet": [ + 87590775, 87479163, 87201711, 87754445, 87446147 + ], + "datafusion:vortex-file-compressed": [ + 91939751, 90751357, 91794474, 90882596, 91668592 + ], + "duckdb:parquet": [67642047, 67840767, 68072635, 67454233, 67799692] + } + }, + { + "name": "Q2", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyfQ", + "display_name": "tpch sf=10 Q2 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 325468862, 325610921, 328398805, 327267658, 328482079 + ], + "datafusion:parquet": [ + 173001227, 175200822, 173665320, 173381519, 173132245 + ], + "datafusion:vortex-file-compressed": [ + 64749044, 64358184, 65424879, 65985132, 64157437 + ], + "duckdb:parquet": [87453718, 86997836, 87578713, 87180253, 87272399] + } + }, + { + "name": "Q3", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjozfQ", + "display_name": "tpch sf=10 Q3 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1411430437, 1409388169, 1407097625, 1414220087, 1411639294 + ], + "datafusion:parquet": [ + 135649557, 135293400, 135558336, 134782889, 134804820 + ], + "datafusion:vortex-file-compressed": [ + 64050691, 65382830, 63710306, 63695061, 63657237 + ], + "duckdb:parquet": [ + 110567483, 110753947, 111719608, 109983956, 110545515 + ] + } + }, + { + "name": "Q4", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo0fQ", + "display_name": "tpch sf=10 Q4 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 695421961, 695067243, 694326652, 697055193, 697051899 + ], + "datafusion:parquet": [ + 57202538, 57054007, 57577536, 57207812, 57289061 + ], + "datafusion:vortex-file-compressed": [ + 34283004, 34846774, 34882784, 34668351, 34162712 + ], + "duckdb:parquet": [76733018, 76915000, 78743355, 76702096, 76936356] + } + }, + { + "name": "Q5", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo1fQ", + "display_name": "tpch sf=10 Q5 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1596080999, 1601701359, 1608356042, 1613509389, 1612466734 + ], + "datafusion:parquet": [ + 167683320, 167079408, 167744958, 167521409, 167714923 + ], + "datafusion:vortex-file-compressed": [ + 118037852, 119736703, 119020421, 118432923, 120157165 + ], + "duckdb:parquet": [ + 124407206, 124844237, 124478575, 124357441, 122898948 + ] + } + }, + { + "name": "Q6", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo2fQ", + "display_name": "tpch sf=10 Q6 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 692133649, 695416944, 692953352, 688203270, 691791011 + ], + "datafusion:parquet": [ + 36560516, 36618152, 36545996, 36466591, 36435117 + ], + "datafusion:vortex-file-compressed": [ + 9697022, 9572409, 9401868, 9410264, 9274230 + ], + "duckdb:parquet": [35792629, 35730042, 35325890, 35878982, 35797151] + } + }, + { + "name": "Q7", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo3fQ", + "display_name": "tpch sf=10 Q7 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1144480237, 1148337279, 1144749823, 1152967420, 1149568207 + ], + "datafusion:parquet": [ + 215148399, 216042322, 217405649, 216482379, 215000845 + ], + "datafusion:vortex-file-compressed": [ + 99114538, 97744817, 98425578, 97366359, 96655771 + ], + "duckdb:parquet": [ + 117687174, 117024456, 120768731, 118863439, 117515950 + ] + } + }, + { + "name": "Q8", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo4fQ", + "display_name": "tpch sf=10 Q8 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1802590691, 1801024858, 1812809695, 1811432715, 1803238744 + ], + "datafusion:parquet": [ + 218050936, 218193790, 217839688, 218218522, 218370744 + ], + "datafusion:vortex-file-compressed": [ + 143681291, 143213373, 142744673, 141487147, 142357582 + ], + "duckdb:parquet": [ + 148112126, 148716748, 148689359, 147423156, 147783655 + ] + } + }, + { + "name": "Q9", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4Ijo5fQ", + "display_name": "tpch sf=10 Q9 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 2330970908, 2324447702, 2335361710, 2328712740, 2332323197 + ], + "datafusion:parquet": [ + 310282771, 309796004, 310341156, 309566267, 308626137 + ], + "datafusion:vortex-file-compressed": [ + 187258459, 185612738, 185741301, 184786697, 186327450 + ], + "duckdb:parquet": [ + 237150963, 236740278, 237484738, 236763466, 236623990 + ] + } + }, + { + "name": "Q10", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxMH0", + "display_name": "tpch sf=10 Q10 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1375552441, 1371995942, 1376372509, 1372663617, 1369853390 + ], + "datafusion:parquet": [ + 311558361, 311780691, 313553301, 312510795, 311384572 + ], + "datafusion:vortex-file-compressed": [ + 70349202, 70279009, 72061446, 70842590, 70422309 + ], + "duckdb:parquet": [ + 315915097, 311936978, 313520469, 314231421, 313917651 + ] + } + }, + { + "name": "Q11", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxMX0", + "display_name": "tpch sf=10 Q11 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 306870434, 308174608, 308416604, 309559021, 308141885 + ], + "datafusion:parquet": [ + 126818447, 128074912, 128307377, 127115284, 127111318 + ], + "datafusion:vortex-file-compressed": [ + 45342712, 45612543, 45930139, 45447034, 45940037 + ], + "duckdb:parquet": [43807706, 43592664, 43677931, 43609854, 43662905] + } + }, + { + "name": "Q12", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxMn0", + "display_name": "tpch sf=10 Q12 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1808707516, 1812153786, 1806159854, 1805362030, 1809536704 + ], + "datafusion:parquet": [ + 82484423, 82584828, 82364790, 82646717, 82958453 + ], + "datafusion:vortex-file-compressed": [ + 35295049, 35826007, 35314120, 35260424, 35795803 + ], + "duckdb:parquet": [71842260, 71201480, 71018144, 70838812, 70400241] + } + }, + { + "name": "Q13", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxM30", + "display_name": "tpch sf=10 Q13 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1439196425, 1436911374, 1437951598, 1438515807, 1438391539 + ], + "datafusion:parquet": [ + 210666696, 211213193, 211072492, 210560923, 210637606 + ], + "datafusion:vortex-file-compressed": [ + 48856530, 48407672, 49584256, 49049902, 50223057 + ], + "duckdb:parquet": [ + 293498245, 292624997, 294018449, 293948861, 291009422 + ] + } + }, + { + "name": "Q14", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxNH0", + "display_name": "tpch sf=10 Q14 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 259944197, 259281176, 259774022, 258786668, 258909276 + ], + "datafusion:parquet": [ + 83865637, 85404641, 86406150, 86550088, 87085322 + ], + "datafusion:vortex-file-compressed": [ + 20813907, 20824769, 20659509, 20717499, 20473150 + ], + "duckdb:parquet": [ + 101002991, 101089869, 102804064, 101216362, 100304314 + ] + } + }, + { + "name": "Q15", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxNX0", + "display_name": "tpch sf=10 Q15 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 546640887, 540551102, 545745314, 547927148, 544767525 + ], + "datafusion:parquet": [ + 106399009, 107755461, 106583407, 104275468, 106605982 + ], + "datafusion:vortex-file-compressed": [ + 33819085, 33824513, 34137690, 33112722, 33123464 + ], + "duckdb:parquet": [57913875, 57951643, 59733786, 58034774, 57560768] + } + }, + { + "name": "Q16", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxNn0", + "display_name": "tpch sf=10 Q16 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 194034768, 194447028, 195005258, 194846573, 196029253 + ], + "datafusion:parquet": [ + 95410082, 95115969, 94518003, 94912151, 95351671 + ], + "datafusion:vortex-file-compressed": [ + 37111541, 36953983, 37020151, 36764337, 36651435 + ], + "duckdb:parquet": [100674942, 100051125, 100839649, 100437198, 99948268] + } + }, + { + "name": "Q17", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxN30", + "display_name": "tpch sf=10 Q17 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1326854613, 1333766859, 1331206683, 1335413967, 1336674736 + ], + "datafusion:parquet": [ + 217225687, 213988397, 215796077, 215246776, 216689697 + ], + "datafusion:vortex-file-compressed": [ + 182349051, 178043973, 178145666, 178579225, 177137907 + ], + "duckdb:parquet": [82764740, 81663301, 83844693, 82798057, 84697233] + } + }, + { + "name": "Q18", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxOH0", + "display_name": "tpch sf=10 Q18 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1440649652, 1447565213, 1457970951, 1453512110, 1453180343 + ], + "datafusion:parquet": [ + 267223597, 269449039, 268909981, 269683567, 269806321 + ], + "datafusion:vortex-file-compressed": [ + 208477602, 206819944, 206825599, 209891546, 205584619 + ], + "duckdb:parquet": [ + 160233436, 156207522, 160111090, 145561039, 158546132 + ] + } + }, + { + "name": "Q19", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoxOX0", + "display_name": "tpch sf=10 Q19 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1291069493, 1290659659, 1285501663, 1277919355, 1277806242 + ], + "datafusion:parquet": [ + 108524663, 109301510, 109216745, 108063994, 108285967 + ], + "datafusion:vortex-file-compressed": [ + 28141283, 28254835, 27634644, 28033767, 28559250 + ], + "duckdb:parquet": [98397556, 99043844, 97268315, 97841079, 98916638] + } + }, + { + "name": "Q20", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyMH0", + "display_name": "tpch sf=10 Q20 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 503763831, 504950036, 505145380, 508262948, 504450435 + ], + "datafusion:parquet": [ + 185529402, 185329106, 185575719, 186207395, 186134779 + ], + "datafusion:vortex-file-compressed": [ + 60140392, 61384303, 60699908, 60466586, 60820058 + ], + "duckdb:parquet": [ + 147049224, 146755400, 147628594, 146875965, 147543765 + ] + } + }, + { + "name": "Q21", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyMX0", + "display_name": "tpch sf=10 Q21 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 1832732880, 1833507460, 1835857805, 1839811977, 1834144425 + ], + "datafusion:parquet": [ + 242714876, 242244157, 245362052, 245593952, 244867534 + ], + "datafusion:vortex-file-compressed": [ + 179957786, 183142490, 182840804, 181406322, 183253187 + ], + "duckdb:parquet": [ + 256489495, 253369561, 253195089, 255978191, 257690252 + ] + } + }, + { + "name": "Q22", + "slug": "qm.eyJrIjoiUXVlcnlNZWFzdXJlbWVudCIsImRhdGFzZXQiOiJ0cGNoIiwiZGF0YXNldF92YXJpYW50IjpudWxsLCJzY2FsZV9mYWN0b3IiOiIxMCIsInN0b3JhZ2UiOiJudm1lIiwicXVlcnlfaWR4IjoyMn0", + "display_name": "tpch sf=10 Q22 [nvme]", + "unit_kind": "time_ns", + "series": { + "datafusion:lance": [ + 224384137, 224452763, 223847181, 225566381, 223264068 + ], + "datafusion:parquet": [ + 149828009, 149997964, 149129297, 149601609, 149767211 + ], + "datafusion:vortex-file-compressed": [ + 25806272, 25906261, 25316392, 25608014, 25882764 + ], + "duckdb:parquet": [ + 168308776, 168238576, 168707850, 168478030, 168086527 + ] + } + } + ] +} diff --git a/tests/unit/llms.test.ts b/tests/unit/llms.test.ts index 21eb714..a1d8565 100644 --- a/tests/unit/llms.test.ts +++ b/tests/unit/llms.test.ts @@ -18,6 +18,7 @@ const fixture = (name: string) => ) as Record; const PANELS = panelsFrom(fixture("bench-groups.json").groups, { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }); const READ = new Date("2026-11-02T12:00:00Z"); @@ -29,13 +30,17 @@ describe("resultsSentences", () => { it("says what each row measures, its figures, and the verdict", () => { assert.ok(PANELS); const sentences = resultsSentences(PANELS); - assert.equal(sentences.length, 6); + assert.equal(sentences.length, 7); for (const s of sentences) assert.match(s, /Vortex against Parquet: .+\.$/); assert.match(sentences[0] ?? "", /^Scan, how fast each format scans/); - assert.match(sentences[3] ?? "", /^GPU, cuDF, query time on a GPU in cuDF/); - // Lance cannot run in cuDF, and the sentence says why, not a figure. assert.match( sentences[3] ?? "", + /^TPC-H, the geometric mean time of the 22/ + ); + assert.match(sentences[4] ?? "", /^GPU, cuDF, query time on a GPU in cuDF/); + // Lance cannot run in cuDF, and the sentence says why, not a figure. + assert.match( + sentences[4] ?? "", /Lance not available, no cuDF integration/ ); }); diff --git a/tests/unit/results-table.test.tsx b/tests/unit/results-table.test.tsx index 9645af4..7b7e8ea 100644 --- a/tests/unit/results-table.test.tsx +++ b/tests/unit/results-table.test.tsx @@ -13,6 +13,7 @@ const fixture = (name: string) => ) as Record; const PANELS = panelsFrom(fixture("bench-groups.json").groups, { Clickbench: fixture("bench-group-clickbench.json"), + "TPC-H (NVMe) (SF=10)": fixture("bench-group-tpch-sf10-nvme.json"), "Random Access (NVMe)": fixture("bench-group-random-access-nvme.json") }); const READ = new Date("2026-11-02T12:00:00Z"); @@ -41,6 +42,7 @@ describe("ResultsTable", () => { "Scan", "Write", "ClickBench", + "TPC-H", "GPU, cuDF", "Random access", "File size" @@ -49,7 +51,7 @@ describe("ResultsTable", () => { html.includes(`${name}`), name ); - assert.equal(html.match(/