Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 22 additions & 41 deletions src/lib/bench-summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,7 @@ export type Panel = {
source?: { note: string };
};

/**
* The results table's panels, the panels /faq adds to them
* (`morePanelsFrom`), and when bench answered.
*/
export type BenchSummary = { panels: Panel[]; more: Panel[]; readAt: Date };
export type BenchSummary = { panels: Panel[]; readAt: Date };

// The systems the table can compare, by bench's format name, in the order the
// rows are listed (the table sorts them best first). Each panel chooses the
Expand Down Expand Up @@ -125,9 +121,10 @@ type Source = {
};
};

// The six panels, in the table's order: five from bench and, beside
// ClickBench, the GPU result bench does not publish yet, kept by hand in
// gpu-result.ts with its source.
// The seven panels, in the table's order: six from bench and, after the two
// query benchmarks, the GPU result bench does not publish yet, kept by hand
// in gpu-result.ts with its source. ClickBench and TPC-H run on DataFusion,
// the one engine that runs all three formats.
const SOURCES: (Source | Panel)[] = [
{
title: "Scan throughput",
Expand Down Expand Up @@ -168,6 +165,19 @@ const SOURCES: (Source | Panel)[] = [
systems: WITH_LANCE,
times: { series: (f) => `datafusion:${f}`, bucket: (chart) => chart }
},
{
title: "TPC-H SF10 on DataFusion",
short: "TPC-H",
unit: "ms",
kind: "speed",
better: "lower",
group: "TPC-H (NVMe) (SF=10)",
rankings: (s) => s.rankings,
name: (f) => `datafusion:${f}`,
score: "score",
systems: WITH_LANCE,
times: { series: (f) => `datafusion:${f}`, bucket: (chart) => chart }
},
GPU_PANEL,
{
title: "Random access, NVMe, hot",
Expand Down Expand Up @@ -319,24 +329,6 @@ function timeRows(
});
}

// The panels /faq adds to the table's: TPC-H at scale factor 10 on
// DataFusion, the one engine that runs all three formats. A panel shows
// bench's scores against the fastest, so it needs no charts.
const MORE_SOURCES: Source[] = [
{
title: "TPC-H SF10 on DataFusion",
short: "TPC-H",
unit: "× fastest",
kind: "speed",
better: "lower",
group: "TPC-H (NVMe) (SF=10)",
rankings: (s) => s.rankings,
name: (f) => `datafusion:${f}`,
score: "score",
systems: WITH_LANCE
}
];

/**
* One bench panel from its group in bench's `/api/groups` payload and, for
* a time panel, the group's charts. A shown system bench did not report
Expand Down Expand Up @@ -397,10 +389,10 @@ function panelFor(
}

/**
* The results table's six panels: five from bench (its `/api/groups`
* The results table's seven panels: six from bench (its `/api/groups`
* payload's `groups` array and, for a time panel, `charts`, its group's
* `/api/group` payload by group name) and the hand-kept GPU panel. A bench
* panel that `panelFor` cannot build gives `null`: the table shows six
* panel that `panelFor` cannot build gives `null`: the table shows seven
* panels or none.
*/
export function panelsFrom(
Expand All @@ -421,16 +413,6 @@ export function panelsFrom(
return panels;
}

/**
* The panels /faq adds to the table's (`MORE_SOURCES`), from the same
* `/api/groups` payload. One that bench does not give is left out, so the
* table's panels never depend on these.
*/
export function morePanelsFrom(groups: unknown): Panel[] {
if (!Array.isArray(groups)) return [];
return MORE_SOURCES.flatMap((source) => panelFor(source, groups, {}) ?? []);
}

/**
* The homepage's benchmark figures, read from bench.vortex.dev at build time
* and again when the page rebuilds, at most once a day (ISR). A figure is never typed into the page.
Expand All @@ -439,7 +421,7 @@ export function morePanelsFrom(groups: unknown): Panel[] {
* bench answered the first read (its Date header), so a cached answer keeps
* its own date. A read that fails or gets a 5xx or 429 is tried again, up to
* three times. Any failure after that (a timeout, an error status, a payload
* without all six panels) gives `null`, with a warning that says why, and
* without all seven panels) gives `null`, with a warning that says why, and
* the page then leaves the table out (but see `summaryForPage`). A time
* panel whose charts are missing from the window shows bench's scores.
* `fetchImpl` and `io` are for tests.
Expand Down Expand Up @@ -514,11 +496,10 @@ export async function getBenchSummary(
)
);
const panels = panelsFrom(groups, charts);
if (!panels) return fail("bench's groups do not give all six panels");
if (!panels) return fail("bench's groups do not give all seven panels");
const date = new Date(answer.date ?? "");
return {
panels,
more: morePanelsFrom(groups),
readAt: Number.isNaN(date.getTime()) ? new Date() : date
};
} catch (e) {
Expand Down
21 changes: 2 additions & 19 deletions src/lib/faq.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,6 @@ export type Faq = {
// The table formats in the Ecosystem band, for the answer on Lance.
const TABLE_FORMATS = ["Apache Iceberg", "Apache Hudi", "Apache Paimon"];

/**
* The panels the answers compare: the table's, with the panels /faq adds
* (`more`) after ClickBench, so the query benchmarks sit together.
*/
function comparedPanels(
panels: readonly Panel[],
more: readonly Panel[]
): Panel[] {
const at = panels.findIndex((p) => p.title === "ClickBench on DataFusion");
const cut = at === -1 ? panels.length : at + 1;
return [...panels.slice(0, cut), ...more, ...panels.slice(cut)];
}

/**
* Vortex against Lance in a panel, in the words the table uses against
* Parquet ("14× faster", "48% smaller"), or "Lance not measured". `null`
Expand Down Expand Up @@ -100,15 +87,11 @@ export function faqs({
sparkVersion,
ecosystem
}: {
bench: {
panels: readonly Panel[];
more?: readonly Panel[];
readAt: Date;
} | null;
bench: { panels: readonly Panel[]; readAt: Date } | null;
sparkVersion: string | null;
ecosystem: readonly EcosystemEntry[];
}): Faq[] {
const compared = bench ? comparedPanels(bench.panels, bench.more ?? []) : [];
const compared = bench?.panels ?? [];
const systems = resultsSystems(compared);
const verdict = (panel: Panel) => resultsRow(panel, systems).verdict;
const hasGpu = compared.some((p) => p.title === GPU_PANEL.title);
Expand Down
4 changes: 4 additions & 0 deletions src/lib/llms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ function about(panel: Panel): string {
return panel.unit === "ms"
? "the geometric mean time of the ClickBench queries on DataFusion, reading files on local NVMe, so lower is better"
: "the ClickBench queries on DataFusion, reading files on local NVMe: the geometric mean of each query's time divided by the fastest format's, so lower is better";
case "TPC-H SF10 on DataFusion":
return panel.unit === "ms"
? "the geometric mean time of the 22 TPC-H queries at scale factor 10 on DataFusion, reading files on local NVMe, so lower is better"
: "the TPC-H queries at scale factor 10 on DataFusion, reading files on local NVMe: the geometric mean of each query's time divided by the fastest format's, so lower is better";
case "Random access, NVMe, hot":
return panel.unit === "ms"
? "the geometric mean time to take rows from each of the random access benchmark's datasets on local NVMe, reusing an open file, so lower is better"
Expand Down
7 changes: 4 additions & 3 deletions tests/smoke.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ test.describe("home", () => {
});

test.describe("home bands", () => {
test("the results table shows six benchmarks with a source link, or is absent", async ({
test("the results table shows seven benchmarks with a source link, or is absent", async ({
page
}) => {
await page.goto("/");
Expand All @@ -194,8 +194,8 @@ test.describe("home bands", () => {
return;
}
await expect(table).toBeVisible();
await expect(table.locator("tbody tr")).toHaveCount(6);
await expect(table.locator("tbody td.is-ours")).toHaveCount(6);
await expect(table.locator("tbody tr")).toHaveCount(7);
await expect(table.locator("tbody td.is-ours")).toHaveCount(7);
await expect(table).toContainText(/read \d+ \w{3} \d{4}/);
const source = table.locator('a[href="https://bench.vortex.dev"]');
await expect(source).toBeVisible();
Expand Down Expand Up @@ -226,6 +226,7 @@ test.describe("home bands", () => {
"Scan",
"Write",
"ClickBench",
"TPC-H",
"GPU, cuDF",
"Random access",
"File size"
Expand Down
Loading
Loading