Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 62 additions & 8 deletions src/bench/components/SummaryCard.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,26 @@ describe('SummaryCard', () => {
expect(html.match(/#1/g)).toHaveLength(2);
});

it('shows a dash for a size total the formats cannot share', () => {
const html = render({
type: 'compressionSize',
title: 'Compression Size Summary',
rankings: [
{
name: 'parquet',
minRatio: 1,
ratio: 1,
maxRatio: 1,
compressionRatio: 4,
totalBytes: null,
},
],
explanation: 'e',
});
expect(html).toContain('class="score-runtime compression-size-value">—</span>');
expect(html).toContain('class="compression-size-heading compression-size-total-unit"></span>');
});

it('renders nothing for a compression card with no rankings', () => {
expect(render({ type: 'compression', title: 't', rankings: [], explanation: 'e' })).toBe('');
});
Expand All @@ -155,20 +175,23 @@ describe('SummaryCard', () => {
ratio: 0.45,
maxRatio: 1.55,
compressionRatio: 8.25,
totalBytes: 1_603_980_000,
},
{
name: 'parquet',
minRatio: 1,
ratio: 1,
maxRatio: 1,
compressionRatio: 4.5,
totalBytes: 1_428_150_000,
},
{
name: 'lance',
minRatio: 1.1,
ratio: 1.2,
maxRatio: 1.3,
compressionRatio: null,
totalBytes: 2_500_000,
},
],
explanation: 'higher is better',
Expand All @@ -177,25 +200,42 @@ describe('SummaryCard', () => {
expect(html).toContain('class="compression-size-scores"');
expect(html).toContain('>vs Arrow<');
expect(html).toContain('>vs Parquet<');
expect(html).not.toContain('GeoMean');
expect(html.match(/>Mean</g)).toHaveLength(1);
expect(html.match(/>GeoMean</g)).toHaveLength(1);
expect(html).not.toContain('>Mean<');
expect(html.match(/>Total</g)).toHaveLength(1);
expect(html).toContain('>Min<');
expect(html).toContain('>Max<');
expect(html).not.toMatch(/⬇️|📊|⬆️/);
expect(html).toContain('class="score-value compression-size-value">8.25x</span>');
expect(html).toContain('class="score-runtime compression-size-value">0.55x</span>');
expect(html).toContain(
'class="score-runtime compression-size-value compression-size-range">0.55x</span>',
);
expect(html).toContain('class="score-value compression-size-value">0.45x</span>');
expect(html).toContain('class="score-runtime compression-size-value">1.55x</span>');
expect(html).toContain(
'class="score-runtime compression-size-value compression-size-range">1.55x</span>',
);
// Phones hide Min and Max, never the geomean.
expect(html.match(/compression-size-range/g)).toHaveLength(2 + 2 * 3);
expect(html.indexOf('8.25x')).toBeLessThan(html.indexOf('0.55x'));
// Every total is in one unit, which the header names once.
expect(html).toContain(
'class="compression-size-heading compression-size-total-unit">MB</span>',
);
const total = (value: string) =>
`class="score-runtime compression-size-value"><span class="vx-sr-only">Total: </span>${value}<span class="vx-sr-only"> MB</span></span>`;
expect(html).toContain(total('1,604.0'));
expect(html).toContain(total('1,428.2'));
expect(html).toContain(total('2.5'));
// Total sits between the format's name and its ratio to Arrow.
expect(html.indexOf('1,604.0')).toBeLessThan(html.indexOf('8.25x'));
expect(html).toContain('parquet');
expect(html).toContain('lance');
expect(html).toContain('class="score-rank">#3</span>');
expect(html).toContain('class="score-value compression-size-value">—</span>');
expect(html).toContain(
'class="vx-sr-only">vortex: minimum 0.55x, mean 0.45x, maximum 1.55x</span>',
'class="vx-sr-only">vortex: minimum 0.55x, geometric mean 0.45x, maximum 1.55x</span>',
);
expect(html).toContain('aria-hidden="true"');
expect(html).not.toContain('GB');
});

it('renders a queryBenchmark card with scores and total runtimes', () => {
Expand Down Expand Up @@ -264,8 +304,22 @@ describe('SummaryCard', () => {
type: 'compressionSize',
title: 'Compression Size Summary',
rankings: [
{ name: 'vortex-file-compressed', minRatio: 1, ratio: 1, maxRatio: 1, compressionRatio: 8 },
{ name: 'lance', minRatio: 1, ratio: 1, maxRatio: 1, compressionRatio: 2 },
{
name: 'vortex-file-compressed',
minRatio: 1,
ratio: 1,
maxRatio: 1,
compressionRatio: 8,
totalBytes: 100,
},
{
name: 'lance',
minRatio: 1,
ratio: 1,
maxRatio: 1,
compressionRatio: 2,
totalBytes: 200,
},
],
explanation: 'e',
});
Expand Down
46 changes: 39 additions & 7 deletions src/bench/components/SummaryCard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

import { displayFormat, displaySeriesLabel } from '@/bench/lib/chart-format';
import { formatCompressionSizeRatio, formatThroughput, formatTimeNs } from '@/bench/lib/format';
import {
byteUnit,
formatBytesIn,
formatCompressionSizeRatio,
formatThroughput,
formatTimeNs,
} from '@/bench/lib/format';
import type { SeriesRanking, Summary } from '@/bench/lib/summary';

/**
Expand Down Expand Up @@ -119,11 +125,20 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
if (summary.rankings.length === 0) {
return null;
}
// One unit for every total, named once in the header, so the rows compare at a glance.
const totals = summary.rankings.flatMap((item) =>
item.totalBytes === null ? [] : [item.totalBytes],
);
const totalUnit = totals.length === 0 ? null : byteUnit(Math.max(...totals));
return (
<section className="benchmark-scores-summary" aria-label={summary.title}>
<div className="compression-size-scores">
<div className="compression-size-panel compression-size-arrow-panel">
<div className="compression-size-arrow-header">
<span className="compression-size-group compression-size-total-group">Total</span>
<span className="compression-size-heading compression-size-total-unit">
{totalUnit}
</span>
<span className="compression-size-group">vs Arrow</span>
</div>
<div className="compression-size-arrow-list">
Expand All @@ -138,6 +153,17 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
<span className="score-series" title={label}>
{label}
</span>
<span className="score-runtime compression-size-value">
{item.totalBytes === null || totalUnit === null ? (
'—'
) : (
<>
<span className="vx-sr-only">Total: </span>
{formatBytesIn(item.totalBytes, totalUnit)}
<span className="vx-sr-only"> {totalUnit}</span>
</>
)}
</span>
<span className="score-value compression-size-value">
{item.compressionRatio === null
? '—'
Expand All @@ -151,9 +177,9 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
<div className="compression-size-panel compression-size-parquet-panel">
<div className="compression-size-parquet-header">
<span className="compression-size-group">vs {summary.baseline ?? 'Parquet'}</span>
<span className="compression-size-heading">Min</span>
<span className="compression-size-heading">Mean</span>
<span className="compression-size-heading">Max</span>
<span className="compression-size-heading compression-size-range">Min</span>
<span className="compression-size-heading">GeoMean</span>
<span className="compression-size-heading compression-size-range">Max</span>
</div>
<div className="compression-size-parquet-list">
{summary.rankings.map((item) => {
Expand All @@ -166,15 +192,21 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
key={item.name}
>
<span className="vx-sr-only">
{`${displayFormat(item.name)}: minimum ${minRatio}, mean ${meanRatio}, maximum ${maxRatio}`}
{`${displayFormat(item.name)}: minimum ${minRatio}, geometric mean ${meanRatio}, maximum ${maxRatio}`}
</span>
<span aria-hidden="true" className="score-runtime compression-size-value">
<span
aria-hidden="true"
className="score-runtime compression-size-value compression-size-range"
>
{minRatio}
</span>
<span aria-hidden="true" className="score-value compression-size-value">
{meanRatio}
</span>
<span aria-hidden="true" className="score-runtime compression-size-value">
<span
aria-hidden="true"
className="score-runtime compression-size-value compression-size-range"
>
{maxRatio}
</span>
</div>
Expand Down
20 changes: 20 additions & 0 deletions src/bench/lib/compression-suite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,26 @@ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [
'sparse-wide-%',
];

/**
* `LIKE` patterns for the datasets the compression size summary also leaves out, on top of
* `SUMMARY_EXCLUDED_DATASET_PATTERNS`. They still chart in the group, and the compression time
* summary keeps them.
*
* - `tpc-h l_comment chunked` writes the same column as `tpc-h l_comment canonical`, so their
* files are the same size. Only the input to the encoder differs, which matters for time but
* not for size. The size summary counts the column once, as `canonical`.
* - `statpopgen`, which joined `compress-bench` on 2026-11-06, is left out of the size summary.
* The pattern also matches the older `statpopgen/100000` rows, which have no Parquet size and
* so were never in the summary.
*
* Both names are exact. `_` is a `LIKE` wildcard, but no other dataset name matches.
*/
export const SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [
...SUMMARY_EXCLUDED_DATASET_PATTERNS,
'tpc-h l_comment chunked',
'statpopgen',
];

export function compressionSuite(dataset: string): CompressionSuite {
return dataset.startsWith(STRING_DATASET_PREFIX) ? 'strings' : 'compress';
}
4 changes: 2 additions & 2 deletions src/bench/lib/data-cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,9 @@ describe('data-cache wrappers', () => {
expect(BENCH_DATA_TAG).toBe('bench-data');
expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([
'data-cache:group-charts:v11:n100',
'data-cache:group-charts:v12:n100',
]);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v11']);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v12']);
for (const call of cacheCalls) {
expect(call.options.tags).toEqual([BENCH_DATA_TAG]);
expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS);
Expand Down
2 changes: 1 addition & 1 deletion src/bench/lib/data-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_
* code can serve payloads the old code computed until the next ingest.
* It is independent of the producer-facing benchmark schema version.
*/
const GROUP_PAYLOAD_CACHE_VERSION = 'v11';
const GROUP_PAYLOAD_CACHE_VERSION = 'v12';

// The default last-100 group bundle, keyed by group slug. The slug is the cache
// key (an `unstable_cache` argument), so one wrapper covers every group. A
Expand Down
24 changes: 23 additions & 1 deletion src/bench/lib/format.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import { describe, expect, it } from 'vitest';

import { formatTimeNs } from './format';
import { byteUnit, formatBytes, formatBytesIn, formatTimeNs } from './format';

describe('formatTimeNs', () => {
it('renders sub-microsecond values as whole nanoseconds', () => {
Expand Down Expand Up @@ -40,3 +40,25 @@ describe('formatTimeNs', () => {
expect(formatTimeNs(-2_000_000)).toBe('-2.00 ms');
});
});

describe('formatBytes', () => {
it('keeps sizes from 1 MB up in MB, with a thousands separator', () => {
expect(formatBytes(1_000_000)).toBe('1.0 MB');
expect(formatBytes(1_428_150_000)).toBe('1,428.2 MB');
expect(formatBytes(20_000_000_000)).toBe('20,000.0 MB');
});

it('uses KB and B below 1 MB', () => {
expect(formatBytes(999_949)).toBe('999.9 KB');
expect(formatBytes(2_500)).toBe('2.5 KB');
expect(formatBytes(999)).toBe('999 B');
});

it('gives a set of sizes the unit of its largest', () => {
const unit = byteUnit(1_428_150_000);
expect(unit).toBe('MB');
expect(formatBytesIn(2_500, unit)).toBe('0.0');
expect(formatBytesIn(1_428_150_000, unit)).toBe('1,428.2');
expect(formatBytesIn(999.6, 'B')).toBe('1000');
});
});
34 changes: 34 additions & 0 deletions src/bench/lib/format.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,40 @@ export function formatCompressionSizeRatio(value: number): string {
return `${Number(value.toPrecision(3))}x`;
}

const BYTES_FORMAT = new Intl.NumberFormat('en-US', {
minimumFractionDigits: 1,
maximumFractionDigits: 1,
});

/** A decimal byte unit. Sizes stay in MB from 1 MB up, so large totals compare at a glance. */
export type ByteUnit = 'MB' | 'KB' | 'B';

/** The unit `formatBytes` gives `bytes`. Pass the largest of a set to give the set one unit. */
export function byteUnit(bytes: number): ByteUnit {
if (bytes >= 1_000_000) {
return 'MB';
}
return bytes >= 1_000 ? 'KB' : 'B';
}

/** A byte count in `unit` without the unit: one decimal place for MB and KB, as `1,428.2`. */
export function formatBytesIn(bytes: number, unit: ByteUnit): string {
switch (unit) {
case 'MB':
return BYTES_FORMAT.format(bytes / 1_000_000);
case 'KB':
return BYTES_FORMAT.format(bytes / 1_000);
case 'B':
return String(Math.round(bytes));
}
}

/** A byte count in decimal units, as `1,428.2 MB`. */
export function formatBytes(bytes: number): string {
const unit = byteUnit(bytes);
return `${formatBytesIn(bytes, unit)} ${unit}`;
}

/** Aggregate throughput as the summary card shows it: MB/s for writes, GB/s for scans. */
export function formatThroughput(valueGbS: number, operation: 'encode' | 'decode'): string {
return operation === 'encode'
Expand Down
4 changes: 3 additions & 1 deletion src/bench/lib/headline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,9 @@ describe('groupHeadline', () => {
type: 'compressionSize',
title: 's',
explanation: 'e',
rankings: [{ name: VX, minRatio: ratio, ratio, maxRatio: ratio, compressionRatio: 5 }],
rankings: [
{ name: VX, minRatio: ratio, ratio, maxRatio: ratio, compressionRatio: 5, totalBytes: 1 },
],
});
expect(h && headlineText(h)).toBe(expected);
expect(h?.parts[0]?.tone).toBe(tone);
Expand Down
5 changes: 5 additions & 0 deletions src/bench/lib/methodology.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,11 @@ export const METHODOLOGY: readonly MethodologySection[] = [
'rank well by skipping work. Each summary says how many a series ran.',
`Write and scan throughput scores are ${SCORES.throughput}`,
`File size scores are ${SCORES.size}`,
'Total is the sum of a format’s file sizes over the datasets that every format has.',
'The compression summaries leave out the wide table and sparse-wide datasets, which ' +
'have many variants each. The file size summary also leaves out statpopgen, and counts ' +
'the TPC-H l_comment column once, because its canonical and chunked files are the ' +
'same size. These datasets still have charts.',
'The string encoding benchmark compares its encoders with FSST instead of Parquet, so ' +
'its throughput and size scores divide by FSST’s.',
'Each summary uses each series’ newest result on develop.',
Expand Down
11 changes: 9 additions & 2 deletions src/bench/lib/site-files.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,19 @@ describe('summaryLines', () => {
title: 't',
explanation: 'e',
rankings: [
{ name: VX, minRatio: 0.677, ratio: 1.137, maxRatio: 1.549, compressionRatio: 5.744 },
{
name: VX,
minRatio: 0.677,
ratio: 1.137,
maxRatio: 1.549,
compressionRatio: 5.744,
totalBytes: 1_603_980_000,
},
],
}).at(-1),
).toBe(
'1. Vortex: 1.14x the size of Parquet (from 0.677x to 1.55x across datasets), ' +
'5.74x smaller than Arrow in memory',
'5.74x smaller than Arrow in memory, 1,604.0 MB in total',
);
});
});
Expand Down
Loading
Loading