From 8b232dd6162df6deaddde264a507873a6cfafe13 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 10 Nov 2026 17:12:45 -0500 Subject: [PATCH 1/2] bench: count l_comment once and drop statpopgen from the size summary, add Total The Compression Size summary left out only the wide table and sparse-wide datasets. It now also leaves out statpopgen, and it counts the TPC-H l_comment column once: the canonical and chunked files are the same size. The compression time summary keeps both datasets. Each format now shows Total, the sum of its file sizes over the same datasets. The header names the unit once. The vs Parquet column "Mean" is now "GeoMean". Phones hide Min and Max to make room for Total; the screen-reader text still gives them. The group payload cache moves to v12, because the summary shape changed. Signed-off-by: Will Manning --- src/bench/components/SummaryCard.test.tsx | 50 +++++++++++++++++++---- src/bench/components/SummaryCard.tsx | 37 +++++++++++++---- src/bench/lib/compression-suite.ts | 20 +++++++++ src/bench/lib/data-cache.test.ts | 4 +- src/bench/lib/data-cache.ts | 2 +- src/bench/lib/format.test.ts | 24 ++++++++++- src/bench/lib/format.ts | 34 +++++++++++++++ src/bench/lib/headline.test.ts | 4 +- src/bench/lib/methodology.ts | 5 +++ src/bench/lib/site-files.test.ts | 11 ++++- src/bench/lib/site-files.ts | 10 ++++- src/bench/lib/summary.test.ts | 8 +++- src/bench/lib/summary.ts | 29 +++++++++---- src/bench/styles/globals.css | 31 +++++++++----- 14 files changed, 226 insertions(+), 43 deletions(-) diff --git a/src/bench/components/SummaryCard.test.tsx b/src/bench/components/SummaryCard.test.tsx index 96e49611..ef27a5c9 100644 --- a/src/bench/components/SummaryCard.test.tsx +++ b/src/bench/components/SummaryCard.test.tsx @@ -155,6 +155,7 @@ describe('SummaryCard', () => { ratio: 0.45, maxRatio: 1.55, compressionRatio: 8.25, + totalBytes: 1_603_980_000, }, { name: 'parquet', @@ -162,6 +163,7 @@ describe('SummaryCard', () => { ratio: 1, maxRatio: 1, compressionRatio: 4.5, + totalBytes: 1_428_150_000, }, { name: 'lance', @@ -169,6 +171,7 @@ describe('SummaryCard', () => { ratio: 1.2, maxRatio: 1.3, compressionRatio: null, + totalBytes: 2_500_000, }, ], explanation: 'higher is better', @@ -177,25 +180,42 @@ describe('SummaryCard', () => { expect(html).toContain('class="compression-size-scores"'); expect(html).toContain('>vs Arrow<'); expect(html).toContain('>vs Parquet<'); - expect(html).not.toContain('GeoMean'); - expect(html.match(/>MeanGeoMeanMean<'); + expect(html.match(/>TotalMin<'); expect(html).toContain('>Max<'); expect(html).not.toMatch(/⬇️|📊|⬆️/); expect(html).toContain('class="score-value compression-size-value">8.25x'); - expect(html).toContain('class="score-runtime compression-size-value">0.55x'); + expect(html).toContain( + 'class="score-runtime compression-size-value compression-size-range">0.55x', + ); expect(html).toContain('class="score-value compression-size-value">0.45x'); - expect(html).toContain('class="score-runtime compression-size-value">1.55x'); + expect(html).toContain( + 'class="score-runtime compression-size-value compression-size-range">1.55x', + ); + // Phones hide Min and Max, never the geomean. + expect(html.match(/compression-size-range/g)).toHaveLength(2 + 2 * 3); expect(html.indexOf('8.25x')).toBeLessThan(html.indexOf('0.55x')); + // Every total is in one unit, which the header names once. + expect(html).toContain( + 'class="compression-size-heading compression-size-total-unit">MB', + ); + const total = (value: string) => + `class="score-runtime compression-size-value">Total: ${value} MB`; + expect(html).toContain(total('1,604.0')); + expect(html).toContain(total('1,428.2')); + expect(html).toContain(total('2.5')); + // Total sits between the format's name and its ratio to Arrow. + expect(html.indexOf('1,604.0')).toBeLessThan(html.indexOf('8.25x')); expect(html).toContain('parquet'); expect(html).toContain('lance'); expect(html).toContain('class="score-rank">#3'); expect(html).toContain('class="score-value compression-size-value">—'); expect(html).toContain( - 'class="vx-sr-only">vortex: minimum 0.55x, mean 0.45x, maximum 1.55x', + 'class="vx-sr-only">vortex: minimum 0.55x, geometric mean 0.45x, maximum 1.55x', ); expect(html).toContain('aria-hidden="true"'); - expect(html).not.toContain('GB'); }); it('renders a queryBenchmark card with scores and total runtimes', () => { @@ -264,8 +284,22 @@ describe('SummaryCard', () => { type: 'compressionSize', title: 'Compression Size Summary', rankings: [ - { name: 'vortex-file-compressed', minRatio: 1, ratio: 1, maxRatio: 1, compressionRatio: 8 }, - { name: 'lance', minRatio: 1, ratio: 1, maxRatio: 1, compressionRatio: 2 }, + { + name: 'vortex-file-compressed', + minRatio: 1, + ratio: 1, + maxRatio: 1, + compressionRatio: 8, + totalBytes: 100, + }, + { + name: 'lance', + minRatio: 1, + ratio: 1, + maxRatio: 1, + compressionRatio: 2, + totalBytes: 200, + }, ], explanation: 'e', }); diff --git a/src/bench/components/SummaryCard.tsx b/src/bench/components/SummaryCard.tsx index ee1545bb..c0e49ba4 100644 --- a/src/bench/components/SummaryCard.tsx +++ b/src/bench/components/SummaryCard.tsx @@ -2,7 +2,13 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors import { displayFormat, displaySeriesLabel } from '@/bench/lib/chart-format'; -import { formatCompressionSizeRatio, formatThroughput, formatTimeNs } from '@/bench/lib/format'; +import { + byteUnit, + formatBytesIn, + formatCompressionSizeRatio, + formatThroughput, + formatTimeNs, +} from '@/bench/lib/format'; import type { SeriesRanking, Summary } from '@/bench/lib/summary'; /** @@ -119,11 +125,17 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) { if (summary.rankings.length === 0) { return null; } + // One unit for every total, named once in the header, so the rows compare at a glance. + const totalUnit = byteUnit(Math.max(...summary.rankings.map((item) => item.totalBytes))); return (
+ Total + + {totalUnit} + vs Arrow
@@ -138,6 +150,11 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) { {label} + + Total: + {formatBytesIn(item.totalBytes, totalUnit)} + {totalUnit} + {item.compressionRatio === null ? '—' @@ -151,9 +168,9 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
vs {summary.baseline ?? 'Parquet'} - Min - Mean - Max + Min + GeoMean + Max
{summary.rankings.map((item) => { @@ -166,15 +183,21 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) { key={item.name} > - {`${displayFormat(item.name)}: minimum ${minRatio}, mean ${meanRatio}, maximum ${maxRatio}`} + {`${displayFormat(item.name)}: minimum ${minRatio}, geometric mean ${meanRatio}, maximum ${maxRatio}`} -
diff --git a/src/bench/lib/compression-suite.ts b/src/bench/lib/compression-suite.ts index 6cc464b0..e9919e4d 100644 --- a/src/bench/lib/compression-suite.ts +++ b/src/bench/lib/compression-suite.ts @@ -26,6 +26,26 @@ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ 'sparse-wide-%', ]; +/** + * `LIKE` patterns for the datasets the compression size summary also leaves out, on top of + * `SUMMARY_EXCLUDED_DATASET_PATTERNS`. They still chart in the group, and the compression time + * summary keeps them. + * + * - `tpc-h l_comment chunked` writes the same column as `tpc-h l_comment canonical`, so their + * files are the same size. Only the input to the encoder differs, which matters for time but + * not for size. The size summary counts the column once, as `canonical`. + * - `statpopgen`, which joined `compress-bench` on 2026-11-06, is left out of the size summary. + * The pattern also matches the older `statpopgen/100000` rows, which have no Parquet size and + * so were never in the summary. + * + * Both names are exact. `_` is a `LIKE` wildcard, but no other dataset name matches. + */ +export const SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ + ...SUMMARY_EXCLUDED_DATASET_PATTERNS, + 'tpc-h l_comment chunked', + 'statpopgen', +]; + export function compressionSuite(dataset: string): CompressionSuite { return dataset.startsWith(STRING_DATASET_PREFIX) ? 'strings' : 'compress'; } diff --git a/src/bench/lib/data-cache.test.ts b/src/bench/lib/data-cache.test.ts index 3a01b286..c77e9b75 100644 --- a/src/bench/lib/data-cache.test.ts +++ b/src/bench/lib/data-cache.test.ts @@ -69,9 +69,9 @@ describe('data-cache wrappers', () => { expect(BENCH_DATA_TAG).toBe('bench-data'); expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400); expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([ - 'data-cache:group-charts:v11:n100', + 'data-cache:group-charts:v12:n100', ]); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v11']); + expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v12']); for (const call of cacheCalls) { expect(call.options.tags).toEqual([BENCH_DATA_TAG]); expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS); diff --git a/src/bench/lib/data-cache.ts b/src/bench/lib/data-cache.ts index e0a2c5eb..f77ea34d 100644 --- a/src/bench/lib/data-cache.ts +++ b/src/bench/lib/data-cache.ts @@ -57,7 +57,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_ * code can serve payloads the old code computed until the next ingest. * It is independent of the producer-facing benchmark schema version. */ -const GROUP_PAYLOAD_CACHE_VERSION = 'v11'; +const GROUP_PAYLOAD_CACHE_VERSION = 'v12'; // The default last-100 group bundle, keyed by group slug. The slug is the cache // key (an `unstable_cache` argument), so one wrapper covers every group. A diff --git a/src/bench/lib/format.test.ts b/src/bench/lib/format.test.ts index f619ac85..c57328f6 100644 --- a/src/bench/lib/format.test.ts +++ b/src/bench/lib/format.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest'; -import { formatTimeNs } from './format'; +import { byteUnit, formatBytes, formatBytesIn, formatTimeNs } from './format'; describe('formatTimeNs', () => { it('renders sub-microsecond values as whole nanoseconds', () => { @@ -40,3 +40,25 @@ describe('formatTimeNs', () => { expect(formatTimeNs(-2_000_000)).toBe('-2.00 ms'); }); }); + +describe('formatBytes', () => { + it('keeps sizes from 1 MB up in MB, with a thousands separator', () => { + expect(formatBytes(1_000_000)).toBe('1.0 MB'); + expect(formatBytes(1_428_150_000)).toBe('1,428.2 MB'); + expect(formatBytes(20_000_000_000)).toBe('20,000.0 MB'); + }); + + it('uses KB and B below 1 MB', () => { + expect(formatBytes(999_949)).toBe('999.9 KB'); + expect(formatBytes(2_500)).toBe('2.5 KB'); + expect(formatBytes(999)).toBe('999 B'); + }); + + it('gives a set of sizes the unit of its largest', () => { + const unit = byteUnit(1_428_150_000); + expect(unit).toBe('MB'); + expect(formatBytesIn(2_500, unit)).toBe('0.0'); + expect(formatBytesIn(1_428_150_000, unit)).toBe('1,428.2'); + expect(formatBytesIn(999.6, 'B')).toBe('1000'); + }); +}); diff --git a/src/bench/lib/format.ts b/src/bench/lib/format.ts index 93253fa7..b2ebd2de 100644 --- a/src/bench/lib/format.ts +++ b/src/bench/lib/format.ts @@ -35,6 +35,40 @@ export function formatCompressionSizeRatio(value: number): string { return `${Number(value.toPrecision(3))}x`; } +const BYTES_FORMAT = new Intl.NumberFormat('en-US', { + minimumFractionDigits: 1, + maximumFractionDigits: 1, +}); + +/** A decimal byte unit. Sizes stay in MB from 1 MB up, so large totals compare at a glance. */ +export type ByteUnit = 'MB' | 'KB' | 'B'; + +/** The unit `formatBytes` gives `bytes`. Pass the largest of a set to give the set one unit. */ +export function byteUnit(bytes: number): ByteUnit { + if (bytes >= 1_000_000) { + return 'MB'; + } + return bytes >= 1_000 ? 'KB' : 'B'; +} + +/** A byte count in `unit` without the unit: one decimal place for MB and KB, as `1,428.2`. */ +export function formatBytesIn(bytes: number, unit: ByteUnit): string { + switch (unit) { + case 'MB': + return BYTES_FORMAT.format(bytes / 1_000_000); + case 'KB': + return BYTES_FORMAT.format(bytes / 1_000); + case 'B': + return String(Math.round(bytes)); + } +} + +/** A byte count in decimal units, as `1,428.2 MB`. */ +export function formatBytes(bytes: number): string { + const unit = byteUnit(bytes); + return `${formatBytesIn(bytes, unit)} ${unit}`; +} + /** Aggregate throughput as the summary card shows it: MB/s for writes, GB/s for scans. */ export function formatThroughput(valueGbS: number, operation: 'encode' | 'decode'): string { return operation === 'encode' diff --git a/src/bench/lib/headline.test.ts b/src/bench/lib/headline.test.ts index 69f8778c..7db1ee86 100644 --- a/src/bench/lib/headline.test.ts +++ b/src/bench/lib/headline.test.ts @@ -185,7 +185,9 @@ describe('groupHeadline', () => { type: 'compressionSize', title: 's', explanation: 'e', - rankings: [{ name: VX, minRatio: ratio, ratio, maxRatio: ratio, compressionRatio: 5 }], + rankings: [ + { name: VX, minRatio: ratio, ratio, maxRatio: ratio, compressionRatio: 5, totalBytes: 1 }, + ], }); expect(h && headlineText(h)).toBe(expected); expect(h?.parts[0]?.tone).toBe(tone); diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 28f308dd..e061f9e1 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -98,6 +98,11 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'rank well by skipping work. Each summary says how many a series ran.', `Write and scan throughput scores are ${SCORES.throughput}`, `File size scores are ${SCORES.size}`, + 'Total is the sum of a format’s file sizes over the same datasets.', + 'The compression summaries leave out the wide table and sparse-wide datasets, which ' + + 'have many variants each. The file size summary also leaves out statpopgen, and counts ' + + 'the TPC-H l_comment column once, because its canonical and chunked files are the ' + + 'same size. These datasets still have charts.', 'The string encoding benchmark compares its encoders with FSST instead of Parquet, so ' + 'its throughput and size scores divide by FSST’s.', 'Each summary uses each series’ newest result on develop.', diff --git a/src/bench/lib/site-files.test.ts b/src/bench/lib/site-files.test.ts index 2cb110e3..cfb6e131 100644 --- a/src/bench/lib/site-files.test.ts +++ b/src/bench/lib/site-files.test.ts @@ -129,12 +129,19 @@ describe('summaryLines', () => { title: 't', explanation: 'e', rankings: [ - { name: VX, minRatio: 0.677, ratio: 1.137, maxRatio: 1.549, compressionRatio: 5.744 }, + { + name: VX, + minRatio: 0.677, + ratio: 1.137, + maxRatio: 1.549, + compressionRatio: 5.744, + totalBytes: 1_603_980_000, + }, ], }).at(-1), ).toBe( '1. Vortex: 1.14x the size of Parquet (from 0.677x to 1.55x across datasets), ' + - '5.74x smaller than Arrow in memory', + '5.74x smaller than Arrow in memory, 1,604.0 MB in total', ); }); }); diff --git a/src/bench/lib/site-files.ts b/src/bench/lib/site-files.ts index cccbd22f..1a79936b 100644 --- a/src/bench/lib/site-files.ts +++ b/src/bench/lib/site-files.ts @@ -16,7 +16,12 @@ import { groupAnchor, groupAnchors } from '@/bench/lib/anchor'; import { groupDocUrl } from '@/bench/lib/benchmark-docs'; import { higherIsBetter } from '@/bench/lib/chart-format'; -import { formatCompressionSizeRatio, formatThroughput, formatTimeNs } from '@/bench/lib/format'; +import { + formatBytes, + formatCompressionSizeRatio, + formatThroughput, + formatTimeNs, +} from '@/bench/lib/format'; import { groupHeadline, headlineText } from '@/bench/lib/headline'; import { METHODOLOGY, SCORES } from '@/bench/lib/methodology'; import type { ChartKind } from '@/bench/lib/families'; @@ -316,7 +321,8 @@ export function summaryLines(summary: Summary): string[] { `to ${formatCompressionSizeRatio(item.maxRatio)} across datasets)`) + (item.compressionRatio === null ? '' - : `, ${formatCompressionSizeRatio(item.compressionRatio)} smaller than Arrow in memory`), + : `, ${formatCompressionSizeRatio(item.compressionRatio)} smaller than Arrow in memory`) + + `, ${formatBytes(item.totalBytes)} in total`, ), ]; } diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index 69071d88..36e4e2b9 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -61,8 +61,11 @@ describe('compression summaries', () => { expect(byFormat.get('parquet')?.minRatio).toBeCloseTo(1, 6); expect(byFormat.get('parquet')?.maxRatio).toBeCloseTo(1, 6); expect(byFormat.get('parquet')?.compressionRatio).toBeCloseTo(4, 6); + // The total is the plain sum of each format's sizes over its datasets. + expect(byFormat.get('vortex-file-compressed')?.totalBytes).toBe(901); + expect(byFormat.get('parquet')?.totalBytes).toBe(104); expect(summary.explanation).toBe( - 'Geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', + 'Total compressed size, and geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', ); }); @@ -183,6 +186,7 @@ describe('compression summaries', () => { expect(size.rankings.map((ranking) => ranking.name)).toEqual(['onpair-12', 'fsst']); expect(size.rankings[0]?.ratio).toBeCloseTo(2 / 3, 6); expect(size.rankings[0]?.compressionRatio).toBeCloseTo(6, 6); + expect(size.rankings.map((ranking) => ranking.totalBytes)).toEqual([200, 300]); }); it('applies extensible snapshot policies to timings and sizes', async () => { @@ -205,7 +209,7 @@ describe('compression summaries', () => { 'vortex-file-compressed', 'parquet', 'arrow-ipc', - ['wide table %', 'sparse-wide-%'], + ['wide table %', 'sparse-wide-%', 'tpc-h l_comment chunked', 'statpopgen'], ]; const calls = query.mock.calls as Array<[string, unknown[]]>; expect(defined(calls[0])[1]).toEqual(timingParams); diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index 0b200262..b85c16ea 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -41,7 +41,11 @@ import { getPool } from './db'; import { randomAccessStorage, type RandomAccessStorage } from './random-access'; import { compareCodeUnits } from './families'; import type { GroupKey } from './slug'; -import { STRING_DATASET_PATTERN, SUMMARY_EXCLUDED_DATASET_PATTERNS } from './compression-suite'; +import { + SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS, + STRING_DATASET_PATTERN, + SUMMARY_EXCLUDED_DATASET_PATTERNS, +} from './compression-suite'; /** * Freshness buckets for compression-time and compression-size summaries. @@ -88,7 +92,7 @@ function compressionSizeSummaryQueryParams(): [ COMPRESSION_SNAPSHOT_ANCHOR, COMPRESSION_BASELINE, ARROW_IPC_FORMAT, - [...SUMMARY_EXCLUDED_DATASET_PATTERNS], + [...SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS], ]; } @@ -140,6 +144,8 @@ interface CompressionSizeRanking { maxRatio: number; /** Geomean ratio of Arrow memory size to the format size. */ compressionRatio: number | null; + /** Sum of this format's sizes, in bytes, over the datasets in `ratio`. */ + totalBytes: number; } /** @@ -710,7 +716,7 @@ async function collectCompressionSizeSummary(): Promise { title: 'Compression Size Summary', rankings, explanation: - 'Geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', + 'Total compressed size, and geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', }; } @@ -727,9 +733,16 @@ interface CompressionSizeSample { * order is by compression ratio once every format has one, and by baseline ratio until then. */ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): CompressionSizeRanking[] { - const grouped = new Map(); + const grouped = new Map< + string, + { sizeRatios: number[]; compressionRatios: number[]; totalBytes: number } + >(); for (const row of rows) { - const ratios = grouped.get(row.format) ?? { sizeRatios: [], compressionRatios: [] }; + const ratios = grouped.get(row.format) ?? { + sizeRatios: [], + compressionRatios: [], + totalBytes: 0, + }; if ( row.valueBytes > 0 && Number.isFinite(row.valueBytes) && @@ -737,6 +750,7 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre Number.isFinite(row.baselineBytes) ) { ratios.sizeRatios.push(row.valueBytes / row.baselineBytes); + ratios.totalBytes += row.valueBytes; } if ( row.uncompressedBytes !== null && @@ -765,6 +779,7 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre ratio, maxRatio, compressionRatio: geoMean(ratios.compressionRatios), + totalBytes: ratios.totalBytes, }); } } @@ -787,7 +802,7 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre * and compares it with Parquet from the same commit. Arrow IPC uses its newest value for each * dataset and Parquet from the same commit. Compression ratios use the newest available decoded * Arrow memory size for each dataset. The size commit does not need to match. - * The datasets in `SUMMARY_EXCLUDED_DATASET_PATTERNS` are left out. + * The datasets in `SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS` are left out. * `uncompressed_bytes` is read as a plain column, so migration 009 must be applied before a web * deploy that includes this query. A JSON field lookup would tolerate an older schema, but it * serializes whole rows and timed out in production, so it was dropped once 009 was applied there. @@ -980,7 +995,7 @@ async function collectStringEncodingSizeSummary(): Promise { title: 'String Encoding Size Summary', rankings, explanation: - 'Geometric means of file sizes versus Arrow (higher is better) and versus FSST (lower is better)', + 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus FSST (lower is better)', baseline: 'FSST', }; } diff --git a/src/bench/styles/globals.css b/src/bench/styles/globals.css index 3474321f..8a1919fc 100644 --- a/src/bench/styles/globals.css +++ b/src/bench/styles/globals.css @@ -678,7 +678,7 @@ main { .compression-size-arrow-header, .score-item.compression-size-arrow-row { display: grid; - grid-template-columns: 2.25rem minmax(0, 1fr) 7rem; + grid-template-columns: 2.25rem minmax(0, 1fr) 6rem 7rem; column-gap: var(--space-16); } @@ -703,21 +703,19 @@ main { text-align: center; } -.compression-size-arrow-header .compression-size-group, -.compression-size-arrow-header .compression-size-heading { - grid-column: 3; +.compression-size-arrow-header .compression-size-group { + grid-column: 4; + grid-row: 1; text-align: center; } -.compression-size-arrow-header .compression-size-group { - grid-row: 1; +.compression-size-arrow-header .compression-size-total-group { + grid-column: 3; } -.compression-size-arrow-header::after { - content: ''; +.compression-size-arrow-header .compression-size-total-unit { grid-column: 3; grid-row: 2; - min-height: 1em; } .compression-size-parquet-header, @@ -1597,18 +1595,31 @@ main > .chart-card:not(.chart-grid .chart-card) .chart-wrap { gap: 0; align-items: flex-end; } + .compression-size-arrow-header, .score-item.compression-size-arrow-row { - grid-template-columns: 1.75rem minmax(0, 1fr) 3.4rem; + grid-template-columns: 1.75rem minmax(0, 1fr) 3.75rem 3.4rem; column-gap: 0.35rem; } .compression-size-parquet-panel { padding-left: 0.35rem; margin-left: 0.35rem; } + /* A phone has no room for Total and the Min and Max ratios: it shows Total and the + geomean, and the screen-reader text of each row still gives Min and Max. */ + .compression-size-scores { + grid-template-columns: minmax(0, 1fr) auto; + } .compression-size-parquet-header, .compression-size-parquet-row { + grid-template-columns: auto; column-gap: 0.35rem; } + .compression-size-parquet-header .compression-size-group { + grid-column: 1; + } + .compression-size-range { + display: none; + } .toolbar-slider { width: 100%; flex: 1 1 auto; From bccce140a3b8e0487eff6da13dfedc2bdc813786 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 10 Nov 2026 17:40:14 -0500 Subject: [PATCH 2/2] bench: sum size totals over the datasets every format has Lance and Arrow IPC come from their own snapshots, so a format can lack a dataset that the others have. Its Total then summed fewer files and looked smaller. Each Total now sums only the datasets that every ranked format has, and shows a dash when the formats share none. Both size queries now return the dataset key. Signed-off-by: Will Manning --- src/bench/components/SummaryCard.test.tsx | 20 +++++ src/bench/components/SummaryCard.tsx | 17 +++- src/bench/lib/methodology.ts | 2 +- src/bench/lib/site-files.ts | 2 +- src/bench/lib/summary.test.ts | 105 ++++++++++++++++++++-- src/bench/lib/summary.ts | 33 +++++-- 6 files changed, 162 insertions(+), 17 deletions(-) diff --git a/src/bench/components/SummaryCard.test.tsx b/src/bench/components/SummaryCard.test.tsx index ef27a5c9..da04c1d1 100644 --- a/src/bench/components/SummaryCard.test.tsx +++ b/src/bench/components/SummaryCard.test.tsx @@ -140,6 +140,26 @@ describe('SummaryCard', () => { expect(html.match(/#1/g)).toHaveLength(2); }); + it('shows a dash for a size total the formats cannot share', () => { + const html = render({ + type: 'compressionSize', + title: 'Compression Size Summary', + rankings: [ + { + name: 'parquet', + minRatio: 1, + ratio: 1, + maxRatio: 1, + compressionRatio: 4, + totalBytes: null, + }, + ], + explanation: 'e', + }); + expect(html).toContain('class="score-runtime compression-size-value">—'); + expect(html).toContain('class="compression-size-heading compression-size-total-unit">'); + }); + it('renders nothing for a compression card with no rankings', () => { expect(render({ type: 'compression', title: 't', rankings: [], explanation: 'e' })).toBe(''); }); diff --git a/src/bench/components/SummaryCard.tsx b/src/bench/components/SummaryCard.tsx index c0e49ba4..0ad86a9d 100644 --- a/src/bench/components/SummaryCard.tsx +++ b/src/bench/components/SummaryCard.tsx @@ -126,7 +126,10 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) { return null; } // One unit for every total, named once in the header, so the rows compare at a glance. - const totalUnit = byteUnit(Math.max(...summary.rankings.map((item) => item.totalBytes))); + const totals = summary.rankings.flatMap((item) => + item.totalBytes === null ? [] : [item.totalBytes], + ); + const totalUnit = totals.length === 0 ? null : byteUnit(Math.max(...totals)); return (
@@ -151,9 +154,15 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) { {label} - Total: - {formatBytesIn(item.totalBytes, totalUnit)} - {totalUnit} + {item.totalBytes === null || totalUnit === null ? ( + '—' + ) : ( + <> + Total: + {formatBytesIn(item.totalBytes, totalUnit)} + {totalUnit} + + )} {item.compressionRatio === null diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index e061f9e1..86497577 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -98,7 +98,7 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'rank well by skipping work. Each summary says how many a series ran.', `Write and scan throughput scores are ${SCORES.throughput}`, `File size scores are ${SCORES.size}`, - 'Total is the sum of a format’s file sizes over the same datasets.', + 'Total is the sum of a format’s file sizes over the datasets that every format has.', 'The compression summaries leave out the wide table and sparse-wide datasets, which ' + 'have many variants each. The file size summary also leaves out statpopgen, and counts ' + 'the TPC-H l_comment column once, because its canonical and chunked files are the ' + diff --git a/src/bench/lib/site-files.ts b/src/bench/lib/site-files.ts index 1a79936b..61d70c2c 100644 --- a/src/bench/lib/site-files.ts +++ b/src/bench/lib/site-files.ts @@ -322,7 +322,7 @@ export function summaryLines(summary: Summary): string[] { (item.compressionRatio === null ? '' : `, ${formatCompressionSizeRatio(item.compressionRatio)} smaller than Arrow in memory`) + - `, ${formatBytes(item.totalBytes)} in total`, + (item.totalBytes === null ? '' : `, ${formatBytes(item.totalBytes)} in total`), ), ]; } diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index 36e4e2b9..63d2918e 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -27,18 +27,36 @@ describe('compression summaries', () => { rows: [ { format: 'vortex-file-compressed', + dataset: 'a', + datasetVariant: null, valueBytes: 1, baselineBytes: 4, uncompressedBytes: 16, }, { format: 'vortex-file-compressed', + dataset: 'b', + datasetVariant: null, valueBytes: 900, baselineBytes: 100, uncompressedBytes: 3_600, }, - { format: 'parquet', valueBytes: 4, baselineBytes: 4, uncompressedBytes: 16 }, - { format: 'parquet', valueBytes: 100, baselineBytes: 100, uncompressedBytes: null }, + { + format: 'parquet', + dataset: 'a', + datasetVariant: null, + valueBytes: 4, + baselineBytes: 4, + uncompressedBytes: 16, + }, + { + format: 'parquet', + dataset: 'b', + datasetVariant: null, + valueBytes: 100, + baselineBytes: 100, + uncompressedBytes: null, + }, ], }); @@ -61,7 +79,7 @@ describe('compression summaries', () => { expect(byFormat.get('parquet')?.minRatio).toBeCloseTo(1, 6); expect(byFormat.get('parquet')?.maxRatio).toBeCloseTo(1, 6); expect(byFormat.get('parquet')?.compressionRatio).toBeCloseTo(4, 6); - // The total is the plain sum of each format's sizes over its datasets. + // Both formats have both datasets, so each total sums both. expect(byFormat.get('vortex-file-compressed')?.totalBytes).toBe(901); expect(byFormat.get('parquet')?.totalBytes).toBe(104); expect(summary.explanation).toBe( @@ -69,6 +87,69 @@ describe('compression summaries', () => { ); }); + it('sums totals over the datasets every format has', async () => { + const row = (format: string, dataset: string, valueBytes: number, datasetVariant = null) => ({ + format, + dataset, + datasetVariant, + valueBytes, + baselineBytes: 10, + uncompressedBytes: null, + }); + query.mockResolvedValueOnce({ + rows: [ + row('vortex-file-compressed', 'a', 10), + row('vortex-file-compressed', 'b', 1_000), + row('parquet', 'a', 20), + row('parquet', 'b', 2_000), + // Lance has no size for dataset b, so no total counts b. + row('lance', 'a', 30), + ], + }); + + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + if (summary === null || summary.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + const totals = new Map(summary.rankings.map((ranking) => [ranking.name, ranking.totalBytes])); + expect(totals).toEqual( + new Map([ + ['vortex-file-compressed', 10], + ['parquet', 20], + ['lance', 30], + ]), + ); + }); + + it('gives no total when the formats share no dataset', async () => { + query.mockResolvedValueOnce({ + rows: [ + { + format: 'parquet', + dataset: 'a', + datasetVariant: null, + valueBytes: 10, + baselineBytes: 10, + uncompressedBytes: null, + }, + { + format: 'lance', + dataset: 'b', + datasetVariant: null, + valueBytes: 30, + baselineBytes: 10, + uncompressedBytes: null, + }, + ], + }); + + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + if (summary === null || summary.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + expect(summary.rankings.map((ranking) => ranking.totalBytes)).toEqual([null, null]); + }); + it('uses available Arrow memory sizes for aggregate throughput', async () => { query.mockResolvedValueOnce({ rows: [ @@ -157,8 +238,22 @@ describe('compression summaries', () => { }); query.mockResolvedValueOnce({ rows: [ - { format: 'fsst', valueBytes: 300, baselineBytes: 300, uncompressedBytes: 1_200 }, - { format: 'onpair-12', valueBytes: 200, baselineBytes: 300, uncompressedBytes: 1_200 }, + { + format: 'fsst', + dataset: 'strings/x', + datasetVariant: null, + valueBytes: 300, + baselineBytes: 300, + uncompressedBytes: 1_200, + }, + { + format: 'onpair-12', + dataset: 'strings/x', + datasetVariant: null, + valueBytes: 200, + baselineBytes: 300, + uncompressedBytes: 1_200, + }, ], }); diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index b85c16ea..d58a176b 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -144,8 +144,11 @@ interface CompressionSizeRanking { maxRatio: number; /** Geomean ratio of Arrow memory size to the format size. */ compressionRatio: number | null; - /** Sum of this format's sizes, in bytes, over the datasets in `ratio`. */ - totalBytes: number; + /** + * Sum of this format's sizes, in bytes, over the datasets that every format in the summary + * has, so the totals compare directly. `null` when the formats share no dataset. + */ + totalBytes: number | null; } /** @@ -723,6 +726,8 @@ async function collectCompressionSizeSummary(): Promise { /** One size paired with the baseline's size for the same dataset and commit. */ interface CompressionSizeSample { format: string; + dataset: string; + datasetVariant: string | null; valueBytes: number; baselineBytes: number; uncompressedBytes: number | null; @@ -735,13 +740,13 @@ interface CompressionSizeSample { function compressionSizeRankings(rows: readonly CompressionSizeSample[]): CompressionSizeRanking[] { const grouped = new Map< string, - { sizeRatios: number[]; compressionRatios: number[]; totalBytes: number } + { sizeRatios: number[]; compressionRatios: number[]; bytesByDataset: Map } >(); for (const row of rows) { const ratios = grouped.get(row.format) ?? { sizeRatios: [], compressionRatios: [], - totalBytes: 0, + bytesByDataset: new Map(), }; if ( row.valueBytes > 0 && @@ -750,7 +755,7 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre Number.isFinite(row.baselineBytes) ) { ratios.sizeRatios.push(row.valueBytes / row.baselineBytes); - ratios.totalBytes += row.valueBytes; + ratios.bytesByDataset.set(JSON.stringify([row.dataset, row.datasetVariant]), row.valueBytes); } if ( row.uncompressedBytes !== null && @@ -763,6 +768,12 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre } grouped.set(row.format, ratios); } + // Lance and Arrow IPC come from their own snapshots, so a format can lack a dataset that the + // others have. Totals use only the datasets every ranked format has. + const ranked = [...grouped.values()].filter((ratios) => ratios.sizeRatios.length > 0); + const commonDatasets = [...(ranked[0]?.bytesByDataset.keys() ?? [])].filter((dataset) => + ranked.every((ratios) => ratios.bytesByDataset.has(dataset)), + ); const rankings: CompressionSizeRanking[] = []; for (const [name, ratios] of grouped) { const ratio = geoMean(ratios.sizeRatios); @@ -779,7 +790,13 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre ratio, maxRatio, compressionRatio: geoMean(ratios.compressionRatios), - totalBytes: ratios.totalBytes, + totalBytes: + commonDatasets.length === 0 + ? null + : commonDatasets.reduce( + (sum, dataset) => sum + (ratios.bytesByDataset.get(dataset) ?? 0), + 0, + ), }); } } @@ -877,6 +894,8 @@ async function compressionSizeSamples(): Promise { c.timestamp DESC, s.commit_sha DESC ) SELECT selected.format AS format, + selected.dataset AS dataset, + selected.dataset_variant AS "datasetVariant", selected.value_bytes AS "valueBytes", selected.parquet_bytes AS "baselineBytes", COALESCE(uncompressed.uncompressed_bytes, selected.uncompressed_bytes) @@ -964,6 +983,8 @@ async function collectStringEncodingSizeSummary(): Promise { LIMIT 1 ) SELECT s.dataset_variant AS format, + s.dataset AS dataset, + NULL AS "datasetVariant", s.value_bytes::float8 AS "valueBytes", b.value_bytes::float8 AS "baselineBytes", s.uncompressed_bytes::float8 AS "uncompressedBytes"