diff --git a/.changeset/severity-class-classifier.md b/.changeset/severity-class-classifier.md new file mode 100644 index 0000000000..b68c374492 --- /dev/null +++ b/.changeset/severity-class-classifier.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +--- + +Add `core/severity` with the OpenTelemetry log severity rules: +`getSeverityClass` (from SeverityNumber 1-24, falling back to a case-insensitive +severity text prefix), `getSeverityShortName` and `getSeverityLabel`. diff --git a/.changeset/severity-number-display.md b/.changeset/severity-number-display.md new file mode 100644 index 0000000000..609bfcc8ab --- /dev/null +++ b/.changeset/severity-number-display.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/app': patch +--- + +When a log source sets Severity Number Expression, log levels follow the +OpenTelemetry SeverityNumber (1-24) in the row side panel, the search results +and pattern table level cells, the pattern trend bars, and the trace waterfall's +log error and warning flags. Levels show the spec's short name, such as `INFO2`, +add the stored text in parentheses when it differs, as in +`INFO (Informational)`, and take their color from the number. The search results +and pattern table use the number only when the expression names a stored numeric +column. Copy and filter actions still use the stored text. Rows without a number +in 1-24, and sources without the field, show the stored text as before. Level +labels and the waterfall flags also treat `panic`, `dpanic`, `ftl` and `severe` +as errors and `wrn` as a warning. diff --git a/.changeset/source-severity-number-expression.md b/.changeset/source-severity-number-expression.md index 6ac30b61ca..15b2628347 100644 --- a/.changeset/source-severity-number-expression.md +++ b/.changeset/source-severity-number-expression.md @@ -6,5 +6,4 @@ Add an optional `severityNumberExpression` to log sources, pointing at the numeric OpenTelemetry SeverityNumber (1-24). Source auto-detection sets it when -an OTel log table has an integer `SeverityNumber` column. The field is stored -and editable only: log levels still come from the severity text. +an OTel log table has an integer `SeverityNumber` column. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 9f5f62522c..ef1bf3bebe 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -3031,6 +3031,7 @@ function DBSearchPageContent() { selectionResetKey={selectionResetKey} onSelectedRowsChange={onSelectedRowsChange} onResolvedColumnsChange={onResolvedColumnsChange} + colorLevelBySeverityNumber /> )} diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index 864d5ba1b2..0287178aaf 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -1,3 +1,8 @@ +import { + collapseSeverityClass, + getSeverityClass, + TEXT_PREFIX_RULES, +} from '@hyperdx/common-utils/dist/core/severity'; import { ChartPaletteTokenSchema, ColorCondition, @@ -20,6 +25,7 @@ import { formatNumber, getAllMetricTables, getColorFromCSSToken, + getLogLevelClass, getLogSeverityLevel, getMetricTableName, isColumnInSelect, @@ -1547,6 +1553,8 @@ describe('getLogSeverityLevel', () => { [undefined, 'Warning', 'warn'], ['', 'fatal', 'error'], [undefined, 'info', undefined], + [25, 'warn', 'warn'], + [0, 'panic', 'error'], ])('maps %p and %p to %p', (severityNumber, severityText, expected) => { expect(getLogSeverityLevel({ severityNumber, severityText })).toBe( expected, @@ -1554,6 +1562,31 @@ describe('getLogSeverityLevel', () => { }); }); +describe('getLogLevelClass', () => { + // Log-only vocabulary that trace StatusCode and chart colors don't take. + const LOG_ONLY_PREFIXES = [ + 'panic', + 'dpanic', + 'ftl', + 'severe', + 'wrn', + 'inf', + 'dbg', + 'trc', + ]; + + it.each(TEXT_PREFIX_RULES.filter(r => !LOG_ONLY_PREFIXES.includes(r.prefix)))( + 'agrees with the log severity class on $prefix', + ({ prefix }) => { + const severityClass = getSeverityClass({ severityText: prefix }); + expect(getLogLevelClass(prefix)).toBe( + collapseSeverityClass(severityClass) ?? + (severityClass === 'UNKNOWN' ? undefined : 'info'), + ); + }, + ); +}); + describe('getColorFromCSSToken', () => { afterEach(() => { jest.restoreAllMocks(); diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index ddf77f48f5..e8bda178d3 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -12,6 +12,7 @@ import { Box } from '@mantine/core'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getKnownColumnsList, + getSelectableSeverityNumberExpression, getSelectAllColumnsQuerySettings, useSelectAllColumnsQuerySettings, } from '@/hooks/useMaterializedAliasColumnsOption'; @@ -35,6 +36,7 @@ export enum ROW_DATA_ALIASES { TRACE_ID = '__hdx_trace_id', SPAN_ID = '__hdx_span_id', SEVERITY_TEXT = '__hdx_severity_text', + SEVERITY_NUMBER = '__hdx_severity_number', SERVICE_NAME = '__hdx_service_name', RESOURCE_ATTRIBUTES = '__hdx_resource_attributes', EVENT_ATTRIBUTES = '__hdx_event_attributes', @@ -97,6 +99,8 @@ export function useRowData({ // select that instead of `*` when fetching full row data. const knownColumns = getKnownColumnsList(source); + const severityNumberExpr = getSelectableSeverityNumberExpression(source); + const baseConfig = { source: source.id, connection: source.connection, @@ -141,6 +145,14 @@ export function useRowData({ }, ] : []), + ...(severityNumberExpr + ? [ + { + valueExpression: severityNumberExpr, + alias: ROW_DATA_ALIASES.SEVERITY_NUMBER, + }, + ] + : []), ...((isLogSource(source) || isTraceSource(source)) && source.serviceNameExpression ? [ diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index cf0650612d..6f5603504e 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -446,6 +446,7 @@ export const DBRowSidePanelInner = ({ : undefined; const severityText: string | undefined = normalizedRow?.['__hdx_severity_text']; + const severityNumber = normalizedRow?.['__hdx_severity_number']; // Owns the View Trace nudge's auto-open latch. Kept here (above the loading // gate that unmounts the row content on every navigation) so the one-time @@ -915,7 +916,9 @@ export const DBRowSidePanelInner = ({ {controls} - {!sourceIsTrace && severityText && } + {!sourceIsTrace && ( + + )} {timestampDate && !isNaN(timestampDate.getTime()) && ( ·{' '} diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 6e5885b4a8..8ab9c36fdf 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -8,7 +8,7 @@ import React, { } from 'react'; import cx from 'classnames'; import { formatDistance } from 'date-fns'; -import { isString } from 'lodash'; +import { escapeRegExp, isString } from 'lodash'; import curry from 'lodash/curry'; import ms from 'ms'; import { useHotkeys } from 'react-hotkeys-hook'; @@ -23,12 +23,14 @@ import { import { chSqlToAliasMap, ClickHouseQueryError, + ColumnMeta, ColumnMetaType, convertCHDataTypeToJSType, extractColumnReferencesFromKey, isJSDataTypeJSONStringifiable, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; +import { parseSeverityNumber } from '@hyperdx/common-utils/dist/core/severity'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { DENOISE_NOISE_THRESHOLD, @@ -36,6 +38,7 @@ import { } from '@hyperdx/common-utils/dist/drain'; import { BuilderChartConfigWithDateRange, + isLogSource, SelectList, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -78,6 +81,7 @@ import { useRenderedSqlChartConfig, } from '@/hooks/useChartConfig'; import { CsvColumn, useCsvExport } from '@/hooks/useCsvExport'; +import { getSelectableSeverityNumberExpression } from '@/hooks/useMaterializedAliasColumnsOption'; import { useColumns, useTableMetadata } from '@/hooks/useMetadata'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; import { useGroupedPatterns } from '@/hooks/usePatterns'; @@ -104,6 +108,7 @@ import { import { getChartColorInfo, getLogLevelClass, + getLogSeverityLevel, logLevelColor, useLocalStorage, usePrevious, @@ -389,6 +394,7 @@ export const RawLogTable = memo( getRowWhere, variant = 'default', onRemoveColumn, + severityTextColumn: severityTextColumnProp, }: { wrapLines?: boolean; displayedColumns: string[]; @@ -432,6 +438,9 @@ export const RawLogTable = memo( getRowWhere?: (row: Record) => RowWhereResult; variant?: DBRowTableVariant; onRemoveColumn?: (column: string) => void; + // The column holding the severity text when it isn't the source's + // severityTextExpression, e.g. an aliased column. + severityTextColumn?: string; }) => { const dedupedRows = useMemo(() => { const lIds = new Set(); @@ -551,9 +560,26 @@ export const RawLogTable = memo( } }, [isLive, prevIsLive, tableContainerRef]); + // The inferred level column can be another text column (a ServiceName + // like "tracer"), which must not take the number's color. + const severityTextColumn = + severityTextColumnProp ?? + (source != null && isLogSource(source) + ? source.severityTextExpression?.trim() + : undefined); + const logLevelColumn = useMemo(() => { + // Inference skips empty text, so it would miss the level column on a + // page of number-only rows. + if ( + severityTextColumn != null && + displayedColumns.includes(severityTextColumn) && + dedupedRows.some(row => INTERNAL_ROW_FIELDS.SEVERITY_NUMBER in row) + ) { + return severityTextColumn; + } return inferLogLevelColumn(dedupedRows); - }, [dedupedRows]); + }, [dedupedRows, displayedColumns, severityTextColumn]); const csvColumns = useMemo( () => @@ -682,7 +708,15 @@ export const RawLogTable = memo( ); @@ -700,7 +734,23 @@ export const RawLogTable = memo( const strValue = typeof value === 'string' ? value : `${value}`; if (column === logLevelColumn) { - return ; + const severityNumber = + column === severityTextColumn + ? info.row.original[INTERNAL_ROW_FIELDS.SEVERITY_NUMBER] + : undefined; + // A NULL text next to a valid number is a number-only row, + // not the text "null". + return ( + + ); } const maxLen = wrapLinesEnabled @@ -750,6 +800,7 @@ export const RawLogTable = memo( columnNameMap, columnTypeMap, logLevelColumn, + severityTextColumn, expandedRows, toggleRowExpansion, showExpandButton, @@ -1543,6 +1594,70 @@ export function getPersistedBlockColumns(engineFull: string): string[] { return columns; } +/** + * The ` AS __hyperdx_severity_number` select item for the level cell, or + * undefined. It needs the severity text in the select and one stored numeric + * column, so unknown data can't fail the query or make it costly, and skips + * DISTINCT and GROUP BY, where an extra column changes the rows. + */ +export function getSeverityNumberSelect({ + source, + config, + tableColumns, + severityTextAlias, +}: { + source: TSource; + config: Pick; + tableColumns: + | Pick[] + | undefined; + // Set when the select shows the text as ` as ` instead. + severityTextAlias?: string; +}): string | undefined { + if (!isLogSource(source)) return undefined; + const numberExpr = getSelectableSeverityNumberExpression(source); + const textExpr = source.severityTextExpression?.trim(); + const { select, groupBy } = config; + if ( + !numberExpr || + !textExpr || + typeof select !== 'string' || + /^\s*distinct\b/i.test(select) || + (groupBy != null && groupBy.length > 0) + ) { + return undefined; + } + const selectItems = splitAndTrimWithBracket(select); + // ClickHouse resolves a select alias before a table column, so a user item + // aliased to the column's name would replace the column checked below. + const userAlias = new RegExp( + `\\bas\\s+[\`"]?${escapeRegExp(numberExpr)}[\`"]?$`, + 'i', + ); + const textItem = + severityTextAlias == null + ? textExpr + : `${textExpr} as ${severityTextAlias}`; + if ( + !selectItems.includes(textItem) || + selectItems.some(item => userAlias.test(item)) + ) { + return undefined; + } + const column = tableColumns?.find(c => c.name === numberExpr); + // An ALIAS column is computed on every read and an EPHEMERAL one can't be + // selected at all. + if ( + column == null || + column.default_type === 'ALIAS' || + column.default_type === 'EPHEMERAL' || + convertCHDataTypeToJSType(column.type) !== JSDataType.Number + ) { + return undefined; + } + return `${numberExpr} AS ${INTERNAL_ROW_FIELDS.SEVERITY_NUMBER}`; +} + function getSelectLength(select: SelectList): number { if (typeof select === 'string') { return select.split(',').filter(s => s.trim().length > 0).length; @@ -1554,7 +1669,16 @@ function getSelectLength(select: SelectList): number { export function useConfigWithAdditionalSelect( config: BuilderChartConfigWithDateRange, sourceId?: string, + { + severityNumberSource, + severityTextAlias, + }: { severityNumberSource?: TSource; severityTextAlias?: string } = {}, ) { + const wantsSeverityNumber = + severityNumberSource != null && + isLogSource(severityNumberSource) && + !!severityNumberSource.severityNumberExpression; + const { data: tableMetadata } = useTableMetadata({ databaseName: config.from.databaseName, tableName: config.from.tableName, @@ -1562,15 +1686,27 @@ export function useConfigWithAdditionalSelect( }); // Only check for row-ID columns for row-level queries (sourceId present). - // Skip for aggregate queries (e.g. patterns) where extra keys are irrelevant. - const { data: columns } = useColumns( + // Skip for aggregate queries (e.g. patterns) where extra keys are irrelevant, + // unless the severity number's column type is needed. + const { + data: columns, + isLoading: isColumnsLoading, + failureCount: columnsFailureCount, + } = useColumns( { databaseName: config.from.databaseName, tableName: config.from.tableName, connectionId: config.connection, }, - { enabled: !!sourceId }, + { + enabled: !!sourceId || wantsSeverityNumber, + ...(wantsSeverityNumber ? { retry: 1 } : {}), + }, ); + // React Query keeps the retry option of whichever observer started the fetch + // (the search page uses the default), so stop waiting after the first failure. + const isWaitingForColumns = + wantsSeverityNumber && isColumnsLoading && !columnsFailureCount; const primaryKey = tableMetadata?.primary_key; const partitionKey = tableMetadata?.partition_key; @@ -1579,6 +1715,10 @@ export function useConfigWithAdditionalSelect( if (primaryKey == null || partitionKey == null) { return undefined; } + // Wait for the column types so the query isn't rerun with a new select. + if (isWaitingForColumns) { + return undefined; + } let extraKeys: string[] = []; let hasBlockColumns = false; @@ -1619,11 +1759,39 @@ export function useConfigWithAdditionalSelect( ]) : undefined; - return { ...config, select, additionalKeysLength, rowKeyColumns }; - }, [primaryKey, partitionKey, config, tableMetadata, columns, sourceId]); + const severityNumberSelect = + severityNumberSource != null && typeof select === 'string' + ? getSeverityNumberSelect({ + source: severityNumberSource, + config, + tableColumns: columns, + severityTextAlias, + }) + : undefined; + + return { + ...config, + select: + severityNumberSelect == null + ? select + : `${select},${severityNumberSelect}`, + additionalKeysLength, + rowKeyColumns, + }; + }, [ + primaryKey, + partitionKey, + config, + tableMetadata, + columns, + isWaitingForColumns, + sourceId, + severityNumberSource, + severityTextAlias, + ]); } -function selectColumnMapWithoutAdditionalKeys( +export function selectColumnMapWithoutAdditionalKeys( selectMeta: ColumnMetaType[] | undefined, additionalKeysLength: number | undefined, ): Map< @@ -1636,7 +1804,15 @@ function selectColumnMapWithoutAdditionalKeys( if (selectMeta == null || additionalKeysLength == null) { return new Map(); } - const sm = selectMeta.slice(0, selectMeta.length - additionalKeysLength); + // The severity number is removed by name rather than counted as a key: a + // live-tail placeholder can be the previous result, with or without it. + const userAndKeyMeta = selectMeta.filter( + c => c.name !== INTERNAL_ROW_FIELDS.SEVERITY_NUMBER, + ); + const sm = userAndKeyMeta.slice( + 0, + userAndKeyMeta.length - additionalKeysLength, + ); return new Map( sm?.map(c => [ @@ -1677,6 +1853,7 @@ function DBSqlRowTableComponent({ tableId, errorVariant, onResolvedColumnsChange, + colorLevelBySeverityNumber = false, }: { config: BuilderChartConfigWithDateRange; sourceId?: string; @@ -1710,6 +1887,9 @@ function DBSqlRowTableComponent({ tableId?: string; errorVariant?: ChartErrorStateVariant; onResolvedColumnsChange?: (meta: ColumnMetaType[]) => void; + // Selects the source's SeverityNumber for the level cell. Opt-in so only the + // search page's query changes. + colorLevelBySeverityNumber?: boolean; }) { const { data: me } = api.useMe(); const { toggleColumn, displayedColumns: contextDisplayedColumns } = @@ -1767,7 +1947,19 @@ function DBSqlRowTableComponent({ return base; }, [me, config, orderByArray]); - const mergedConfig = useConfigWithAdditionalSelect(mergedConfigObj, sourceId); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: sourceId, + }); + const configWithAdditionalSelect = useConfigWithAdditionalSelect( + mergedConfigObj, + sourceId, + { severityNumberSource: colorLevelBySeverityNumber ? source : undefined }, + ); + // Wait for the source so the first query isn't rerun with the severity number. + const mergedConfig = + colorLevelBySeverityNumber && isSourceLoading + ? undefined + : configWithAdditionalSelect; const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = useOffsetPaginatedQuery(mergedConfig ?? config, { @@ -1888,7 +2080,6 @@ function DBSqlRowTableComponent({ } }, [data?.meta, onResolvedColumnsChange]); - const { data: source } = useSource({ id: sourceId }); const patternColumn = columns[columns.length - 1]; const groupedPatterns = useGroupedPatterns({ config, diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index c683786711..db877c11e6 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -50,6 +50,7 @@ interface Props { selectionResetKey?: string; onSelectedRowsChange?: (hasSelectedRows: boolean) => void; onResolvedColumnsChange?: (meta: ColumnMetaType[]) => void; + colorLevelBySeverityNumber?: boolean; // Clicking outside the row side panel (and outside `keepOpenSelector`) closes // it. Enabled by default; pass `false` to opt out. closeOnClickOutside?: boolean; @@ -82,6 +83,7 @@ export default function DBSqlRowTableWithSideBar({ selectionResetKey, onSelectedRowsChange, onResolvedColumnsChange, + colorLevelBySeverityNumber, closeOnClickOutside = true, keepOpenSelector = DEFAULT_KEEP_OPEN_SELECTOR, }: Props) { @@ -163,6 +165,7 @@ export default function DBSqlRowTableWithSideBar({ selectionResetKey={selectionResetKey} onSelectedRowsChange={onSelectedRowsChange} onResolvedColumnsChange={onResolvedColumnsChange} + colorLevelBySeverityNumber={colorLevelBySeverityNumber} /> ); diff --git a/packages/app/src/components/DBTable/__tests__/rowExport.test.ts b/packages/app/src/components/DBTable/__tests__/rowExport.test.ts new file mode 100644 index 0000000000..1587fe9ef2 --- /dev/null +++ b/packages/app/src/components/DBTable/__tests__/rowExport.test.ts @@ -0,0 +1,9 @@ +import { toExportableRow } from '@/components/DBTable/rowExport'; + +describe('toExportableRow', () => { + it('drops the hidden severity number', () => { + expect( + toExportableRow({ SeverityText: 'info', __hyperdx_severity_number: 17 }), + ).toEqual({ SeverityText: 'info' }); + }); +}); diff --git a/packages/app/src/components/DBTable/rowExport.ts b/packages/app/src/components/DBTable/rowExport.ts index cfc43da754..676262e329 100644 --- a/packages/app/src/components/DBTable/rowExport.ts +++ b/packages/app/src/components/DBTable/rowExport.ts @@ -16,6 +16,7 @@ export function toExportableRow( const { [INTERNAL_ROW_FIELDS.ID]: _id, [INTERNAL_ROW_FIELDS.ALIAS_WITH]: _aliasWith, + [INTERNAL_ROW_FIELDS.SEVERITY_NUMBER]: _severityNumber, ...rest } = row; diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index 759fdebe29..08963045fe 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -57,7 +57,7 @@ import { TRACE_WATERFALL_FETCH_LIMIT, TRACE_WATERFALL_ROW_LIMIT, } from '@/components/traceWaterfallLimits'; -import { useColumns } from '@/hooks/useMetadata'; +import { getSelectableSeverityNumberExpression } from '@/hooks/useMaterializedAliasColumnsOption'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; import useRowWhere, { WithClause } from '@/hooks/useRowWhere'; import useWaterfallSearchState from '@/hooks/useWaterfallSearchState'; @@ -99,7 +99,7 @@ export type SpanRow = { StatusCode?: string; ServiceName?: string; SeverityText?: string; - SeverityNumber?: number; + SeverityNumber?: number | string; HyperDXEventType: 'span'; type?: string; SpanAttributes?: Record; @@ -214,7 +214,7 @@ function getConfig( traceId: string, hiddenRowExpression?: string, hiddenRowExpressionLanguage: 'lucene' | 'sql' = 'lucene', - selectSeverityNumber = false, + severityNumberExpression?: string, ) { const alias: Record = { Body: getTableBody(source), @@ -348,10 +348,10 @@ function getConfig( }, ] : []), - ...(selectSeverityNumber + ...(severityNumberExpression ? [ { - valueExpression: 'SeverityNumber', + valueExpression: severityNumberExpression, alias: 'SeverityNumber', }, ] @@ -399,7 +399,7 @@ export function useEventsAroundFocus({ enabled, hiddenRowExpression, hiddenRowExpressionLanguage = 'lucene', - selectSeverityNumber = false, + severityNumberExpression, }: { tableSource: TTraceSource | TLogSource; focusDate: Date; @@ -409,7 +409,7 @@ export function useEventsAroundFocus({ /** An expression (in `hiddenRowExpressionLanguage`) that identifies rows to be hidden. Hidden rows will be returned with a `__hdx_hidden: true` column. */ hiddenRowExpression?: string; hiddenRowExpressionLanguage?: 'lucene' | 'sql'; - selectSeverityNumber?: boolean; + severityNumberExpression?: string; }) { const { config, alias, type } = useMemo( () => @@ -418,14 +418,14 @@ export function useEventsAroundFocus({ traceId, hiddenRowExpression, hiddenRowExpressionLanguage, - selectSeverityNumber, + severityNumberExpression, ), [ tableSource, traceId, hiddenRowExpression, hiddenRowExpressionLanguage, - selectSeverityNumber, + severityNumberExpression, ], ); @@ -484,7 +484,7 @@ export function useEventsAroundFocus({ return [...trimmedBefore, ...trimmedAfter].map(cd => { // Omit SpanAttributes, SpanEvents, __hdx_hidden and SeverityNumber from rowWhere id generation. // SpanAttributes and SpanEvents can be large objects, and __hdx_hidden may be a lucene expression. - // SeverityNumber is only selected when the column exists; leaving it out keeps row ids stable. + // SeverityNumber is optional and may be an expression; leaving it out keeps row ids stable. const rowWhereResult = getRowWhere( omit(cd, [ 'SpanAttributes', @@ -779,16 +779,11 @@ export function DBTraceWaterfallChartContainer({ enabled: true, }); - const { data: logColumns, isLoading: isLogColumnsLoading } = useColumns( - { - databaseName: logTableSource?.from.databaseName ?? '', - tableName: logTableSource?.from.tableName ?? '', - connectionId: logTableSource?.connection ?? '', - }, - { retry: 1 }, - ); - const hasSeverityNumber = - logColumns?.some(column => column.name === 'SeverityNumber') ?? false; + // Sources without Severity Number Expression color logs by their text. + const severityNumberExpression = + logTableSource != null + ? getSelectableSeverityNumberExpression(logTableSource) + : undefined; const { rows: logRowsData, @@ -805,13 +800,11 @@ export function DBTraceWaterfallChartContainer({ traceId, hiddenRowExpression: logWhere ? `NOT (${logWhere})` : undefined, hiddenRowExpressionLanguage: logFilterLanguage, - // Wait for the columns (a failed lookup runs without SeverityNumber) so the - // select doesn't change mid-flight and run the log query twice. - enabled: !!logTableSource && !isLogColumnsLoading, - selectSeverityNumber: hasSeverityNumber, + enabled: !!logTableSource, + severityNumberExpression, }); - const isFetching = traceIsFetching || logIsFetching || isLogColumnsLoading; + const isFetching = traceIsFetching || logIsFetching; // Only a fatal trace failure (bad source / connection) blanks the waterfall. // A *filter* error on either side is non-fatal: the offending query falls back // to unfiltered data and the error is surfaced inline next to its input, so a diff --git a/packages/app/src/components/LogLevel.tsx b/packages/app/src/components/LogLevel.tsx index 0bae041141..765f523221 100644 --- a/packages/app/src/components/LogLevel.tsx +++ b/packages/app/src/components/LogLevel.tsx @@ -1,13 +1,21 @@ +import { + collapseSeverityClass, + getSeverityClass, + getSeverityLabel, +} from '@hyperdx/common-utils/dist/core/severity'; import { Text, TextProps } from '@mantine/core'; -import { getLogLevelClass } from '@/utils'; - export default function LogLevel({ level, + severityNumber, ...props -}: { level: string } & TextProps) { - const levelClass = getLogLevelClass(level); +}: { level?: string | null; severityNumber?: unknown } & TextProps) { + const label = getSeverityLabel({ severityNumber, severityText: level }); + if (!label) return null; + const levelClass = collapseSeverityClass( + getSeverityClass({ severityNumber, severityText: level }), + ); return ( - {level} + {label} ); } diff --git a/packages/app/src/components/PatternTable.tsx b/packages/app/src/components/PatternTable.tsx index bbecce94ab..6098632efa 100644 --- a/packages/app/src/components/PatternTable.tsx +++ b/packages/app/src/components/PatternTable.tsx @@ -80,6 +80,7 @@ export default function PatternTable({ ((source?.kind === SourceKind.Log || source?.kind === SourceKind.Trace) && source.serviceNameExpression) || '', + severityNumberSource: source, totalCount, }); @@ -164,6 +165,7 @@ export default function PatternTable({ }} config={patternQueryConfig} showExpandButton={false} + severityTextColumn="level" /> {selectedPattern && source && ( { expect(config.select).not.toContainEqual({ valueExpression: '*' }); }); + it('selects severityNumberExpression unless the Known Columns List omits it', () => { + const selectsSeverityNumber = (overrides: Partial) => { + mockUseQueriedChartConfig.mockClear(); + renderHook(() => + useRowData({ source: { ...source, ...overrides }, rowId: "id='a'" }), + ); + const [config] = mockUseQueriedChartConfig.mock.calls[0]; + return config.select.some( + (s: { valueExpression: string; alias?: string }) => + s.alias === '__hdx_severity_number' && + s.valueExpression === 'SeverityNumber', + ); + }; + + expect( + selectsSeverityNumber({ severityNumberExpression: 'SeverityNumber' }), + ).toBe(true); + expect(selectsSeverityNumber({})).toBe(false); + expect( + selectsSeverityNumber({ + severityNumberExpression: 'SeverityNumber', + knownColumnsListExpression: 'Timestamp, Body', + }), + ).toBe(false); + }); + it('adds no query settings while the viewer option is off', () => { renderHook(() => useRowData({ source, rowId: "id='abc123'" })); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.rememberedTab.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.rememberedTab.test.tsx index 57629e6aac..3284f4244a 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.rememberedTab.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.rememberedTab.test.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { TSource } from '@hyperdx/common-utils/dist/types'; import { MantineProvider } from '@mantine/core'; -import { act, render } from '@testing-library/react'; +import { act, render, screen } from '@testing-library/react'; // Write-through in-memory nuqs so a real click *sequence* can be replayed: each // setter updates the store and re-renders every subscribed hook, the way a URL @@ -63,6 +63,7 @@ const mockRow = { __hdx_timestamp: 1700000000, __hdx_body: 'a log line', __hdx_severity_text: 'info', + __hdx_severity_number: 17, }; jest.mock('../DBRowDataPanel', () => ({ @@ -155,7 +156,6 @@ jest.mock('../SidePanelBreadcrumbs', () => ({ __esModule: true, default: () => null, })); -jest.mock('../LogLevel', () => ({ __esModule: true, default: () => null })); jest.mock('../ServiceMap/ServiceMapSidePanel', () => ({ __esModule: true, default: () => null, @@ -281,3 +281,10 @@ describe('DBRowSidePanelInner — remembered tab across a surrounding-context dr expect(activeTab()).toBe(Tab.Overview); }); }); + +describe('DBRowSidePanelInner severity badge', () => { + it('labels the badge from the severity number', () => { + renderPanel('row-1'); + expect(screen.getByText('ERROR (info)')).toBeInTheDocument(); + }); +}); diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index 61730b121e..e4a689e2bd 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -1,18 +1,27 @@ import { Provider } from 'jotai'; -import type { ReactElement } from 'react'; -import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import type { ComponentProps, ReactElement } from 'react'; +import { + type BuilderChartConfigWithDateRange, + SourceKind, + type TLogSource, +} from '@hyperdx/common-utils/dist/types'; import { renderHook, screen, waitFor, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; +import { RowSidePanelContext } from '@/components/DBRowSidePanel'; import { appendSelectWithAdditionalKeys, getPersistedBlockColumns, + getSeverityNumberSelect, RawLogTable, + selectColumnMapWithoutAdditionalKeys, useConfigWithAdditionalSelect, } from '@/components/DBRowTable'; import * as useChartConfigModule from '@/hooks/useChartConfig'; import { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; import { RowClickAction, UserPreferences } from '@/useUserPreferences'; +import * as utilsModule from '@/utils'; +import * as clipboardModule from '@/utils/clipboard'; const mockSetQueryState = jest.fn(); @@ -45,6 +54,54 @@ jest.mock('nuqs', () => ({ const mockRowWhereResult: RowWhereResult = { where: '', aliasWith: [] }; +// jsdom does no layout, so the virtualizer renders no rows. Fake one screen +// of geometry. +function fakeOneScreenOfRows() { + const VIEWPORT_HEIGHT = 900; + let rectSpy: jest.SpyInstance; + let clientHeightSpy: jest.SpyInstance; + + beforeAll(() => { + rectSpy = jest + .spyOn(HTMLElement.prototype, 'getBoundingClientRect') + .mockImplementation(function (this: HTMLElement) { + const height = + this.dataset.testid === 'search-results-table' ? VIEWPORT_HEIGHT : 23; + return { + width: 580, + height, + top: 0, + left: 0, + right: 580, + bottom: height, + x: 0, + y: 0, + toJSON: () => ({}), + }; + }); + clientHeightSpy = jest + .spyOn(HTMLElement.prototype, 'clientHeight', 'get') + .mockReturnValue(VIEWPORT_HEIGHT); + }); + + afterAll(() => { + rectSpy.mockRestore(); + clientHeightSpy.mockRestore(); + }); +} + +const logSource: TLogSource = { + id: 'source-id', + kind: SourceKind.Log, + name: 'logs', + connection: 'conn', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, ServiceName, SeverityText, Body', + severityTextExpression: 'SeverityText', + severityNumberExpression: 'SeverityNumber', +}; + describe('RawLogTable', () => { beforeEach(() => { jest.clearAllMocks(); @@ -275,43 +332,8 @@ describe('RawLogTable', () => { describe('Row click behavior', () => { const ROW_ID = 'Timestamp = 1'; - const VIEWPORT_HEIGHT = 900; - - // The table is virtualized and jsdom performs no layout, so every element - // measures 0px and the virtualizer renders no rows at all. Fake just enough - // geometry for one screenful. Same approach as the MetricExplorer tests. - let rectSpy: jest.SpyInstance; - let clientHeightSpy: jest.SpyInstance; - - beforeAll(() => { - rectSpy = jest - .spyOn(HTMLElement.prototype, 'getBoundingClientRect') - .mockImplementation(function (this: HTMLElement) { - const height = - this.dataset.testid === 'search-results-table' - ? VIEWPORT_HEIGHT - : 23; - return { - width: 580, - height, - top: 0, - left: 0, - right: 580, - bottom: height, - x: 0, - y: 0, - toJSON: () => ({}), - }; - }); - clientHeightSpy = jest - .spyOn(HTMLElement.prototype, 'clientHeight', 'get') - .mockReturnValue(VIEWPORT_HEIGHT); - }); - - afterAll(() => { - rectSpy.mockRestore(); - clientHeightSpy.mockRestore(); - }); + + fakeOneScreenOfRows(); const baseProps = { displayedColumns: ['col1'], @@ -535,6 +557,143 @@ describe('RawLogTable', () => { }); }); + describe('Level cell', () => { + fakeOneScreenOfRows(); + + const onPropertyAddClick = jest.fn(); + const renderLevelTable = ( + row: Record, + displayedColumns: string[], + props: Partial> = {}, + ) => + renderWithMantine( + + mockRowWhereResult} + columnTypeMap={new Map()} + source={logSource} + {...props} + /> + , + ); + + it('labels and colors from the number but copies and filters on the stored text', async () => { + const copyText = jest + .spyOn(clipboardModule, 'copyTextToClipboard') + .mockResolvedValue(true); + renderLevelTable( + { SeverityText: 'info', __hyperdx_severity_number: 17 }, + ['SeverityText'], + ); + + const cell = await screen.findByText('ERROR (info)'); + expect(cell.getAttribute('style')).toContain('red'); + await userEvent.hover(cell); + await userEvent.click(await screen.findByLabelText('Copy field value')); + await userEvent.click(screen.getByLabelText('Exclude this value')); + + expect(copyText).toHaveBeenCalledWith('info'); + expect(onPropertyAddClick).toHaveBeenCalledWith( + 'SeverityText', + 'info', + 'exclude', + ); + copyText.mockRestore(); + }); + + // Inference alone would pick ServiceName, the only column with level text. + it.each([ + ['', 17, 'ERROR'], + [null, 21, 'FATAL'], + ])( + 'labels a number-only row (text %p) in the severity text column', + async (text, severityNumber, label) => { + renderLevelTable( + { + ServiceName: 'tracer', + SeverityText: text, + __hyperdx_severity_number: severityNumber, + }, + ['ServiceName', 'SeverityText'], + ); + + expect( + (await screen.findByText(label)).getAttribute('style'), + ).toContain('red'); + expect(screen.getByText('tracer').getAttribute('style')).toBeNull(); + }, + ); + + it('keeps the inferred level column without the hidden severity number', async () => { + renderLevelTable({ ServiceName: 'tracer', SeverityText: '' }, [ + 'ServiceName', + 'SeverityText', + ]); + + expect( + (await screen.findByText('tracer')).getAttribute('style'), + ).not.toBeNull(); + }); + + it.each([ + // A live-tail placeholder can keep the number after the select dropped + // the severity text column. + [ + 'an inferred level column that is not the severity text', + { lvl: 'info', __hyperdx_severity_number: 17 }, + ['lvl'], + 'info', + ], + [ + 'NULL text next to an unspecified number', + { SeverityText: null, __hyperdx_severity_number: 0 }, + ['SeverityText'], + 'null', + ], + ])('shows the stored text for %s', async (_name, row, columns, text) => { + renderLevelTable(row, columns); + + expect(await screen.findByText(text)).toBeInTheDocument(); + }); + + it('colors an aliased severity text column, as on the patterns table', async () => { + renderLevelTable( + { level: '', __hyperdx_severity_number: 13 }, + ['level'], + { + source: undefined, + severityTextColumn: 'level', + }, + ); + + expect(await screen.findByText('WARN')).toBeInTheDocument(); + }); + + it.each([ + [{ level: 'info', __hyperdx_severity_number: 17 }, 'error'], + [{ level: 'warn' }, 'warn'], + ])('colors the pattern trend of %p as %p', async (row, level) => { + const logLevelColor = jest.spyOn(utilsModule, 'logLevelColor'); + renderLevelTable( + { + ...row, + __hdx_pattern_trend: { + data: [], + dateRange: [new Date(0), new Date(1)], + }, + }, + ['__hdx_pattern_trend'], + { source: undefined }, + ); + + await waitFor(() => expect(logLevelColor).toHaveBeenCalledWith(level)); + logLevelColor.mockRestore(); + }); + }); + describe('Error state', () => { const paginationProps = { displayedColumns: ['col1'], @@ -985,3 +1144,233 @@ describe('useConfigWithAdditionalSelect', () => { expect(result.current?.rowKeyColumns).toBeUndefined(); }); }); + +describe('getSeverityNumberSelect', () => { + const select = 'Timestamp, ServiceName, SeverityText, Body'; + const tableColumns = [ + { name: 'SeverityNumber', type: 'UInt8', default_type: '' }, + ]; + const ITEM = 'SeverityNumber AS __hyperdx_severity_number'; + + type Overrides = Partial[0]>; + + it.each<[string, Overrides]>([ + ['a plain select', {}], + [ + 'a known columns list with the column', + { + source: { + ...logSource, + knownColumnsListExpression: 'Timestamp, SeverityText, SeverityNumber', + }, + }, + ], + [ + 'a pattern select with the aliased text', + { + config: { + select: 'Body as __hdx_pattern_field,SeverityText as __hdx_level', + }, + severityTextAlias: '__hdx_level', + }, + ], + ])('selects the number for %s', (_name, overrides) => { + expect( + getSeverityNumberSelect({ + source: logSource, + config: { select }, + tableColumns, + ...overrides, + }), + ).toBe(ITEM); + }); + + it.each<[string, Overrides]>([ + [ + 'no severity number expression', + { source: { ...logSource, severityNumberExpression: undefined } }, + ], + ['a select without the severity text', { config: { select: 'Body' } }], + ['an aliased severity text', { config: { select: 'SeverityText AS lvl' } }], + [ + 'a DISTINCT select', + { config: { select: 'DISTINCT Timestamp, SeverityText, Body' } }, + ], + ['a grouped select', { config: { select, groupBy: 'SeverityText' } }], + [ + 'a known columns list without the column', + { + source: { + ...logSource, + knownColumnsListExpression: 'Timestamp, SeverityText, Body', + }, + }, + ], + [ + 'a user item aliased to the column name', + { + config: { select: `${select}, LogAttributes['sev'] AS SeverityNumber` }, + }, + ], + ['a missing column', { tableColumns: [] }], + [ + 'a String column', + { + tableColumns: [ + { name: 'SeverityNumber', type: 'String', default_type: '' }, + ], + }, + ], + [ + 'an ALIAS column', + { + tableColumns: [ + { name: 'SeverityNumber', type: 'UInt8', default_type: 'ALIAS' }, + ], + }, + ], + [ + 'an expression rather than a column', + { + source: { + ...logSource, + severityNumberExpression: "toUInt8(LogAttributes['severity'])", + }, + }, + ], + [ + 'a pattern select without the aliased severity text', + { config: { select }, severityTextAlias: '__hdx_level' }, + ], + ])('skips %s', (_name, overrides) => { + expect( + getSeverityNumberSelect({ + source: logSource, + config: { select }, + tableColumns, + ...overrides, + }), + ).toBeUndefined(); + }); +}); + +describe('useConfigWithAdditionalSelect with a severity number source', () => { + const config: BuilderChartConfigWithDateRange = { + connection: 'conn', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: 'Timestamp, ServiceName, SeverityText, Body', + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [new Date(0), new Date(1)], + }; + const KEYS = 'Timestamp,ServiceName,SeverityText,Body,toDate(Timestamp)'; + + beforeEach(() => { + mockUseTableMetadata.mockReturnValue({ + data: { + primary_key: 'Timestamp', + partition_key: 'toDate(Timestamp)', + engine_full: 'MergeTree ORDER BY Timestamp', + }, + }); + mockUseColumns.mockReturnValue({ + data: [{ name: 'SeverityNumber', type: 'UInt8' }], + isLoading: false, + }); + }); + + afterEach(() => { + mockUseTableMetadata.mockReset(); + mockUseColumns.mockReset(); + }); + + it('appends the number last as a hidden column', () => { + const { result } = renderHook(() => + useConfigWithAdditionalSelect(config, 'source-id', { + severityNumberSource: logSource, + }), + ); + + expect(result.current?.select).toBe( + `${KEYS},SeverityNumber AS __hyperdx_severity_number`, + ); + expect(result.current?.additionalKeysLength).toBe(1); + }); + + it('leaves the select alone without the option', () => { + const { result } = renderHook(() => + useConfigWithAdditionalSelect(config, 'source-id'), + ); + + expect(result.current?.select).toBe(KEYS); + expect(result.current?.additionalKeysLength).toBe(1); + }); + + it('loads the column types for a query without a source id', () => { + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + { ...config, select: 'Body as p,SeverityText as __hdx_level' }, + undefined, + { severityNumberSource: logSource, severityTextAlias: '__hdx_level' }, + ), + ); + + expect(mockUseColumns).toHaveBeenCalledWith(expect.anything(), { + enabled: true, + retry: 1, + }); + expect(result.current?.select).toMatch( + /,SeverityNumber AS __hyperdx_severity_number$/, + ); + }); + + // A failing lookup started by another observer keeps the default retries. + it('stops waiting for the column types after the first failure', () => { + mockUseColumns.mockReturnValue({ + data: undefined, + isLoading: true, + failureCount: 1, + }); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect(config, 'source-id', { + severityNumberSource: logSource, + }), + ); + + expect(result.current?.select).toBe(KEYS); + }); + + it('waits for the column types before building the select', () => { + mockUseColumns.mockReturnValue({ data: undefined, isLoading: true }); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect(config, 'source-id', { + severityNumberSource: logSource, + }), + ); + + expect(result.current).toBeUndefined(); + }); +}); + +describe('selectColumnMapWithoutAdditionalKeys', () => { + const meta = [ + { name: 'Body', type: 'String' }, + { name: 'Timestamp', type: 'DateTime64(9)' }, + ]; + + it('hides the severity number whether or not the result carries it', () => { + const withNumber = [ + ...meta, + { name: '__hyperdx_severity_number', type: 'UInt8' }, + ]; + expect([...selectColumnMapWithoutAdditionalKeys(meta, 1).keys()]).toEqual([ + 'Body', + ]); + expect([ + ...selectColumnMapWithoutAdditionalKeys(withNumber, 1).keys(), + ]).toEqual(['Body']); + }); +}); diff --git a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx index eb498c616d..797b10cd40 100644 --- a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx @@ -125,14 +125,9 @@ const mockUseRowWhere = useRowWhere as jest.Mock; const mockUseColumns = useColumns as jest.Mock; const MockTimelineChart = TimelineChart as any; -// Mirrors useColumns, which is disabled, so never loading, without a table. -const setupColumnsMock = (state: { data?: object[]; isLoading?: boolean }) => - mockUseColumns.mockImplementation(({ tableName }: { tableName: string }) => - tableName ? { isLoading: false, ...state } : { isLoading: false }, - ); - -const selectsSeverityNumber = (query: { select: Array<{ alias?: string }> }) => - query.select.some(s => s.alias === 'SeverityNumber'); +const selectedSeverityNumber = (query: { + select: Array<{ alias?: string; valueExpression?: string }>; +}) => query.select.find(s => s.alias === 'SeverityNumber')?.valueExpression; describe('DBTraceWaterfallChartContainer', () => { // Common test data @@ -218,7 +213,6 @@ describe('DBTraceWaterfallChartContainer', () => { mockUseRowWhere.mockReturnValue(() => ({ where: 'row-id', aliasWith: [] })); MockTimelineChart.latestProps = {}; mockUseWaterfallSearchState.mockReturnValue(makeWaterfallSearchState()); - setupColumnsMock({ data: [] }); }); // Helper functions @@ -817,32 +811,35 @@ describe('DBTraceWaterfallChartContainer', () => { }); it.each([ - ['has it', [true], { data: [{ name: 'SeverityNumber', type: 'UInt8' }] }], - ['lacks it', [false], { data: [{ name: 'SeverityText', type: 'String' }] }], - // A failed lookup still shows logs, just without SeverityNumber. - ['failed', [false], {}], - // Held until the columns settle so a changed select can't run it twice. - ['is loading', [], { isLoading: true }], + [undefined, undefined, [undefined]], + [ + "toUInt8(LogAttributes['sev'])", + undefined, + ["toUInt8(LogAttributes['sev'])"], + ], + ['SeverityNumber', 'Timestamp, Body', [undefined]], ])( - 'when the log column lookup %s, enabled log queries select SeverityNumber: %p', - (_name, expected, columns) => { - setupColumnsMock(columns); + 'with Severity Number Expression %p and known columns %p, enabled log queries select %p without a column lookup', + (severityNumberExpression, knownColumnsListExpression, expected) => { setupQueryMocks({ traceData: mockTraceData, logData: mockLogData }); - renderComponent(); + renderComponent({ + ...mockLogTableSource, + severityNumberExpression, + knownColumnsListExpression, + }); const selects = mockUseOffsetPaginatedQuery.mock.calls .filter( ([query, opts]) => query?.from?.tableName === 'log_table' && opts?.enabled, ) - .map(([query]) => selectsSeverityNumber(query)); + .map(([query]) => selectedSeverityNumber(query)); expect([...new Set(selects)]).toEqual(expected); + expect(mockUseColumns).not.toHaveBeenCalled(); }, ); it('renders a trace without a log source', async () => { - // Without a log table the lookup is disabled, so it must not hold the chart. - setupColumnsMock({ isLoading: true }); setupQueryMocks({ traceData: mockTraceData }); renderComponent(null); await waitForLoading(); @@ -1093,14 +1090,14 @@ describe('useEventsAroundFocus', () => { afterData?: any; enabled?: boolean; tableSource?: TTraceSource | TLogSource; - selectSeverityNumber?: boolean; + severityNumberExpression?: string; }) => { const { beforeData = { data: [], meta: [] }, afterData = { data: [], meta: [] }, enabled = true, tableSource = mockTableSource, - selectSeverityNumber, + severityNumberExpression, } = options; mockUseOffsetPaginatedQuery.mockReset(); @@ -1123,22 +1120,22 @@ describe('useEventsAroundFocus', () => { dateRange: mockDateRange, traceId: mockTraceId, enabled, - selectSeverityNumber, + severityNumberExpression, }), ); return result.current; }; - it.each([true, false])( - 'selects SeverityNumber from a log source only with the flag (%p), never in row ids', - selectSeverityNumber => { + it.each([undefined, "toUInt8(LogAttributes['severity'])"])( + 'selects SeverityNumber from a log source only when given (%p), never in row ids', + severityNumberExpression => { const getRowWhere = jest.fn(() => ({ where: 'row-id', aliasWith: [] })); mockUseRowWhere.mockReturnValue(getRowWhere); const result = testEventsAroundFocus({ tableSource: { ...mockTableSource, kind: SourceKind.Log } as TLogSource, - selectSeverityNumber, + severityNumberExpression, beforeData: { data: [{ Body: 'log line', SeverityNumber: 17 }], meta: [{ name: 'Body', type: 'String' }], @@ -1146,7 +1143,7 @@ describe('useEventsAroundFocus', () => { }); const [query] = mockUseOffsetPaginatedQuery.mock.calls[0]; - expect(selectsSeverityNumber(query)).toBe(selectSeverityNumber); + expect(selectedSeverityNumber(query)).toBe(severityNumberExpression); expect(getRowWhere).toHaveBeenCalledWith({ Body: 'log line' }); expect(result.rows[0]).toMatchObject({ SeverityNumber: 17 }); }, diff --git a/packages/app/src/components/__tests__/LogLevel.test.tsx b/packages/app/src/components/__tests__/LogLevel.test.tsx new file mode 100644 index 0000000000..5227ed4737 --- /dev/null +++ b/packages/app/src/components/__tests__/LogLevel.test.tsx @@ -0,0 +1,31 @@ +import { screen } from '@testing-library/react'; + +import LogLevel from '@/components/LogLevel'; + +describe('LogLevel', () => { + it.each([ + ['info', '17', 'ERROR (info)'], + [null, 18, 'ERROR2'], + ['error', 0, 'error'], + ])( + 'labels text %p with number %p as %p in red', + (level, severityNumber, label) => { + renderWithMantine( + , + ); + + expect(screen.getByText(label).getAttribute('style')).toContain('red'); + }, + ); + + it.each([0, undefined])( + 'renders nothing without text or a valid number (%p)', + severityNumber => { + const { container } = renderWithMantine( + , + ); + + expect(container.querySelector('.mantine-Text-root')).toBeNull(); + }, + ); +}); diff --git a/packages/app/src/hooks/__tests__/usePatterns.test.tsx b/packages/app/src/hooks/__tests__/usePatterns.test.tsx new file mode 100644 index 0000000000..beba32f649 --- /dev/null +++ b/packages/app/src/hooks/__tests__/usePatterns.test.tsx @@ -0,0 +1,98 @@ +import React from 'react'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { renderHook, waitFor } from '@testing-library/react'; + +import { useGroupedPatterns } from '@/hooks/usePatterns'; + +let mockSampleRows: Record[]; + +jest.mock('@/components/DBRowTable', () => ({ + useConfigWithAdditionalSelect: (config: object) => ({ + ...config, + additionalKeysLength: 0, + }), +})); +jest.mock('@/hooks/useChartConfig', () => ({ + useQueriedChartConfig: () => ({ + data: { data: mockSampleRows }, + isLoading: false, + error: null, + }), +})); + +// Every sample mines into one pattern. +const fakePyodide = { + loadPackage: async () => {}, + pyimport: () => ({ install: async () => {} }), + globals: { set: () => {} }, + runPythonAsync: async (code: string) => + code.includes('add_log_message') + ? JSON.stringify( + mockSampleRows.map(() => ({ cluster_id: 1, template_mined: 'x' })), + ) + : undefined, +}; + +describe('useGroupedPatterns', () => { + const config: BuilderChartConfigWithDateRange = { + connection: 'conn', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), + ], + }; + + beforeEach(() => { + Object.assign(window, { loadPyodide: async () => fakePyodide }); + }); + + it("carries the last sample's severity number onto the pattern row", async () => { + mockSampleRows = [ + { + __hdx_pattern_field: 'x', + __hdx_timestamp: '2024-01-01T00:10:00Z', + __hdx_level: 'info', + __hyperdx_severity_number: 9, + }, + { + __hdx_pattern_field: 'x', + __hdx_timestamp: '2024-01-01T00:20:00Z', + __hdx_level: '', + __hyperdx_severity_number: 17, + }, + ]; + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + + const { result } = renderHook( + () => + useGroupedPatterns({ + config, + samples: 2, + bodyValueExpression: 'Body', + levelExpression: 'SeverityText', + }), + { + wrapper: ({ children }) => ( + + {children} + + ), + }, + ); + + await waitFor(() => + expect(result.current.data['1']).toMatchObject({ + level: '', + __hyperdx_severity_number: 17, + }), + ); + }); +}); diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 90b1aed0f1..52122dfba1 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -604,6 +604,44 @@ describe('useRowWhere', () => { ]); }); + describe('hidden severity number', () => { + const meta: ColumnMetaType[] = [ + { name: 'id', type: 'String' }, + { name: '__hyperdx_severity_number', type: 'UInt8' }, + ]; + const row = { id: '123', __hyperdx_severity_number: 17 }; + + // An empty map is what chSqlToAliasMap returns when parsing fails. + it('stays out of the where clause without an alias map', () => { + const { result } = renderHook(() => useRowWhere({ meta, aliasMap: {} })); + + expect(result.current(row)).toEqual({ where: "id='123'", aliasWith: [] }); + }); + + it('stays out of the alias WITH clauses', () => { + const { result } = renderHook(() => + useRowWhere({ + meta, + aliasMap: { + id: 'users.id', + __hyperdx_severity_number: 'SeverityNumber', + }, + }), + ); + + expect(result.current(row)).toEqual({ + where: "users.id='123'", + aliasWith: [ + { + name: 'id', + sql: { sql: 'users.id', params: {} }, + isSubquery: false, + }, + ], + }); + }); + }); + it('should memoize the column map', () => { const meta: ColumnMetaType[] = [{ name: 'id', type: 'String' }]; diff --git a/packages/app/src/hooks/useMaterializedAliasColumnsOption.ts b/packages/app/src/hooks/useMaterializedAliasColumnsOption.ts index ba70435379..90e06bd757 100644 --- a/packages/app/src/hooks/useMaterializedAliasColumnsOption.ts +++ b/packages/app/src/hooks/useMaterializedAliasColumnsOption.ts @@ -1,3 +1,4 @@ +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { isLogSource, isTraceSource, @@ -25,6 +26,23 @@ export function getKnownColumnsList(source: TSource): string | undefined { : undefined; } +// Undefined when the Known Columns List omits it: a Distributed or Merge +// source can read only those columns, and the number isn't worth failing a +// query over. +export function getSelectableSeverityNumberExpression( + source: TSource, +): string | undefined { + const expression = isLogSource(source) + ? source.severityNumberExpression?.trim() + : undefined; + if (!expression) return undefined; + const knownColumns = getKnownColumnsList(source); + return knownColumns == null || + splitAndTrimWithBracket(knownColumns).includes(expression) + ? expression + : undefined; +} + // The settings that the row query adds for this source while the option is on, // if any. A setting that the source's own query settings define keeps the // source's value. diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index ece24e70cd..b18b16bb3a 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -1,13 +1,17 @@ import { useMemo } from 'react'; import stripAnsi from 'strip-ansi'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { + BuilderChartConfigWithDateRange, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { useQuery } from '@tanstack/react-query'; import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils'; import { useConfigWithAdditionalSelect } from '@/components/DBRowTable'; import { reconstructTemplate } from '@/components/Patterns/reconstructTemplate'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { INTERNAL_ROW_FIELDS } from '@/hooks/useRowWhere'; import { getFirstTimestampValueExpression } from '@/source'; // We don't want to load pyodide over and over again, use react query to cache the async instance @@ -131,6 +135,7 @@ function usePatterns({ bodyValueExpression, levelExpression, serviceNameExpression, + severityNumberSource, enabled = true, }: { config: BuilderChartConfigWithDateRange; @@ -138,25 +143,30 @@ function usePatterns({ bodyValueExpression: string; levelExpression?: string; serviceNameExpression?: string; + severityNumberSource?: TSource; enabled?: boolean; }) { - const configWithPrimaryAndPartitionKey = useConfigWithAdditionalSelect({ - ...config, - // TODO: User-configurable pattern columns and non-pattern/group by columns - select: [ - `${bodyValueExpression} as ${PATTERN_COLUMN_ALIAS}`, - `${getFirstTimestampValueExpression(config.timestampValueExpression)} as ${TIMESTAMP_COLUMN_ALIAS}`, - ...(levelExpression - ? [`${levelExpression} as ${LEVEL_COLUMN_ALIAS}`] - : []), - ...(serviceNameExpression - ? [`${serviceNameExpression} as ${SERVICE_NAME_COLUMN_ALIAS}`] - : []), - ].join(','), - // TODO: Proper sampling - orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], - limit: { limit: samples }, - }); + const configWithPrimaryAndPartitionKey = useConfigWithAdditionalSelect( + { + ...config, + // TODO: User-configurable pattern columns and non-pattern/group by columns + select: [ + `${bodyValueExpression} as ${PATTERN_COLUMN_ALIAS}`, + `${getFirstTimestampValueExpression(config.timestampValueExpression)} as ${TIMESTAMP_COLUMN_ALIAS}`, + ...(levelExpression + ? [`${levelExpression} as ${LEVEL_COLUMN_ALIAS}`] + : []), + ...(serviceNameExpression + ? [`${serviceNameExpression} as ${SERVICE_NAME_COLUMN_ALIAS}`] + : []), + ].join(','), + // TODO: Proper sampling + orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], + limit: { limit: samples }, + }, + undefined, + { severityNumberSource, severityTextAlias: LEVEL_COLUMN_ALIAS }, + ); const { data: sampleRows, @@ -226,6 +236,7 @@ export function useGroupedPatterns({ bodyValueExpression, levelExpression, serviceNameExpression, + severityNumberSource, totalCount, enabled = true, }: { @@ -234,6 +245,8 @@ export function useGroupedPatterns({ bodyValueExpression: string; levelExpression?: string; serviceNameExpression?: string; + // Adds the source's SeverityNumber to the samples to color pattern levels. + severityNumberSource?: TSource; totalCount?: number; enabled?: boolean; }) { @@ -248,6 +261,7 @@ export function useGroupedPatterns({ bodyValueExpression, levelExpression, serviceNameExpression, + severityNumberSource, enabled, }); @@ -309,6 +323,12 @@ export function useGroupedPatterns({ count, countStr: `~${count}`, level: lastRow?.[LEVEL_COLUMN_ALIAS], // last level is usually representative of the entire pattern set + ...(lastRow != null && INTERNAL_ROW_FIELDS.SEVERITY_NUMBER in lastRow + ? { + [INTERNAL_ROW_FIELDS.SEVERITY_NUMBER]: + lastRow[INTERNAL_ROW_FIELDS.SEVERITY_NUMBER], + } + : {}), samples: rows, __hdx_pattern_trend: { data: Object.entries(bucketCounts).map(([bucket, count]) => ({ diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index ad0f5ede8e..bc0a14575d 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -20,6 +20,7 @@ export type WithClause = z.infer; export const INTERNAL_ROW_FIELDS = { ID: '__hyperdx_id', ALIAS_WITH: '__hyperdx_alias_with', + SEVERITY_NUMBER: '__hyperdx_severity_number', } as const; export const getRowId = (row: Record): string => @@ -168,10 +169,13 @@ export default function useRowWhere({ ); // Memoize the aliasWith array since it only depends on aliasMap - const aliasWith = useMemo( - () => aliasMapToWithClauses(aliasMap) ?? [], - [aliasMap], - ); + const aliasWith = useMemo(() => { + // The severity number only colors the level cell, so it stays out of the + // row lookup. + const { [INTERNAL_ROW_FIELDS.SEVERITY_NUMBER]: _severityNumber, ...rest } = + aliasMap ?? {}; + return aliasMapToWithClauses(rest) ?? []; + }, [aliasMap]); return useCallback( (row: Record): RowWhereResult => { @@ -179,6 +183,7 @@ export default function useRowWhere({ const { [INTERNAL_ROW_FIELDS.ID]: _id, [INTERNAL_ROW_FIELDS.ALIAS_WITH]: _aliasWith, + [INTERNAL_ROW_FIELDS.SEVERITY_NUMBER]: _severityNumber, ...dbRow } = row; diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index b8e0e91ea3..1d5212dad7 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -8,6 +8,10 @@ import { TableConnection, unquoteIdentifier, } from '@hyperdx/common-utils/dist/core/metadata'; +import { + collapseSeverityClass, + getSeverityClass, +} from '@hyperdx/common-utils/dist/core/severity'; import { CATEGORICAL_PALETTE_TOKENS, ChartPaletteToken, @@ -328,6 +332,9 @@ export function formatmmss(milliseconds?: number) { return `${minutes}:${seconds < 10 ? '0' : ''}${seconds}`; } +// Separate from getSeverityClass because it also infers the level column +// (possibly a trace StatusCode) and colors chart series. A test pins the +// shared prefixes. export const getLogLevelClass = (lvl: string | undefined) => { const level = lvl?.toLowerCase(); if (level == null) { @@ -353,25 +360,17 @@ export const getLogLevelClass = (lvl: string | undefined) => { : undefined; }; -/** - * Error or warn level of a log row, or undefined when it is neither. - * SeverityNumber is normalized by the SDK, so it wins when set; 0 means - * unspecified, which falls back to matching SeverityText. - * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.59.0/specification/logs/data-model.md#field-severitynumber - */ +/** Error or warn level of a log row, or undefined when it is neither. */ export function getLogSeverityLevel({ severityNumber, severityText, }: { severityNumber?: unknown; - severityText?: string; + severityText?: string | null; }): 'error' | 'warn' | undefined { - const number = Number(severityNumber); - if (number > 0) { - return number >= 17 ? 'error' : number >= 13 ? 'warn' : undefined; - } - const level = getLogLevelClass(severityText); - return level === 'error' || level === 'warn' ? level : undefined; + return collapseSeverityClass( + getSeverityClass({ severityNumber, severityText }), + ); } /** diff --git a/packages/common-utils/src/core/__tests__/severity.test.ts b/packages/common-utils/src/core/__tests__/severity.test.ts new file mode 100644 index 0000000000..381bf5ac49 --- /dev/null +++ b/packages/common-utils/src/core/__tests__/severity.test.ts @@ -0,0 +1,135 @@ +import { + collapseSeverityClass, + getSeverityClass, + getSeverityLabel, + getSeverityShortName, + NUMBER_RANGES, + TEXT_PREFIX_RULES, +} from '@/core/severity'; + +describe('getSeverityClass', () => { + it.each([ + [1, 'TRACE'], + [4, 'TRACE'], + [5, 'DEBUG'], + [8, 'DEBUG'], + [9, 'INFO'], + [12, 'INFO'], + [13, 'WARN'], + [16, 'WARN'], + [17, 'ERROR'], + [20, 'ERROR'], + [21, 'FATAL'], + [24, 'FATAL'], + ['17', 'ERROR'], + ['+17', 'ERROR'], + ['017', 'ERROR'], + ])('classifies number %p as %p', (severityNumber, expected) => { + expect(getSeverityClass({ severityNumber })).toBe(expected); + }); + + it('prefers the number over the text', () => { + expect(getSeverityClass({ severityNumber: 9, severityText: 'ERROR' })).toBe( + 'INFO', + ); + }); + + it.each([0, 25, 255, -1, 17.5, ' 17', '17 ', '0x11', '', null, undefined])( + 'falls back to the text for number %p', + severityNumber => { + expect(getSeverityClass({ severityNumber, severityText: 'Warn' })).toBe( + 'WARN', + ); + }, + ); + + it.each(TEXT_PREFIX_RULES.map(r => [r.prefix, r.severityClass]))( + 'matches text prefix %p in any case as %p', + (prefix, expected) => { + const mixedCase = `${prefix[0].toUpperCase()}${prefix.slice(1)}X`; + expect(getSeverityClass({ severityText: prefix.toUpperCase() })).toBe( + expected, + ); + expect(getSeverityClass({ severityText: mixedCase })).toBe(expected); + }, + ); + + it.each([ + ['panic', 'FATAL'], + ['dpanic', 'FATAL'], + ['ftl', 'FATAL'], + ['severe', 'ERROR'], + ['wrn', 'WARN'], + ])('classifies %p as %p', (severityText, expected) => { + expect(getSeverityClass({ severityText })).toBe(expected); + }); + + it.each(['', ' error', 'E', 'log', 'Normal', null, undefined])( + 'classifies unmatched text %p as unknown', + severityText => { + expect(getSeverityClass({ severityNumber: 0, severityText })).toBe( + 'UNKNOWN', + ); + }, + ); + + it('keeps the ranges contiguous and four wide and the prefixes order-independent', () => { + NUMBER_RANGES.forEach((range, i) => { + expect(range.min).toBe(i * 4 + 1); + expect(range.max).toBe(range.min + 3); + }); + TEXT_PREFIX_RULES.forEach(a => { + expect(a.prefix).toMatch(/^[a-z]+$/); + TEXT_PREFIX_RULES.filter(b => b.prefix.startsWith(a.prefix)).forEach(b => + expect(b.severityClass).toBe(a.severityClass), + ); + }); + }); +}); + +describe('collapseSeverityClass', () => { + it.each([ + ['FATAL', 'error'], + ['ERROR', 'error'], + ['WARN', 'warn'], + ['INFO', undefined], + ['DEBUG', undefined], + ['TRACE', undefined], + ['UNKNOWN', undefined], + ] as const)('collapses %p to %p', (severityClass, expected) => { + expect(collapseSeverityClass(severityClass)).toBe(expected); + }); +}); + +describe('getSeverityShortName', () => { + it.each([ + [1, 'TRACE'], + [4, 'TRACE4'], + [10, 'INFO2'], + [21, 'FATAL'], + [24, 'FATAL4'], + ['10', 'INFO2'], + [0, undefined], + ])('names %p %p', (severityNumber, expected) => { + expect(getSeverityShortName(severityNumber)).toBe(expected); + }); +}); + +describe('getSeverityLabel', () => { + it.each([ + [9, 'info', 'INFO'], + [9, 'Informational', 'INFO (Informational)'], + [10, 'info', 'INFO2 (info)'], + [13, 'warning', 'WARN (warning)'], + [17, 'info', 'ERROR (info)'], + [17, '', 'ERROR'], + [18, null, 'ERROR2'], + [0, 'error', 'error'], + [null, 'Warning', 'Warning'], + [25, 'info', 'info'], + [0, '', undefined], + [undefined, null, undefined], + ])('labels %p with %p as %p', (severityNumber, severityText, expected) => { + expect(getSeverityLabel({ severityNumber, severityText })).toBe(expected); + }); +}); diff --git a/packages/common-utils/src/core/severity.ts b/packages/common-utils/src/core/severity.ts new file mode 100644 index 0000000000..dff0814208 --- /dev/null +++ b/packages/common-utils/src/core/severity.ts @@ -0,0 +1,169 @@ +/** + * OpenTelemetry log severity classes (uppercase, as in the spec), from the + * SeverityNumber with a severity text fallback. Parsing mirrors ClickHouse + * `accurateCastOrNull(n, 'UInt8')` and `lower()` so SQL built from these + * tables classifies rows the same way. + * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.61.0/specification/logs/data-model.md#field-severitynumber + */ + +export const SEVERITY_CLASSES = [ + 'FATAL', + 'ERROR', + 'WARN', + 'INFO', + 'DEBUG', + 'TRACE', + 'UNKNOWN', +] as const; + +export type SeverityClass = (typeof SEVERITY_CLASSES)[number]; + +export const SEVERITY_CLASS_ERROR_VALUES: readonly SeverityClass[] = [ + 'FATAL', + 'ERROR', +]; + +// Must stay contiguous and four wide: getNumberRange indexes by (n - 1) / 4. +export const NUMBER_RANGES: readonly { + min: number; + max: number; + severityClass: SeverityClass; +}[] = [ + { min: 1, max: 4, severityClass: 'TRACE' }, + { min: 5, max: 8, severityClass: 'DEBUG' }, + { min: 9, max: 12, severityClass: 'INFO' }, + { min: 13, max: 16, severityClass: 'WARN' }, + { min: 17, max: 20, severityClass: 'ERROR' }, + { min: 21, max: 24, severityClass: 'FATAL' }, +]; + +// Takes a number already parsed to 1-24. +function getNumberRange(number: number) { + return NUMBER_RANGES[Math.floor((number - 1) / 4)]; +} + +/** + * The spec's short name of a SeverityNumber in 1-24: the first number of each + * range takes the class name and the rest add their position (9 INFO, 10 INFO2). + */ +export function getSeverityShortName( + severityNumber: unknown, +): string | undefined { + const number = parseSeverityNumber(severityNumber); + if (number == null) return undefined; + + const range = getNumberRange(number); + const position = number - range.min + 1; + + return position === 1 + ? range.severityClass + : `${range.severityClass}${position}`; +} + +// Matched against the start of the lowercased text. No prefix starts another +// prefix of a different class, so rule order never changes the result. +export const TEXT_PREFIX_RULES: readonly { + prefix: string; + severityClass: SeverityClass; +}[] = [ + { prefix: 'fatal', severityClass: 'FATAL' }, + { prefix: 'emerg', severityClass: 'FATAL' }, + { prefix: 'panic', severityClass: 'FATAL' }, + { prefix: 'dpanic', severityClass: 'FATAL' }, + { prefix: 'ftl', severityClass: 'FATAL' }, + { prefix: 'err', severityClass: 'ERROR' }, + { prefix: 'crit', severityClass: 'ERROR' }, + { prefix: 'alert', severityClass: 'ERROR' }, + { prefix: 'severe', severityClass: 'ERROR' }, + { prefix: 'warn', severityClass: 'WARN' }, + { prefix: 'wrn', severityClass: 'WARN' }, + { prefix: 'info', severityClass: 'INFO' }, + { prefix: 'inf', severityClass: 'INFO' }, + { prefix: 'notice', severityClass: 'INFO' }, + { prefix: 'ok', severityClass: 'INFO' }, + { prefix: 'unset', severityClass: 'INFO' }, + { prefix: 'debug', severityClass: 'DEBUG' }, + { prefix: 'dbg', severityClass: 'DEBUG' }, + { prefix: 'trace', severityClass: 'TRACE' }, + { prefix: 'trc', severityClass: 'TRACE' }, + { prefix: 'verbose', severityClass: 'TRACE' }, +]; + +/** + * The SeverityNumber as an integer in 1-24, or undefined. ClickHouse quotes + * 64-bit integers in JSON, so digit strings count too; they are not trimmed + * because accurateCastOrNull rejects surrounding spaces. + */ +export function parseSeverityNumber(value: unknown): number | undefined { + const number = + typeof value === 'number' + ? value + : typeof value === 'string' && /^\+?\d+$/.test(value) + ? Number(value) + : undefined; + + return number != null && + Number.isInteger(number) && + number >= 1 && + number <= 24 + ? number + : undefined; +} + +// ClickHouse lower() only folds ASCII. +function lowerAscii(text: string): string { + return text.replace(/[A-Z]/g, c => c.toLowerCase()); +} + +/** + * The severity class of a log row. A SeverityNumber in 1-24 wins because the + * SDK normalizes it; anything else (0 means unspecified) falls back to a + * case-insensitive prefix match on the text. + */ +export function getSeverityClass({ + severityNumber, + severityText, +}: { + severityNumber?: unknown; + severityText?: string | null; +}): SeverityClass { + const number = parseSeverityNumber(severityNumber); + if (number != null) return getNumberRange(number).severityClass; + + const text = lowerAscii(severityText ?? ''); + + return ( + TEXT_PREFIX_RULES.find(r => text.startsWith(r.prefix))?.severityClass ?? + 'UNKNOWN' + ); +} + +/** + * The level label of a log row. A valid SeverityNumber shows its short name, + * followed by the stored text when that says something else, e.g. + * "INFO (Informational)". Without one the stored text shows as is, so rows from + * SDKs that leave the number at 0 look as they always have. + */ +export function getSeverityLabel({ + severityNumber, + severityText, +}: { + severityNumber?: unknown; + severityText?: string | null; +}): string | undefined { + const shortName = getSeverityShortName(severityNumber); + if (shortName == null) return severityText || undefined; + + return severityText && lowerAscii(severityText) !== lowerAscii(shortName) + ? `${shortName} (${severityText})` + : shortName; +} + +/** The error or warn level of a class, or undefined when it is neither. */ +export function collapseSeverityClass( + severityClass: SeverityClass, +): 'error' | 'warn' | undefined { + if (SEVERITY_CLASS_ERROR_VALUES.includes(severityClass)) return 'error'; + + return severityClass === 'WARN' ? 'warn' : undefined; +}