feat: rework getAllKeyValues and add filters populated from text indices

.changeset/sharp-fishes-yell.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/app": minor
+---
+
+feat: upgrade filters and autocomplete to intelligently route queries through the best rollup

docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql

@@ -1,6 +1,6 @@
 -- +goose Up
 
--- Map Attributes Key Value Rollup (all map columns in one table)
+-- Key Value Rollup (many columns in one table)
 CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_kv_rollup_15m
 (
     `Timestamp` DateTime,
@@ -17,58 +17,9 @@ ORDER BY (ColumnIdentifier, Key, Timestamp, Value)
 TTL Timestamp + ${TABLES_TTL}
 SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
 
--- Key Only Rollup (derived from KV rollup)
-CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_key_rollup_15m
-(
-    `Timestamp` DateTime,
-    `ColumnIdentifier` LowCardinality(String),
-    `Key` LowCardinality(String),
-    `count` UInt64,
-    INDEX idx_count_minmax count TYPE minmax GRANULARITY 1,
-    INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1
-)
-ENGINE = SummingMergeTree
-PARTITION BY toDate(Timestamp)
-ORDER BY (ColumnIdentifier, Key, Timestamp)
-TTL Timestamp + ${TABLES_TTL}
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
-
-CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_logs_key_rollup_15m_mv TO ${DATABASE}.otel_logs_key_rollup_15m
-AS SELECT
-    Timestamp,
-    ColumnIdentifier,
-    Key,
-    sum(count) as count
-FROM ${DATABASE}.otel_logs_kv_rollup_15m
-GROUP BY ColumnIdentifier, Key, Timestamp;
-
 -- Single MV: CTE with UNION ALL across all columns, then aggregate
 CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_logs_attr_kv_rollup_15m_mv TO ${DATABASE}.otel_logs_kv_rollup_15m
 AS WITH elements AS (
-    SELECT
-        'ResourceAttributes' AS ColumnIdentifier,
-        toStartOfFifteenMinutes(Timestamp) AS Timestamp,
-        replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
-        CAST(entry.2 AS String) AS Value
-    FROM ${DATABASE}.otel_logs
-    ARRAY JOIN ResourceAttributes AS entry
-    UNION ALL
-    SELECT
-        'LogAttributes' AS ColumnIdentifier,
-        toStartOfFifteenMinutes(Timestamp) AS Timestamp,
-        replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
-        CAST(entry.2 AS String) AS Value
-    FROM ${DATABASE}.otel_logs
-    ARRAY JOIN LogAttributes AS entry
-    UNION ALL
-    SELECT
-        'ScopeAttributes' AS ColumnIdentifier,
-        toStartOfFifteenMinutes(Timestamp) AS Timestamp,
-        replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
-        CAST(entry.2 AS String) AS Value
-    FROM ${DATABASE}.otel_logs
-    ARRAY JOIN ScopeAttributes AS entry
-    UNION ALL
     SELECT
         'NativeColumn' AS ColumnIdentifier,
         toStartOfFifteenMinutes(Timestamp) AS Timestamp,

docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql

@@ -1,6 +1,6 @@
 -- +goose Up
 
--- Key Value Rollup (all columns in one table)
+-- Key Value Rollup (many columns in one table)
 CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m
 (
     `Timestamp` DateTime,
@@ -17,51 +17,9 @@ ORDER BY (ColumnIdentifier, Key, Timestamp, Value)
 TTL Timestamp + ${TABLES_TTL}
 SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
 
--- Key Only Rollup (derived from KV rollup)
-CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_key_rollup_15m
-(
-    `Timestamp` DateTime,
-    `ColumnIdentifier` LowCardinality(String),
-    `Key` LowCardinality(String),
-    `count` UInt64,
-    INDEX idx_count_minmax count TYPE minmax GRANULARITY 1,
-    INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1
-)
-ENGINE = SummingMergeTree
-PARTITION BY toDate(Timestamp)
-ORDER BY (ColumnIdentifier, Key, Timestamp)
-TTL Timestamp + ${TABLES_TTL}
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
-
--- Key only rollup MV
-CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_traces_key_rollup_15m_mv TO ${DATABASE}.otel_traces_key_rollup_15m
-AS SELECT
-    Timestamp,
-    ColumnIdentifier,
-    Key,
-    sum(count) AS count
-FROM ${DATABASE}.otel_traces_kv_rollup_15m
-GROUP BY ColumnIdentifier, Key, Timestamp;
-
 -- Single MV: CTE with UNION ALL across all columns, then aggregate
 CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m_mv TO ${DATABASE}.otel_traces_kv_rollup_15m
 AS WITH elements AS (
-    SELECT
-        'ResourceAttributes' AS ColumnIdentifier,
-        toStartOfFifteenMinutes(Timestamp) AS Timestamp,
-        replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
-        CAST(entry.2 AS String) AS Value
-    FROM ${DATABASE}.otel_traces
-    ARRAY JOIN ResourceAttributes AS entry
-    UNION ALL
-    SELECT
-        'SpanAttributes' AS ColumnIdentifier,
-        toStartOfFifteenMinutes(Timestamp) AS Timestamp,
-        replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
-        CAST(entry.2 AS String) AS Value
-    FROM ${DATABASE}.otel_traces
-    ARRAY JOIN SpanAttributes AS entry
-    UNION ALL
     SELECT
         'NativeColumn' AS ColumnIdentifier,
         toStartOfFifteenMinutes(Timestamp) AS Timestamp,

packages/api/package.json

@@ -51,6 +51,7 @@
     "promised-handlebars": "^2.0.1",
     "protobufjs": "^7.6.3",
     "serialize-error": "^8.1.0",
+    "sqlstring": "^2.3.3",
     "uuid": "^8.3.2",
     "zod": "3.25",
     "zod-express-middleware": "^1.4.0"
@@ -66,6 +67,7 @@
     "@types/minimist": "^1.2.2",
     "@types/ms": "^0.7.31",
     "@types/passport-local": "^1.0.34",
+    "@types/sqlstring": "^2.3.0",
     "@types/supertest": "^2.0.12",
     "@types/swagger-jsdoc": "^6",
     "@types/uuid": "^8.3.4",

packages/api/src/mcp/tools/sources/describeSource.ts

@@ -9,6 +9,7 @@ import {
 import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types';
+import SqlString from 'sqlstring';
 import { z } from 'zod';
 
 import { getConnectionById } from '@/controllers/connection';
@@ -114,7 +115,7 @@ async function sampleMetricNamesForKind({
     timestampValueExpression,
     signal,
   });
-  const names = nameResults[0]?.value ?? [];
+  const names = nameResults[0]?.value.map(v => v.toString()) ?? [];
   if (names.length === 0) return [];
 
   // Best-effort enrichment with unit + description. One small query
@@ -451,7 +452,7 @@ async function describeSourceSchema(
       });
       for (const { key, value } of results) {
         if (value.length > 0) {
-          lowCardinalityValues[key] = value;
+          lowCardinalityValues[key] = value.map(v => v.toString());
         }
       }
     } catch {
@@ -473,7 +474,11 @@ async function describeSourceSchema(
     const keyExprs: string[] = [];
     for (const [colName, keys] of Object.entries(mapKeysResults)) {
       for (const key of keys.slice(0, MAX_MAP_KEYS_TO_SAMPLE)) {
-        keyExprs.push(`${colName}['${key}']`);
+        // Map keys come from ClickHouse data (customer telemetry) so they can
+        // contain arbitrary characters, including single quotes. Escape as a
+        // SQL string literal — `SqlString.escape` returns a fully-quoted,
+        // safely-escaped value — before embedding in the key expression.
+        keyExprs.push(`${colName}[${SqlString.escape(key)}]`);
       }
     }
 
@@ -491,7 +496,7 @@ async function describeSourceSchema(
       });
       for (const { key, value } of results) {
         if (value.length > 0) {
-          mapAttributeValues[key] = value;
+          mapAttributeValues[key] = value.map(v => v.toString());
         }
       }
     } catch {

packages/app/src/components/DBSearchPageFilters.tsx

@@ -1171,7 +1171,7 @@ const DBSearchPageFiltersComponent = ({
 
   const [showMoreFields, setShowMoreFields] = useState(false);
   const {
-    data: facets,
+    data: fetchFacetsData,
     isLoading: isFacetsLoading,
     isFetching: isFacetsFetching,
     error,
@@ -1186,6 +1186,7 @@ const DBSearchPageFiltersComponent = ({
     filterState,
     showMoreFields,
   });
+  const facets = fetchFacetsData.keyValues;
 
   useEffect(() => {
     if (error) {