From 11f50e6ce64a9f9145dc1530cb4632b62595653c Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 14:56:13 -0500
Subject: [PATCH 01/27] fix: drop the row cap from text index value lookups

---
 .changeset/text-index-values-no-row-cap.md    | 11 ++++++++
 .../src/__tests__/metadata.test.ts            | 26 +++++++++++++++++++
 packages/common-utils/src/core/metadata.ts    | 16 +++++-------
 3 files changed, 43 insertions(+), 10 deletions(-)
 create mode 100644 .changeset/text-index-values-no-row-cap.md

diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md
new file mode 100644
index 000000000..103ed81b0
--- /dev/null
+++ b/.changeset/text-index-values-no-row-cap.md
@@ -0,0 +1,11 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+fix: show values for every key in text index value lookups
+
+"All values" lookups that read a text index (Map key=value indexes and text
+indexes on plain columns) stopped at `max_rows_to_read` with
+`read_overflow_mode = 'break'`. The index is read in sorted token order, so once
+a high-cardinality key used up the budget, every key that sorts after it came
+back with no values and no error. These lookups no longer apply a row cap.
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 8ea619374..74cc3a91e 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1403,6 +1403,32 @@ describe('Metadata', () => {
       expect(sql).toContain('GROUP BY column, key');
     });
 
+    it.each([
+      ['map KV', "LogAttributes['requestId']"],
+      ['native', 'TraceId'],
+    ])(
+      'reads the %s text index without a row cap, even when one is configured',
+      async (_, keyExpression) => {
+        setupDefaultLogsSchema();
+        jest
+          .spyOn(metadata, 'getClickHouseSettings')
+          .mockReturnValue({ max_rows_to_read: '1000' });
+
+        await metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: [keyExpression],
+        });
+
+        const calls = (mockClickhouseClient.query as jest.Mock).mock.calls;
+        const textIndexCall = calls.find(([opts]) =>
+          opts.query.includes('mergeTreeTextIndex('),
+        );
+        expect(textIndexCall[0].clickhouse_settings).toMatchObject({
+          max_rows_to_read: '0',
+        });
+      },
+    );
+
     it.each([
       ['map KV', "LogAttributes['requestId']"],
       ['native', 'TraceId'],
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 81a93eeda..51f1825de 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1819,12 +1819,11 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
-              max_rows_to_read: String(
-                this.getClickHouseSettings().max_rows_to_read ??
-                  DEFAULT_METADATA_MAX_ROWS_TO_READ,
-              ),
-              read_overflow_mode: 'break',
               ...this.getClickHouseSettings(),
+              // The dictionary is read in sorted token order, so a row cap
+              // with read_overflow_mode=break silently leaves every key that
+              // sorts after the cutoff with no values.
+              max_rows_to_read: '0',
             },
             abort_signal: signal,
           })
@@ -1902,12 +1901,9 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
-              max_rows_to_read: String(
-                this.getClickHouseSettings().max_rows_to_read ??
-                  DEFAULT_METADATA_MAX_ROWS_TO_READ,
-              ),
-              read_overflow_mode: 'break',
               ...this.getClickHouseSettings(),
+              // See getMapTextIndexKeyValues.
+              max_rows_to_read: '0',
             },
             abort_signal: signal,
           })

From 144ef4d6a59feedeeb7a23b71c24db796660b77a Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 10 Nov 2026 18:02:35 -0500
Subject: [PATCH 02/27] feat: show the most frequent text index values with
 topKWeighted

---
 .changeset/topk-text-index-values.md          | 11 ++++++++
 .../src/__tests__/metadata.int.test.ts        | 27 +++++++++++++++++++
 .../src/__tests__/metadata.test.ts            | 23 +++++++---------
 packages/common-utils/src/core/metadata.ts    |  6 +++--
 4 files changed, 51 insertions(+), 16 deletions(-)
 create mode 100644 .changeset/topk-text-index-values.md

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
new file mode 100644
index 000000000..332e05a55
--- /dev/null
+++ b/.changeset/topk-text-index-values.md
@@ -0,0 +1,11 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+feat: show the most frequent filter values read from text indexes
+
+"All values" lookups that read a text index (Map key=value indexes and text
+indexes on plain columns) used `groupUniqArray(limit)`, which keeps the
+alphabetically first tokens it reads. They now use `topKWeighted(limit)`
+weighted by each token's row count, so a capped list holds the most common
+values, most common first. This costs more CPU per token than before.
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 0e9133bc8..25ff349cf 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -907,6 +907,33 @@ describe('Metadata Integration Tests', () => {
       );
     });
 
+    it('keeps the most frequent text index values when maxValuesPerKey truncates the list', async () => {
+      if (!textIndexSupported) {
+        console.warn(
+          'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
+        );
+        return;
+      }
+      // The index dictionary is sorted, so 'a-rare' is read first.
+      const freqKey = `freq.${tag}`;
+      const timestamp = testTime.toISOString().replace('T', ' ').slice(0, 23);
+      await client.command({
+        query: `INSERT INTO default.otel_logs (Timestamp, ResourceAttributes)
+          SELECT '${timestamp}', map('${freqKey}', if(number = 0, 'a-rare', 'z-common'))
+          FROM numbers(4)`,
+      });
+
+      const result = await metadata.getAllKeyValues({
+        ...commonArgs,
+        keyExpressions: [`ResourceAttributes['${freqKey}']`],
+        maxValuesPerKey: 1,
+      });
+
+      expect(result).toEqual([
+        { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
+      ]);
+    });
+
     it('routes TraceId through getTextIndexKeyValues', async () => {
       if (!textIndexSupported) {
         console.warn(
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 74cc3a91e..e30fa0c10 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1380,7 +1380,7 @@ describe('Metadata', () => {
       const sql = calls[calls.length - 1][0].query as string;
 
       expect(sql).toContain('mergeTreeTextIndex(');
-      expect(sql).toContain('groupUniqArray(');
+      expect(sql).toContain('topKWeighted(');
       expect(sql).toContain('GROUP BY key');
       expect(sql).not.toContain('startsWith(token,');
     });
@@ -1831,9 +1831,8 @@ describe('Metadata', () => {
     // via text index or raw scan.
     describe('maxValuesPerKey threading', () => {
       // Pick a value that (a) isn't ambient in the test setup (like 20,
-      // dates, or granularity numbers) and (b) is easy to spot in a param
-      // dump when a test fails. Int32 chSql params render as numbers, so
-      // assertions compare against the numeric literal.
+      // dates, or granularity numbers) and (b) is easy to spot in the SQL or a
+      // param dump when a test fails.
       const MAX_VALUES = 7777;
 
       it('threads maxValuesPerKey into the map text-index query', async () => {
@@ -1853,11 +1852,9 @@ describe('Metadata', () => {
             c[0].query.includes('startsWith(token,'),
         );
         expect(mapTextIndexCall).toBeDefined();
-        const params = mapTextIndexCall![0].query_params as Record<
-          string,
-          unknown
-        >;
-        expect(Object.values(params)).toContain(MAX_VALUES);
+        expect(mapTextIndexCall![0].query).toContain(
+          `topKWeighted(${MAX_VALUES})(`,
+        );
       });
 
       it('threads maxValuesPerKey into the native text-index query', async () => {
@@ -1878,11 +1875,9 @@ describe('Metadata', () => {
             !c[0].query.includes('startsWith(token,'),
         );
         expect(nativeTextIndexCall).toBeDefined();
-        const params = nativeTextIndexCall![0].query_params as Record<
-          string,
-          unknown
-        >;
-        expect(Object.values(params)).toContain(MAX_VALUES);
+        expect(nativeTextIndexCall![0].query).toContain(
+          `topKWeighted(${MAX_VALUES})(token, cardinality)`,
+        );
       });
 
       it('threads maxValuesPerKey into the raw-table getKeyValues fallback', async () => {
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 51f1825de..7cb721839 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1798,11 +1798,13 @@ export class Metadata {
             partSelection,
           });
           const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`;
+          // `cardinality` is the token's row count in the part, so this keeps
+          // each key's most frequent values rather than the first ones read.
           const sql = chSql`
         SELECT * FROM (
           SELECT ${{ String: columnName }} as column,
             substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
-            groupUniqArray(${{ Int32: info.limit }})(${valueSql}) AS value
+            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND (${orChain})
@@ -1885,7 +1887,7 @@ export class Metadata {
           const sql = chSql`
         SELECT * FROM (
           SELECT ${{ String: columnName }} AS key,
-            groupUniqArray(${{ Int32: info.limit }})(token) AS value
+            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND token != ''

From 3285fe10504c19edbf8c245190834e6e78d0d608 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 10 Nov 2026 21:31:49 -0500
Subject: [PATCH 03/27] perf: read text index values per key with a token cap

---
 .changeset/topk-text-index-values.md          | 16 +++--
 .../src/__tests__/metadata.int.test.ts        | 16 +++--
 .../src/__tests__/metadata.test.ts            | 30 ++++++++-
 packages/common-utils/src/core/metadata.ts    | 66 ++++++++++---------
 4 files changed, 85 insertions(+), 43 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 332e05a55..a25ccb0f7 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,10 +2,16 @@
 '@hyperdx/common-utils': patch
 ---
 
-feat: show the most frequent filter values read from text indexes
+perf: read each key's text index values separately, and show the most
+frequent ones
 
 "All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) used `groupUniqArray(limit)`, which keeps the
-alphabetically first tokens it reads. They now use `topKWeighted(limit)`
-weighted by each token's row count, so a capped list holds the most common
-values, most common first. This costs more CPU per token than before.
+indexes on plain columns) read every token of every requested key, capped at
+`max_rows_to_read` for the whole query. One key with a value per row (request
+IDs, UUIDs) slowed the whole query down, and because the index is read in
+sorted order, the cap left keys that sort after it with no values. Each key now
+has its own query branch that reads at most 100,000 tokens, and there is no
+whole-query row cap. Values come from `topKWeighted(limit)` weighted by each
+token's row count, so a capped list holds the most common values, most common
+first. For keys with more distinct values than the token cap, they are the most
+common among the alphabetically first ones.
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 25ff349cf..82778f25e 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -925,13 +925,21 @@ describe('Metadata Integration Tests', () => {
 
       const result = await metadata.getAllKeyValues({
         ...commonArgs,
-        keyExpressions: [`ResourceAttributes['${freqKey}']`],
+        keyExpressions: [
+          `ResourceAttributes['${freqKey}']`,
+          `ResourceAttributes['${mapKey}']`,
+        ],
         maxValuesPerKey: 1,
       });
 
-      expect(result).toEqual([
-        { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
-      ]);
+      // UNION ALL branches can arrive in any order.
+      expect(result).toHaveLength(2);
+      expect(result).toEqual(
+        expect.arrayContaining([
+          { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
+          { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] },
+        ]),
+      );
     });
 
     it('routes TraceId through getTextIndexKeyValues', async () => {
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e30fa0c10..c66904a2f 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1381,7 +1381,7 @@ describe('Metadata', () => {
 
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('topKWeighted(');
-      expect(sql).toContain('GROUP BY key');
+      expect(sql).toContain('LIMIT 100000');
       expect(sql).not.toContain('startsWith(token,');
     });
 
@@ -1400,7 +1400,33 @@ describe('Metadata', () => {
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('startsWith(token,');
       expect(sql).toContain('substring(token, position(token,');
-      expect(sql).toContain('GROUP BY column, key');
+      expect(sql).toContain("startsWith(token, 'requestId=')");
+    });
+
+    it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => {
+      setupDefaultLogsSchema();
+
+      await metadata.getAllKeyValues({
+        ...baseArgs,
+        keyExpressions: [
+          "LogAttributes['requestId']",
+          "LogAttributes['region']",
+        ],
+      });
+
+      const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find(
+        ([opts]) => (opts.query as string).includes('startsWith(token,'),
+      )![0];
+      const branches = (call.query as string).split('UNION ALL');
+      expect(branches).toHaveLength(2);
+      expect(branches[0]).toContain("startsWith(token, 'requestId=')");
+      expect(branches[0]).toContain("'requestId' AS key");
+      expect(branches[1]).toContain("startsWith(token, 'region=')");
+      for (const branch of branches) {
+        expect(branch).toContain('LIMIT 100000');
+      }
+      expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
+      expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
     });
 
     it.each([
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 7cb721839..783434c53 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -78,6 +78,11 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20;
 // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle.
 export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 
+// Tokens a text index values lookup reads per key. The dictionary is sorted,
+// so a key with more distinct values than this gets the most frequent among
+// its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
+const TEXT_INDEX_TOKENS_PER_KEY = 100_000;
+
 /**
  * The index read could not be scoped to the requested date range, so it was
  * refused in favour of a path that can. Not a failure — an expected property
@@ -1776,19 +1781,6 @@ export class Metadata {
       try {
         const sqlBranches: Array<ChSql> = [];
         for (const [columnName, info] of queryOptions.entries()) {
-          const orChain = concatChSql(
-            ' OR ',
-            // Inline keys as SQL-escaped literals, not bind params: ~100
-            // per-key params exceed the web client's URL param budget and
-            // silently switch the request to a multipart body that proxies
-            // may reject. Keys are ingest-controlled, hence SqlString.escape.
-            info.keys.map(
-              k =>
-                chSql`startsWith(token, ${{
-                  UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`),
-                }})`,
-            ),
-          );
           const partsFilter = await this.partsOverlapFilter({
             databaseName,
             tableName,
@@ -1798,20 +1790,29 @@ export class Metadata {
             partSelection,
           });
           const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`;
-          // `cardinality` is the token's row count in the part, so this keeps
-          // each key's most frequent values rather than the first ones read.
-          const sql = chSql`
-        SELECT * FROM (
-          SELECT ${{ String: columnName }} as column,
-            substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
-            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value
+          // One branch per key, each with its own LIMIT: the read for a key
+          // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a
+          // value per row can't slow down or starve the others. `cardinality`
+          // is the token's row count in the part, so topKWeighted keeps the
+          // most frequent values among the tokens read.
+          for (const k of info.keys) {
+            // Inline keys as SQL-escaped literals, not bind params: ~100
+            // per-key params exceed the web client's URL param budget and
+            // silently switch the request to a multipart body that proxies
+            // may reject. Keys are ingest-controlled, hence SqlString.escape.
+            sqlBranches.push(chSql`
+        SELECT ${{ String: columnName }} AS column,
+          ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key,
+          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value
+        FROM (
+          SELECT ${valueSql} AS v, cardinality
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
-            AND (${orChain})
-            AND ${valueSql} != ''
-          GROUP BY column, key
-        )`;
-          sqlBranches.push(sql);
+            AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }})
+            AND v != ''
+          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
+        )`);
+          }
         }
         const sql = concatChSql(' UNION ALL ', sqlBranches);
 
@@ -1822,9 +1823,8 @@ export class Metadata {
             connectionId,
             clickhouse_settings: {
               ...this.getClickHouseSettings(),
-              // The dictionary is read in sorted token order, so a row cap
-              // with read_overflow_mode=break silently leaves every key that
-              // sorts after the cutoff with no values.
+              // The per-key LIMIT bounds the read. A row cap would stop the
+              // sorted dictionary read partway and leave later keys empty.
               max_rows_to_read: '0',
             },
             abort_signal: signal,
@@ -1884,14 +1884,16 @@ export class Metadata {
             maxParts: maxParts ?? undefined,
             partSelection,
           });
+          // See getMapTextIndexKeyValues for the per-column LIMIT.
           const sql = chSql`
-        SELECT * FROM (
-          SELECT ${{ String: columnName }} AS key,
-            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
+        SELECT ${{ String: columnName }} AS key,
+          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
+        FROM (
+          SELECT token, cardinality
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND token != ''
-          GROUP BY key
+          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
         )`;
           sqlBranches.push(sql);
         }

From a24aa04d062d0096a4e18841442b088dad9b1f3f Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 10 Nov 2026 21:40:01 -0500
Subject: [PATCH 04/27] perf: run text index values in batches of 10 keys

---
 .../src/__tests__/metadata.test.ts            | 18 ++++++++
 packages/common-utils/src/core/metadata.ts    | 41 +++++++++++--------
 2 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index c66904a2f..e42e62de1 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1537,6 +1537,24 @@ describe('Metadata', () => {
       },
     );
 
+    it('splits Map text index values into queries of at most 10 keys', async () => {
+      setupDefaultLogsSchema();
+
+      await metadata.getAllKeyValues({
+        ...baseArgs,
+        keyExpressions: Array.from(
+          { length: 25 },
+          (_, i) => `LogAttributes['key${i}']`,
+        ),
+      });
+
+      const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls
+        .map(([opts]) => opts.query as string)
+        .filter(q => q.includes('startsWith(token,'))
+        .map(q => q.split('UNION ALL').length);
+      expect(branchCounts).toEqual([10, 10, 5]);
+    });
+
     it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => {
       setupDefaultLogsSchema();
 
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 783434c53..90276fe1d 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -82,6 +82,8 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 // so a key with more distinct values than this gets the most frequent among
 // its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
 const TEXT_INDEX_TOKENS_PER_KEY = 100_000;
+// Keys per text index values query; see dispatchKeyValueQueries.
+const TEXT_INDEX_KEYS_PER_QUERY = 10;
 
 /**
  * The index read could not be scoped to the requested date range, so it was
@@ -3060,28 +3062,35 @@ export class Metadata {
 
     // fire all the kv fetch queries
     const promises: Array<Promise<KeyValues[] | undefined>> = [];
-    if (mapTextIndexQueryOptions.size > 0) {
-      promises.push(
-        this.getMapTextIndexKeyValues({
-          databaseName,
-          tableName,
-          connectionId,
-          queryOptions: mapTextIndexQueryOptions,
-          maxParts: mapKeyIndexMaxParts,
-          partSelection: mapKeyIndexPartSelection,
-          dateRange,
-          timestampValueExpression,
-          signal,
-        }),
-      );
+    // Text index lookups run in small parallel batches, so a slow,
+    // high-cardinality key holds up only its own batch's values.
+    for (const [column, options] of mapTextIndexQueryOptions) {
+      for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) {
+        promises.push(
+          this.getMapTextIndexKeyValues({
+            databaseName,
+            tableName,
+            connectionId,
+            queryOptions: new Map([[column, { ...options, keys }]]),
+            maxParts: mapKeyIndexMaxParts,
+            partSelection: mapKeyIndexPartSelection,
+            dateRange,
+            timestampValueExpression,
+            signal,
+          }),
+        );
+      }
     }
-    if (nativeTextIndexQueryOptions.size > 0) {
+    for (const entries of chunk(
+      [...nativeTextIndexQueryOptions],
+      TEXT_INDEX_KEYS_PER_QUERY,
+    )) {
       promises.push(
         this.getTextIndexKeyValues({
           databaseName,
           tableName,
           connectionId,
-          queryOptions: nativeTextIndexQueryOptions,
+          queryOptions: new Map(entries),
           maxParts: mapKeyIndexMaxParts,
           partSelection: mapKeyIndexPartSelection,
           dateRange,

From 42a2eeb54dc8157bd502440c5f0b8a707e6ae767 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 10 Nov 2026 21:48:14 -0500
Subject: [PATCH 05/27] perf: read text index values in 1024-row blocks so
 per-key LIMITs stop early

---
 packages/common-utils/src/__tests__/metadata.test.ts | 5 +++++
 packages/common-utils/src/core/metadata.ts           | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e42e62de1..8b17d275b 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1383,6 +1383,10 @@ describe('Metadata', () => {
       expect(sql).toContain('topKWeighted(');
       expect(sql).toContain('LIMIT 100000');
       expect(sql).not.toContain('startsWith(token,');
+      expect(calls[calls.length - 1][0].clickhouse_settings).toMatchObject({
+        max_rows_to_read: '0',
+        max_block_size: '1024',
+      });
     });
 
     it("routes LogAttributes['requestId'] through getMapTextIndexKeyValues (map KV text index path)", async () => {
@@ -1427,6 +1431,7 @@ describe('Metadata', () => {
       }
       expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
       expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
+      expect(call.clickhouse_settings.max_block_size).toBe('1024');
     });
 
     it.each([
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 90276fe1d..63be98719 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1828,6 +1828,9 @@ export class Metadata {
               // The per-key LIMIT bounds the read. A row cap would stop the
               // sorted dictionary read partway and leave later keys empty.
               max_rows_to_read: '0',
+              // LIMIT stops a part's read only after a whole block, so the
+              // default 65k-row block makes every key read far past its LIMIT.
+              max_block_size: '1024',
             },
             abort_signal: signal,
           })
@@ -1910,6 +1913,7 @@ export class Metadata {
               ...this.getClickHouseSettings(),
               // See getMapTextIndexKeyValues.
               max_rows_to_read: '0',
+              max_block_size: '1024',
             },
             abort_signal: signal,
           })

From 6da196fa562220ff03503693fa2aab7370a3b474 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 10 Nov 2026 22:13:53 -0500
Subject: [PATCH 06/27] Revert "perf: read text index values in 1024-row blocks
 so per-key LIMITs stop early"

This reverts commit 969a5e04b69a11f23a9b5d925df2cd032a01201f.
---
 packages/common-utils/src/__tests__/metadata.test.ts | 5 -----
 packages/common-utils/src/core/metadata.ts           | 4 ----
 2 files changed, 9 deletions(-)

diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 8b17d275b..e42e62de1 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1383,10 +1383,6 @@ describe('Metadata', () => {
       expect(sql).toContain('topKWeighted(');
       expect(sql).toContain('LIMIT 100000');
       expect(sql).not.toContain('startsWith(token,');
-      expect(calls[calls.length - 1][0].clickhouse_settings).toMatchObject({
-        max_rows_to_read: '0',
-        max_block_size: '1024',
-      });
     });
 
     it("routes LogAttributes['requestId'] through getMapTextIndexKeyValues (map KV text index path)", async () => {
@@ -1431,7 +1427,6 @@ describe('Metadata', () => {
       }
       expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
       expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
-      expect(call.clickhouse_settings.max_block_size).toBe('1024');
     });
 
     it.each([
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 63be98719..90276fe1d 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1828,9 +1828,6 @@ export class Metadata {
               // The per-key LIMIT bounds the read. A row cap would stop the
               // sorted dictionary read partway and leave later keys empty.
               max_rows_to_read: '0',
-              // LIMIT stops a part's read only after a whole block, so the
-              // default 65k-row block makes every key read far past its LIMIT.
-              max_block_size: '1024',
             },
             abort_signal: signal,
           })
@@ -1913,7 +1910,6 @@ export class Metadata {
               ...this.getClickHouseSettings(),
               // See getMapTextIndexKeyValues.
               max_rows_to_read: '0',
-              max_block_size: '1024',
             },
             abort_signal: signal,
           })

From 74d693589ce68b7f8d007b9d0f3dfcd766f719c3 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 10 Nov 2026 22:15:23 -0500
Subject: [PATCH 07/27] Revert "perf: read text index values per key with a
 token cap"

This reverts commit e66388dafe66fd7008bcd1863f0104fdc9219874.
---
 .changeset/topk-text-index-values.md          | 18 ++---
 .../src/__tests__/metadata.int.test.ts        | 16 +---
 .../src/__tests__/metadata.test.ts            | 36 ++-------
 packages/common-utils/src/core/metadata.ts    | 75 ++++++++++---------
 4 files changed, 55 insertions(+), 90 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index a25ccb0f7..95f8e695d 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,16 +2,12 @@
 '@hyperdx/common-utils': patch
 ---
 
-perf: read each key's text index values separately, and show the most
-frequent ones
+perf: show the most frequent text index values, in batches of 10 keys
 
 "All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) read every token of every requested key, capped at
-`max_rows_to_read` for the whole query. One key with a value per row (request
-IDs, UUIDs) slowed the whole query down, and because the index is read in
-sorted order, the cap left keys that sort after it with no values. Each key now
-has its own query branch that reads at most 100,000 tokens, and there is no
-whole-query row cap. Values come from `topKWeighted(limit)` weighted by each
-token's row count, so a capped list holds the most common values, most common
-first. For keys with more distinct values than the token cap, they are the most
-common among the alphabetically first ones.
+indexes on plain columns) used `groupUniqArray(limit)`, which keeps the
+alphabetically first tokens it reads. They now use `topKWeighted(limit)`
+weighted by each token's row count, so a capped list holds the most common
+values, most common first. These lookups also run in parallel batches of at
+most 10 keys instead of 100, so a key with very many values delays only its own
+batch.
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 82778f25e..25ff349cf 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -925,21 +925,13 @@ describe('Metadata Integration Tests', () => {
 
       const result = await metadata.getAllKeyValues({
         ...commonArgs,
-        keyExpressions: [
-          `ResourceAttributes['${freqKey}']`,
-          `ResourceAttributes['${mapKey}']`,
-        ],
+        keyExpressions: [`ResourceAttributes['${freqKey}']`],
         maxValuesPerKey: 1,
       });
 
-      // UNION ALL branches can arrive in any order.
-      expect(result).toHaveLength(2);
-      expect(result).toEqual(
-        expect.arrayContaining([
-          { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
-          { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] },
-        ]),
-      );
+      expect(result).toEqual([
+        { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
+      ]);
     });
 
     it('routes TraceId through getTextIndexKeyValues', async () => {
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e42e62de1..bf140ba03 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1381,7 +1381,7 @@ describe('Metadata', () => {
 
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('topKWeighted(');
-      expect(sql).toContain('LIMIT 100000');
+      expect(sql).toContain('GROUP BY key');
       expect(sql).not.toContain('startsWith(token,');
     });
 
@@ -1400,33 +1400,7 @@ describe('Metadata', () => {
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('startsWith(token,');
       expect(sql).toContain('substring(token, position(token,');
-      expect(sql).toContain("startsWith(token, 'requestId=')");
-    });
-
-    it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => {
-      setupDefaultLogsSchema();
-
-      await metadata.getAllKeyValues({
-        ...baseArgs,
-        keyExpressions: [
-          "LogAttributes['requestId']",
-          "LogAttributes['region']",
-        ],
-      });
-
-      const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find(
-        ([opts]) => (opts.query as string).includes('startsWith(token,'),
-      )![0];
-      const branches = (call.query as string).split('UNION ALL');
-      expect(branches).toHaveLength(2);
-      expect(branches[0]).toContain("startsWith(token, 'requestId=')");
-      expect(branches[0]).toContain("'requestId' AS key");
-      expect(branches[1]).toContain("startsWith(token, 'region=')");
-      for (const branch of branches) {
-        expect(branch).toContain('LIMIT 100000');
-      }
-      expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
-      expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
+      expect(sql).toContain('GROUP BY column, key');
     });
 
     it.each([
@@ -1548,11 +1522,11 @@ describe('Metadata', () => {
         ),
       });
 
-      const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls
+      const keysPerQuery = (mockClickhouseClient.query as jest.Mock).mock.calls
         .map(([opts]) => opts.query as string)
         .filter(q => q.includes('startsWith(token,'))
-        .map(q => q.split('UNION ALL').length);
-      expect(branchCounts).toEqual([10, 10, 5]);
+        .map(q => q.split('startsWith(token,').length - 1);
+      expect(keysPerQuery).toEqual([10, 10, 5]);
     });
 
     it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => {
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 90276fe1d..4f30860f1 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -78,10 +78,6 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20;
 // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle.
 export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 
-// Tokens a text index values lookup reads per key. The dictionary is sorted,
-// so a key with more distinct values than this gets the most frequent among
-// its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
-const TEXT_INDEX_TOKENS_PER_KEY = 100_000;
 // Keys per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 
@@ -1783,6 +1779,19 @@ export class Metadata {
       try {
         const sqlBranches: Array<ChSql> = [];
         for (const [columnName, info] of queryOptions.entries()) {
+          const orChain = concatChSql(
+            ' OR ',
+            // Inline keys as SQL-escaped literals, not bind params: ~100
+            // per-key params exceed the web client's URL param budget and
+            // silently switch the request to a multipart body that proxies
+            // may reject. Keys are ingest-controlled, hence SqlString.escape.
+            info.keys.map(
+              k =>
+                chSql`startsWith(token, ${{
+                  UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`),
+                }})`,
+            ),
+          );
           const partsFilter = await this.partsOverlapFilter({
             databaseName,
             tableName,
@@ -1792,29 +1801,20 @@ export class Metadata {
             partSelection,
           });
           const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`;
-          // One branch per key, each with its own LIMIT: the read for a key
-          // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a
-          // value per row can't slow down or starve the others. `cardinality`
-          // is the token's row count in the part, so topKWeighted keeps the
-          // most frequent values among the tokens read.
-          for (const k of info.keys) {
-            // Inline keys as SQL-escaped literals, not bind params: ~100
-            // per-key params exceed the web client's URL param budget and
-            // silently switch the request to a multipart body that proxies
-            // may reject. Keys are ingest-controlled, hence SqlString.escape.
-            sqlBranches.push(chSql`
-        SELECT ${{ String: columnName }} AS column,
-          ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key,
-          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value
-        FROM (
-          SELECT ${valueSql} AS v, cardinality
+          // `cardinality` is the token's row count in the part, so this keeps
+          // each key's most frequent values rather than the first ones read.
+          const sql = chSql`
+        SELECT * FROM (
+          SELECT ${{ String: columnName }} as column,
+            substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
+            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
-            AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }})
-            AND v != ''
-          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
-        )`);
-          }
+            AND (${orChain})
+            AND ${valueSql} != ''
+          GROUP BY column, key
+        )`;
+          sqlBranches.push(sql);
         }
         const sql = concatChSql(' UNION ALL ', sqlBranches);
 
@@ -1824,10 +1824,12 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
+              max_rows_to_read: String(
+                this.getClickHouseSettings().max_rows_to_read ??
+                  DEFAULT_METADATA_MAX_ROWS_TO_READ,
+              ),
+              read_overflow_mode: 'break',
               ...this.getClickHouseSettings(),
-              // The per-key LIMIT bounds the read. A row cap would stop the
-              // sorted dictionary read partway and leave later keys empty.
-              max_rows_to_read: '0',
             },
             abort_signal: signal,
           })
@@ -1886,16 +1888,14 @@ export class Metadata {
             maxParts: maxParts ?? undefined,
             partSelection,
           });
-          // See getMapTextIndexKeyValues for the per-column LIMIT.
           const sql = chSql`
-        SELECT ${{ String: columnName }} AS key,
-          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
-        FROM (
-          SELECT token, cardinality
+        SELECT * FROM (
+          SELECT ${{ String: columnName }} AS key,
+            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND token != ''
-          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
+          GROUP BY key
         )`;
           sqlBranches.push(sql);
         }
@@ -1907,9 +1907,12 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
+              max_rows_to_read: String(
+                this.getClickHouseSettings().max_rows_to_read ??
+                  DEFAULT_METADATA_MAX_ROWS_TO_READ,
+              ),
+              read_overflow_mode: 'break',
               ...this.getClickHouseSettings(),
-              // See getMapTextIndexKeyValues.
-              max_rows_to_read: '0',
             },
             abort_signal: signal,
           })

From 00821207faacfd26603c1bafd261141221b03738 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 08:30:13 -0500
Subject: [PATCH 08/27] Reapply "perf: read text index values per key with a
 token cap"

This reverts commit 6525149b281abf2773a59667dcc69c153dc3cc00.
---
 .changeset/topk-text-index-values.md          | 18 +++--
 .../src/__tests__/metadata.int.test.ts        | 16 +++-
 .../src/__tests__/metadata.test.ts            | 36 +++++++--
 packages/common-utils/src/core/metadata.ts    | 75 +++++++++----------
 4 files changed, 90 insertions(+), 55 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 95f8e695d..a25ccb0f7 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,12 +2,16 @@
 '@hyperdx/common-utils': patch
 ---
 
-perf: show the most frequent text index values, in batches of 10 keys
+perf: read each key's text index values separately, and show the most
+frequent ones
 
 "All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) used `groupUniqArray(limit)`, which keeps the
-alphabetically first tokens it reads. They now use `topKWeighted(limit)`
-weighted by each token's row count, so a capped list holds the most common
-values, most common first. These lookups also run in parallel batches of at
-most 10 keys instead of 100, so a key with very many values delays only its own
-batch.
+indexes on plain columns) read every token of every requested key, capped at
+`max_rows_to_read` for the whole query. One key with a value per row (request
+IDs, UUIDs) slowed the whole query down, and because the index is read in
+sorted order, the cap left keys that sort after it with no values. Each key now
+has its own query branch that reads at most 100,000 tokens, and there is no
+whole-query row cap. Values come from `topKWeighted(limit)` weighted by each
+token's row count, so a capped list holds the most common values, most common
+first. For keys with more distinct values than the token cap, they are the most
+common among the alphabetically first ones.
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 25ff349cf..82778f25e 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -925,13 +925,21 @@ describe('Metadata Integration Tests', () => {
 
       const result = await metadata.getAllKeyValues({
         ...commonArgs,
-        keyExpressions: [`ResourceAttributes['${freqKey}']`],
+        keyExpressions: [
+          `ResourceAttributes['${freqKey}']`,
+          `ResourceAttributes['${mapKey}']`,
+        ],
         maxValuesPerKey: 1,
       });
 
-      expect(result).toEqual([
-        { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
-      ]);
+      // UNION ALL branches can arrive in any order.
+      expect(result).toHaveLength(2);
+      expect(result).toEqual(
+        expect.arrayContaining([
+          { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
+          { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] },
+        ]),
+      );
     });
 
     it('routes TraceId through getTextIndexKeyValues', async () => {
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index bf140ba03..e42e62de1 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1381,7 +1381,7 @@ describe('Metadata', () => {
 
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('topKWeighted(');
-      expect(sql).toContain('GROUP BY key');
+      expect(sql).toContain('LIMIT 100000');
       expect(sql).not.toContain('startsWith(token,');
     });
 
@@ -1400,7 +1400,33 @@ describe('Metadata', () => {
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('startsWith(token,');
       expect(sql).toContain('substring(token, position(token,');
-      expect(sql).toContain('GROUP BY column, key');
+      expect(sql).toContain("startsWith(token, 'requestId=')");
+    });
+
+    it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => {
+      setupDefaultLogsSchema();
+
+      await metadata.getAllKeyValues({
+        ...baseArgs,
+        keyExpressions: [
+          "LogAttributes['requestId']",
+          "LogAttributes['region']",
+        ],
+      });
+
+      const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find(
+        ([opts]) => (opts.query as string).includes('startsWith(token,'),
+      )![0];
+      const branches = (call.query as string).split('UNION ALL');
+      expect(branches).toHaveLength(2);
+      expect(branches[0]).toContain("startsWith(token, 'requestId=')");
+      expect(branches[0]).toContain("'requestId' AS key");
+      expect(branches[1]).toContain("startsWith(token, 'region=')");
+      for (const branch of branches) {
+        expect(branch).toContain('LIMIT 100000');
+      }
+      expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
+      expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
     });
 
     it.each([
@@ -1522,11 +1548,11 @@ describe('Metadata', () => {
         ),
       });
 
-      const keysPerQuery = (mockClickhouseClient.query as jest.Mock).mock.calls
+      const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls
         .map(([opts]) => opts.query as string)
         .filter(q => q.includes('startsWith(token,'))
-        .map(q => q.split('startsWith(token,').length - 1);
-      expect(keysPerQuery).toEqual([10, 10, 5]);
+        .map(q => q.split('UNION ALL').length);
+      expect(branchCounts).toEqual([10, 10, 5]);
     });
 
     it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => {
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 4f30860f1..90276fe1d 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -78,6 +78,10 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20;
 // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle.
 export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 
+// Tokens a text index values lookup reads per key. The dictionary is sorted,
+// so a key with more distinct values than this gets the most frequent among
+// its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
+const TEXT_INDEX_TOKENS_PER_KEY = 100_000;
 // Keys per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 
@@ -1779,19 +1783,6 @@ export class Metadata {
       try {
         const sqlBranches: Array<ChSql> = [];
         for (const [columnName, info] of queryOptions.entries()) {
-          const orChain = concatChSql(
-            ' OR ',
-            // Inline keys as SQL-escaped literals, not bind params: ~100
-            // per-key params exceed the web client's URL param budget and
-            // silently switch the request to a multipart body that proxies
-            // may reject. Keys are ingest-controlled, hence SqlString.escape.
-            info.keys.map(
-              k =>
-                chSql`startsWith(token, ${{
-                  UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`),
-                }})`,
-            ),
-          );
           const partsFilter = await this.partsOverlapFilter({
             databaseName,
             tableName,
@@ -1801,20 +1792,29 @@ export class Metadata {
             partSelection,
           });
           const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`;
-          // `cardinality` is the token's row count in the part, so this keeps
-          // each key's most frequent values rather than the first ones read.
-          const sql = chSql`
-        SELECT * FROM (
-          SELECT ${{ String: columnName }} as column,
-            substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
-            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value
+          // One branch per key, each with its own LIMIT: the read for a key
+          // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a
+          // value per row can't slow down or starve the others. `cardinality`
+          // is the token's row count in the part, so topKWeighted keeps the
+          // most frequent values among the tokens read.
+          for (const k of info.keys) {
+            // Inline keys as SQL-escaped literals, not bind params: ~100
+            // per-key params exceed the web client's URL param budget and
+            // silently switch the request to a multipart body that proxies
+            // may reject. Keys are ingest-controlled, hence SqlString.escape.
+            sqlBranches.push(chSql`
+        SELECT ${{ String: columnName }} AS column,
+          ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key,
+          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value
+        FROM (
+          SELECT ${valueSql} AS v, cardinality
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
-            AND (${orChain})
-            AND ${valueSql} != ''
-          GROUP BY column, key
-        )`;
-          sqlBranches.push(sql);
+            AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }})
+            AND v != ''
+          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
+        )`);
+          }
         }
         const sql = concatChSql(' UNION ALL ', sqlBranches);
 
@@ -1824,12 +1824,10 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
-              max_rows_to_read: String(
-                this.getClickHouseSettings().max_rows_to_read ??
-                  DEFAULT_METADATA_MAX_ROWS_TO_READ,
-              ),
-              read_overflow_mode: 'break',
               ...this.getClickHouseSettings(),
+              // The per-key LIMIT bounds the read. A row cap would stop the
+              // sorted dictionary read partway and leave later keys empty.
+              max_rows_to_read: '0',
             },
             abort_signal: signal,
           })
@@ -1888,14 +1886,16 @@ export class Metadata {
             maxParts: maxParts ?? undefined,
             partSelection,
           });
+          // See getMapTextIndexKeyValues for the per-column LIMIT.
           const sql = chSql`
-        SELECT * FROM (
-          SELECT ${{ String: columnName }} AS key,
-            topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
+        SELECT ${{ String: columnName }} AS key,
+          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value
+        FROM (
+          SELECT token, cardinality
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND token != ''
-          GROUP BY key
+          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
         )`;
           sqlBranches.push(sql);
         }
@@ -1907,12 +1907,9 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
-              max_rows_to_read: String(
-                this.getClickHouseSettings().max_rows_to_read ??
-                  DEFAULT_METADATA_MAX_ROWS_TO_READ,
-              ),
-              read_overflow_mode: 'break',
               ...this.getClickHouseSettings(),
+              // See getMapTextIndexKeyValues.
+              max_rows_to_read: '0',
             },
             abort_signal: signal,
           })

From f297737c26f3c781313e922a0b0d32743ee2a91c Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 08:31:13 -0500
Subject: [PATCH 09/27] docs: describe per-key text index value batches in the
 changeset

---
 .changeset/topk-text-index-values.md | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index a25ccb0f7..897e03a41 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,16 +2,19 @@
 '@hyperdx/common-utils': patch
 ---
 
-perf: read each key's text index values separately, and show the most
-frequent ones
+perf: read text index values per key, in batches of 10 keys, most frequent
+first
 
 "All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) read every token of every requested key, capped at
-`max_rows_to_read` for the whole query. One key with a value per row (request
-IDs, UUIDs) slowed the whole query down, and because the index is read in
-sorted order, the cap left keys that sort after it with no values. Each key now
-has its own query branch that reads at most 100,000 tokens, and there is no
-whole-query row cap. Values come from `topKWeighted(limit)` weighted by each
-token's row count, so a capped list holds the most common values, most common
-first. For keys with more distinct values than the token cap, they are the most
-common among the alphabetically first ones.
+indexes on plain columns) read every token of up to 100 keys in one query,
+capped at `max_rows_to_read` for the whole query. One key with a value per row
+(request IDs, UUIDs) slowed every other key down, and because the index is read
+in sorted order, the cap left keys that sort after it with no values. These
+lookups now run in parallel queries of at most 10 keys, each key in its own
+branch that reads at most 100,000 tokens, with no whole-query row cap. Values
+come from `topKWeighted(limit)` weighted by each token's row count, so a capped
+list holds the most common values, most common first.
+
+Each branch reads every part's text index once, so this is fastest when the
+index uses large dictionary blocks (e.g. `text_index_dictionary_block_size =
+4096`).

From 93a1c9468b8b33ea1ace808b06ad74456409c947 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 08:36:15 -0500
Subject: [PATCH 10/27] perf: use 4096-token dictionary blocks for key=value
 text indexes in seed schemas

---
 .changeset/text-index-dictionary-block-size.md    | 15 +++++++++++++++
 .../schema/seed/00002_otel_logs.sql               |  9 ++++++---
 .../schema/seed/00005_otel_traces.sql             |  7 +++++--
 3 files changed, 26 insertions(+), 5 deletions(-)
 create mode 100644 .changeset/text-index-dictionary-block-size.md

diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md
new file mode 100644
index 000000000..9801a1ac6
--- /dev/null
+++ b/.changeset/text-index-dictionary-block-size.md
@@ -0,0 +1,15 @@
+---
+'@hyperdx/otel-collector': patch
+---
+
+perf: use 4,096-token dictionary blocks for the key=value text indexes in the
+logs and traces seed schemas
+
+`idx_res_attr_items`, `idx_scope_attr_items` and `idx_log_attr_items` on
+`otel_logs`, and `idx_res_attr_items` and `idx_span_attr_items` on
+`otel_traces`, now set `dictionary_block_size = 4096` (the default is 512).
+Each part's sparse index is about 8x smaller, which cuts the per-part cost of
+reading filter values from these indexes about 10x on large parts. Lookups in
+these indexes decode up to 8x more tokens per block. Only new installs get the
+new schema; existing tables keep their indexes until they are recreated, or
+the indexes are cleared and materialized again.
diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql
index 12ec3fbe3..2872a5d89 100644
--- a/docker/otel-collector/schema/seed/00002_otel_logs.sql
+++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql
@@ -32,11 +32,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
   INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
+  -- Larger dictionary blocks make each part's sparse index smaller. Reading
+  -- a key's values with mergeTreeTextIndex() loads that index per part on
+  -- every read, so this cuts the per-part cost about 10x on large parts.
+  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
   INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'),
+  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
   INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'),
+  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
   INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
 )
 ENGINE = MergeTree
diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql
index b3ad5b3a9..ba3fc5fad 100644
--- a/docker/otel-collector/schema/seed/00005_otel_traces.sql
+++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql
@@ -30,9 +30,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
     INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
     INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
+    -- Larger dictionary blocks make each part's sparse index smaller. Reading
+    -- a key's values with mergeTreeTextIndex() loads that index per part on
+    -- every read, so this cuts the per-part cost about 10x on large parts.
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'),
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
     INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName))
 )

From 07b08f4b54c40cf7e30417ae56aec2a328fcb528 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 11:50:41 -0500
Subject: [PATCH 11/27] perf: read at most 20,000 text index tokens per key

---
 .changeset/topk-text-index-values.md                 | 2 +-
 packages/common-utils/src/__tests__/metadata.test.ts | 4 ++--
 packages/common-utils/src/core/metadata.ts           | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 897e03a41..273b3af25 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -11,7 +11,7 @@ capped at `max_rows_to_read` for the whole query. One key with a value per row
 (request IDs, UUIDs) slowed every other key down, and because the index is read
 in sorted order, the cap left keys that sort after it with no values. These
 lookups now run in parallel queries of at most 10 keys, each key in its own
-branch that reads at most 100,000 tokens, with no whole-query row cap. Values
+branch that reads at most 20,000 tokens, with no whole-query row cap. Values
 come from `topKWeighted(limit)` weighted by each token's row count, so a capped
 list holds the most common values, most common first.
 
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e42e62de1..f1ddc782e 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1381,7 +1381,7 @@ describe('Metadata', () => {
 
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('topKWeighted(');
-      expect(sql).toContain('LIMIT 100000');
+      expect(sql).toContain('LIMIT 20000');
       expect(sql).not.toContain('startsWith(token,');
     });
 
@@ -1423,7 +1423,7 @@ describe('Metadata', () => {
       expect(branches[0]).toContain("'requestId' AS key");
       expect(branches[1]).toContain("startsWith(token, 'region=')");
       for (const branch of branches) {
-        expect(branch).toContain('LIMIT 100000');
+        expect(branch).toContain('LIMIT 20000');
       }
       expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
       expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 90276fe1d..7675c6ee0 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -81,7 +81,7 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 // Tokens a text index values lookup reads per key. The dictionary is sorted,
 // so a key with more distinct values than this gets the most frequent among
 // its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
-const TEXT_INDEX_TOKENS_PER_KEY = 100_000;
+const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
 // Keys per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 

From c2df2c1951abb39ca028af48927d4f16e2ebb8bd Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 12:10:44 -0500
Subject: [PATCH 12/27] perf: use 8192-token dictionary blocks for every text
 index

---
 .../text-index-dictionary-block-size.md       | 21 +++++++++-------
 .changeset/topk-text-index-values.md          |  4 ++--
 docker/clickhouse/local/init-db-e2e.sh        | 20 ++++++++--------
 .../schema/seed/00002_otel_logs.sql           | 24 +++++++++----------
 .../schema/seed/00004_hyperdx_sessions.sql    |  7 ++++--
 .../schema/seed/00005_otel_traces.sql         | 20 ++++++++--------
 .../src/__tests__/kvItemsRewrite.int.test.ts  |  2 +-
 .../src/__tests__/metadata.int.test.ts        |  8 +++----
 8 files changed, 56 insertions(+), 50 deletions(-)

diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md
index 9801a1ac6..da6a7e61b 100644
--- a/.changeset/text-index-dictionary-block-size.md
+++ b/.changeset/text-index-dictionary-block-size.md
@@ -2,14 +2,17 @@
 '@hyperdx/otel-collector': patch
 ---
 
-perf: use 4,096-token dictionary blocks for the key=value text indexes in the
-logs and traces seed schemas
+perf: use 8,192-token dictionary blocks for every text index in the seed
+schemas
 
-`idx_res_attr_items`, `idx_scope_attr_items` and `idx_log_attr_items` on
-`otel_logs`, and `idx_res_attr_items` and `idx_span_attr_items` on
-`otel_traces`, now set `dictionary_block_size = 4096` (the default is 512).
-Each part's sparse index is about 8x smaller, which cuts the per-part cost of
-reading filter values from these indexes about 10x on large parts. Lookups in
-these indexes decode up to 8x more tokens per block. Only new installs get the
-new schema; existing tables keep their indexes until they are recreated, or
+Every text index on `otel_logs`, `otel_traces` and `hyperdx_sessions` now sets
+`dictionary_block_size = 8192` (the default is 512). Each part's sparse index
+is 16x smaller. Reading filter values with `mergeTreeTextIndex()` loads that
+index for every part on every read, and on a 537M-row table 4,096-token blocks
+already made those reads about 10x cheaper per part, made searches for a rare
+attribute value 3-7x faster, and made searches for a common value 10-30%
+faster. The setting is a per-index argument rather than the table setting
+`text_index_dictionary_block_size`, which ClickHouse only accepts from 26.6.
+
+Only new installs get the new schema. Existing tables keep their indexes until
 the indexes are cleared and materialized again.
diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 273b3af25..1628bebb5 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -16,5 +16,5 @@ come from `topKWeighted(limit)` weighted by each token's row count, so a capped
 list holds the most common values, most common first.
 
 Each branch reads every part's text index once, so this is fastest when the
-index uses large dictionary blocks (e.g. `text_index_dictionary_block_size =
-4096`).
+index uses large dictionary blocks (e.g. `dictionary_block_size = 8192`, as the
+seed schemas now do).
diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh
index c5242dc59..4ce993493 100755
--- a/docker/clickhouse/local/init-db-e2e.sh
+++ b/docker/clickhouse/local/init-db-e2e.sh
@@ -40,14 +40,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs
   \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)),
   \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)),
   \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)),
-  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
+  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192)
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
@@ -118,10 +118,10 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces
     INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1,
     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
     INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
 )
diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql
index 2872a5d89..fc046fdcb 100644
--- a/docker/otel-collector/schema/seed/00002_otel_logs.sql
+++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql
@@ -29,18 +29,18 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))),
   `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))),
   `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))),
-  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
-  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  -- Larger dictionary blocks make each part's sparse index smaller. Reading
-  -- a key's values with mergeTreeTextIndex() loads that index per part on
-  -- every read, so this cuts the per-part cost about 10x on large parts.
-  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
-  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
-  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
-  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
+  -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
+  -- part's sparse index is 16x smaller, which makes reading filter values
+  -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
+  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192)
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
index e14258ce0..9f106e9fe 100644
--- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
+++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
@@ -20,8 +20,11 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
   `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)),
   `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)),
   INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
-  INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
+  -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
+  -- part's sparse index is 16x smaller, which makes reading filter values
+  -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
+  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql
index ba3fc5fad..17d040434 100644
--- a/docker/otel-collector/schema/seed/00005_otel_traces.sql
+++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql
@@ -27,17 +27,17 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
     `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(3)),
     `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))),
     `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))),
-    INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
-    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
-    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-    -- Larger dictionary blocks make each part's sparse index smaller. Reading
-    -- a key's values with mergeTreeTextIndex() loads that index per part on
-    -- every read, so this cuts the per-part cost about 10x on large parts.
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
-    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'),
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096),
+    -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
+    -- part's sparse index is 16x smaller, which makes reading filter values
+    -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
+    INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
-    INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName))
+    INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName), dictionary_block_size = 8192)
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
diff --git a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
index 3748c8bd2..7a59955b6 100644
--- a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
+++ b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
@@ -70,7 +70,7 @@ describe('KV items rewrite integration', () => {
         LogAttributeItems Array(String) MATERIALIZED
           arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
           CODEC(ZSTD(1)),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
       )
       ENGINE = MergeTree
       ORDER BY (Timestamp)`,
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 82778f25e..915703f1c 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -697,7 +697,7 @@ describe('Metadata Integration Tests', () => {
             ResourceAttributeItems Array(String) MATERIALIZED
               arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))
               CODEC(ZSTD(1)),
-            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
@@ -712,7 +712,7 @@ describe('Metadata Integration Tests', () => {
             LogAttributeItems Array(String) MATERIALIZED
               arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
               CODEC(ZSTD(1)),
-            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
@@ -1657,14 +1657,14 @@ describe('Metadata Integration Tests', () => {
     {
       path: 'key',
       tableName: 'test_map_keys_key_text_index_order',
-      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`,
+      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
     {
       path: 'kv',
       tableName: 'test_map_keys_kv_text_index_order',
       indexDefinition: `LogAttributeItems Array(String) MATERIALIZED
           arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`,
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
   ])(
     'getMapKeys - $path text index ordering',

From f7f37d95c30278f9596fc450aa6def14d064a614 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:55:51 -0500
Subject: [PATCH 13/27] perf: read Map text index values per key only when
 dictionary blocks are large

---
 .changeset/topk-text-index-values.md          | 28 +++---
 packages/app/src/components/DBRowTable.tsx    | 13 +--
 .../src/__tests__/metadata.test.ts            | 78 +++++++++++++----
 .../common-utils/src/__tests__/utils.test.ts  | 50 +++++++++++
 packages/common-utils/src/core/metadata.ts    | 86 ++++++++++++++++---
 packages/common-utils/src/core/utils.ts       | 35 ++++++++
 6 files changed, 240 insertions(+), 50 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 1628bebb5..f5dd4380c 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,19 +2,21 @@
 '@hyperdx/common-utils': patch
 ---
 
-perf: read text index values per key, in batches of 10 keys, most frequent
-first
+perf: read text index values in batches of 10 keys, most frequent first
 
 "All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) read every token of up to 100 keys in one query,
-capped at `max_rows_to_read` for the whole query. One key with a value per row
-(request IDs, UUIDs) slowed every other key down, and because the index is read
-in sorted order, the cap left keys that sort after it with no values. These
-lookups now run in parallel queries of at most 10 keys, each key in its own
-branch that reads at most 20,000 tokens, with no whole-query row cap. Values
-come from `topKWeighted(limit)` weighted by each token's row count, so a capped
-list holds the most common values, most common first.
+indexes on plain columns) read every token of up to 100 keys in one query, so
+one key with a value per row (request IDs, UUIDs) slowed every other key down.
+These lookups now run in parallel queries of at most 10 keys, and keep at most
+20,000 tokens per key. Values come from `topKWeighted(limit)` weighted by each
+token's row count, so a capped list holds the most common values, most common
+first.
 
-Each branch reads every part's text index once, so this is fastest when the
-index uses large dictionary blocks (e.g. `dictionary_block_size = 8192`, as the
-seed schemas now do).
+Map key=value indexes with large dictionary blocks (4096 or more, e.g.
+`dictionary_block_size = 8192` as the seed schemas now use) read each key
+separately, so a high-cardinality key stops after its 20,000 tokens. Indexes with
+smaller blocks, including ClickHouse's default of 512, read all keys of a batch
+in one pass: every read loads each part's sparse index, which is large with small
+blocks, so one read per key would multiply that cost. The block size comes from
+the index definition, then the table's `text_index_dictionary_block_size`, then
+the server's.
diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx
index 2c90cbf20..a7a251ccc 100644
--- a/packages/app/src/components/DBRowTable.tsx
+++ b/packages/app/src/components/DBRowTable.tsx
@@ -28,7 +28,10 @@ import {
   isJSDataTypeJSONStringifiable,
   JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
+import {
+  getEngineSetting,
+  splitAndTrimWithBracket,
+} from '@hyperdx/common-utils/dist/core/utils';
 import {
   BuilderChartConfigWithDateRange,
   SelectList,
@@ -1541,14 +1544,6 @@ const BLOCK_COLUMN_SETTINGS = [
   { column: '_block_offset', setting: 'enable_block_offset_column' },
 ];
 
-// engine_full ends with the SETTINGS clause.
-function getEngineSetting(engineFull: string, name: string) {
-  const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? '';
-  return splitAndTrimWithBracket(settingsClause)
-    .map(setting => setting.split(' = '))
-    .find(([key]) => key === name)?.[1];
-}
-
 function omitsBlockColumnSetting(engineFull: string) {
   return BLOCK_COLUMN_SETTINGS.some(
     ({ setting }) => getEngineSetting(engineFull, setting) == null,
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index f1ddc782e..e46b78ac2 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1403,8 +1403,20 @@ describe('Metadata', () => {
       expect(sql).toContain("startsWith(token, 'requestId=')");
     });
 
-    it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => {
+    const withDictionaryBlockSize = (size: number) =>
+      jest.spyOn(metadata, 'getSkipIndices').mockResolvedValue([
+        {
+          name: 'idx_log_attr_items',
+          type: 'text',
+          typeFull: `text(tokenizer = 'array', dictionary_block_size = ${size})`,
+          expression: 'LogAttributes',
+          granularity: 100000000,
+        },
+      ]);
+
+    it('reads each Map key in its own capped branch when the index has large dictionary blocks', async () => {
       setupDefaultLogsSchema();
+      withDictionaryBlockSize(8192);
 
       await metadata.getAllKeyValues({
         ...baseArgs,
@@ -1429,6 +1441,31 @@ describe('Metadata', () => {
       expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
     });
 
+    it('reads all Map keys in one pass when the index has small dictionary blocks', async () => {
+      setupDefaultLogsSchema();
+      withDictionaryBlockSize(512);
+
+      await metadata.getAllKeyValues({
+        ...baseArgs,
+        keyExpressions: [
+          "LogAttributes['requestId']",
+          "LogAttributes['region']",
+        ],
+      });
+
+      const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find(
+        ([opts]) => (opts.query as string).includes('startsWith(token,'),
+      )![0];
+      const sql = call.query as string;
+      expect(sql).not.toContain('UNION ALL');
+      expect(sql).toContain(
+        "startsWith(token, 'requestId=') OR startsWith(token, 'region=')",
+      );
+      expect(sql).toContain('LIMIT 20000 BY key');
+      expect(sql).toContain('GROUP BY key');
+      expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
+    });
+
     it.each([
       ['map KV', "LogAttributes['requestId']"],
       ['native', 'TraceId'],
@@ -1537,23 +1574,32 @@ describe('Metadata', () => {
       },
     );
 
-    it('splits Map text index values into queries of at most 10 keys', async () => {
-      setupDefaultLogsSchema();
+    it.each([
+      [8192, [10, 10, 5]],
+      [512, [1, 1, 1]],
+    ])(
+      'splits Map text index values into queries of at most 10 keys (dictionary blocks of %d)',
+      async (blockSize, expectedBranchCounts) => {
+        setupDefaultLogsSchema();
+        withDictionaryBlockSize(blockSize);
 
-      await metadata.getAllKeyValues({
-        ...baseArgs,
-        keyExpressions: Array.from(
-          { length: 25 },
-          (_, i) => `LogAttributes['key${i}']`,
-        ),
-      });
+        await metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: Array.from(
+            { length: 25 },
+            (_, i) => `LogAttributes['key${i}']`,
+          ),
+        });
 
-      const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls
-        .map(([opts]) => opts.query as string)
-        .filter(q => q.includes('startsWith(token,'))
-        .map(q => q.split('UNION ALL').length);
-      expect(branchCounts).toEqual([10, 10, 5]);
-    });
+        const branchCounts = (
+          mockClickhouseClient.query as jest.Mock
+        ).mock.calls
+          .map(([opts]) => opts.query as string)
+          .filter(q => q.includes('startsWith(token,'))
+          .map(q => q.split('UNION ALL').length);
+        expect(branchCounts).toEqual(expectedBranchCounts);
+      },
+    );
 
     it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => {
       setupDefaultLogsSchema();
diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts
index 327585152..452e3f589 100644
--- a/packages/common-utils/src/__tests__/utils.test.ts
+++ b/packages/common-utils/src/__tests__/utils.test.ts
@@ -15,6 +15,7 @@ import {
   getDistributedTableArgs,
   getFirstOrderingItem,
   getPercentileAggFnLevel,
+  getTextIndexDictionaryBlockSize,
   hasNonEmptyOrderBy,
   hasPositiveSeriesLimit,
   isFirstOrderByAscending,
@@ -3620,4 +3621,53 @@ describe('utils', () => {
       expect(resolvePercentileAggFn(select)).toBe(select);
     });
   });
+
+  describe('getTextIndexDictionaryBlockSize', () => {
+    const engineFull =
+      'MergeTree ORDER BY Timestamp SETTINGS index_granularity = 8192, text_index_dictionary_block_size = 4096';
+
+    it.each([
+      [
+        'the index argument first',
+        {
+          typeFull: "text(tokenizer = 'array', dictionary_block_size = 8192)",
+          engineFull,
+          serverMergeTreeSettings: new Map([
+            ['text_index_dictionary_block_size', '1024'],
+          ]),
+        },
+        8192,
+      ],
+      [
+        'the table setting next',
+        {
+          typeFull: "text(tokenizer = 'array')",
+          engineFull,
+          serverMergeTreeSettings: new Map([
+            ['text_index_dictionary_block_size', '1024'],
+          ]),
+        },
+        4096,
+      ],
+      [
+        'the server setting next',
+        {
+          typeFull: "text(tokenizer = 'array')",
+          engineFull: 'MergeTree ORDER BY Timestamp',
+          serverMergeTreeSettings: new Map([
+            ['text_index_dictionary_block_size', '1024'],
+          ]),
+        },
+        1024,
+      ],
+      ['512 when nothing is known', {}, 512],
+      [
+        '512 for a value that is not a positive integer',
+        { typeFull: "text(tokenizer = 'array', dictionary_block_size = 0)" },
+        512,
+      ],
+    ])('uses %s', (_, input, expected) => {
+      expect(getTextIndexDictionaryBlockSize(input)).toBe(expected);
+    });
+  });
 });
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 7675c6ee0..821b828c7 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -59,6 +59,7 @@ import {
 import {
   getAlignedDateRange,
   getDistributedTableArgs,
+  getTextIndexDictionaryBlockSize,
   MetadataMVQueryOptions,
   objectHash,
   splitAndTrimWithBracket,
@@ -84,6 +85,9 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
 // Keys per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
+// Smallest dictionary block size that gets one read per key; see
+// getMapTextIndexKeyValues.
+const PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE = 4096;
 
 /**
  * The index read could not be scoped to the requested date range, so it was
@@ -1758,6 +1762,31 @@ export class Metadata {
     });
   }
 
+  // shortcut: parts written before a table's text_index_dictionary_block_size
+  // changed keep their old block size; good enough to pick a query shape.
+  private async getTextIndexDictionaryBlockSize({
+    databaseName,
+    tableName,
+    connectionId,
+    indexName,
+  }: TableConnection & { indexName: string }): Promise<number> {
+    const [skipIndices, tableMetadata, serverMergeTreeSettings] =
+      await Promise.all([
+        this.getSkipIndices({ databaseName, tableName, connectionId }).catch(
+          () => [] as SkipIndexMetadata[],
+        ),
+        this.getTableMetadata({ databaseName, tableName, connectionId }).catch(
+          () => undefined,
+        ),
+        this.getMergeTreeSettings({ connectionId }),
+      ]);
+    return getTextIndexDictionaryBlockSize({
+      typeFull: skipIndices.find(idx => idx.name === indexName)?.typeFull,
+      engineFull: tableMetadata?.engine_full,
+      serverMergeTreeSettings,
+    });
+  }
+
   private async getMapTextIndexKeyValues({
     databaseName,
     tableName,
@@ -1792,16 +1821,49 @@ export class Metadata {
             partSelection,
           });
           const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`;
-          // One branch per key, each with its own LIMIT: the read for a key
-          // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a
-          // value per row can't slow down or starve the others. `cardinality`
-          // is the token's row count in the part, so topKWeighted keeps the
-          // most frequent values among the tokens read.
+          // Inline keys as SQL-escaped literals, not bind params: ~100
+          // per-key params exceed the web client's URL param budget and
+          // silently switch the request to a multipart body that proxies
+          // may reject. Keys are ingest-controlled, hence SqlString.escape.
+          const keyPrefix = (k: string) =>
+            chSql`${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}`;
+          // `cardinality` is the token's row count in the part, so
+          // topKWeighted keeps the most frequent values among the tokens read.
+          //
+          // Every read of the index loads each part's sparse index, which has
+          // one entry per dictionary block. With large blocks that is cheap, so
+          // each key gets its own read whose LIMIT stops it early, and a key
+          // with a value per row can't slow down the others. With small blocks
+          // (ClickHouse's default 512) the sparse index is large, so all keys
+          // share one read; LIMIT BY bounds their values but not the read.
+          const blockSize = await this.getTextIndexDictionaryBlockSize({
+            databaseName,
+            tableName,
+            connectionId,
+            indexName: info.indexName,
+          });
+          if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) {
+            sqlBranches.push(chSql`
+        SELECT ${{ String: columnName }} AS column,
+          key,
+          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value
+        FROM (
+          SELECT substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
+            ${valueSql} AS v,
+            cardinality
+          FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
+          WHERE ${partsFilter}
+            AND (${concatChSql(
+              ' OR ',
+              info.keys.map(k => chSql`startsWith(token, ${keyPrefix(k)})`),
+            )})
+            AND v != ''
+          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} BY key
+        )
+        GROUP BY key`);
+            continue;
+          }
           for (const k of info.keys) {
-            // Inline keys as SQL-escaped literals, not bind params: ~100
-            // per-key params exceed the web client's URL param budget and
-            // silently switch the request to a multipart body that proxies
-            // may reject. Keys are ingest-controlled, hence SqlString.escape.
             sqlBranches.push(chSql`
         SELECT ${{ String: columnName }} AS column,
           ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key,
@@ -1810,7 +1872,7 @@ export class Metadata {
           SELECT ${valueSql} AS v, cardinality
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
-            AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }})
+            AND startsWith(token, ${keyPrefix(k)})
             AND v != ''
           LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
         )`);
@@ -1825,8 +1887,8 @@ export class Metadata {
             connectionId,
             clickhouse_settings: {
               ...this.getClickHouseSettings(),
-              // The per-key LIMIT bounds the read. A row cap would stop the
-              // sorted dictionary read partway and leave later keys empty.
+              // A row cap would stop the sorted dictionary read partway and
+              // leave the keys that sort after it with no values.
               max_rows_to_read: '0',
             },
             abort_signal: signal,
diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts
index 7c9307845..a79601e5c 100644
--- a/packages/common-utils/src/core/utils.ts
+++ b/packages/common-utils/src/core/utils.ts
@@ -1443,6 +1443,41 @@ export type TextIndexTokenizer =
     }
   | { type: 'array' };
 
+/** Reads a setting from `system.tables.engine_full`, which ends with the SETTINGS clause. */
+export function getEngineSetting(engineFull: string, name: string) {
+  const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? '';
+  return splitAndTrimWithBracket(settingsClause)
+    .map(setting => setting.split(' = '))
+    .find(([key]) => key === name)?.[1];
+}
+
+const DEFAULT_TEXT_INDEX_DICTIONARY_BLOCK_SIZE = 512;
+
+/**
+ * A text index's dictionary block size: the index argument, else the table's
+ * `text_index_dictionary_block_size`, else the server's, else ClickHouse's 512.
+ */
+export function getTextIndexDictionaryBlockSize({
+  typeFull,
+  engineFull,
+  serverMergeTreeSettings,
+}: {
+  typeFull?: string;
+  engineFull?: string;
+  serverMergeTreeSettings?: Map<string, string>;
+}): number {
+  const value =
+    typeFull?.match(/\bdictionary_block_size\s*=\s*(\d+)/)?.[1] ??
+    (engineFull
+      ? getEngineSetting(engineFull, 'text_index_dictionary_block_size')
+      : undefined) ??
+    serverMergeTreeSettings?.get('text_index_dictionary_block_size');
+  const size = Number(value);
+  return Number.isInteger(size) && size > 0
+    ? size
+    : DEFAULT_TEXT_INDEX_DICTIONARY_BLOCK_SIZE;
+}
+
 /**
  * Parses the tokenizer and any associated tokenizer parameters from a text index type definition.
  *

From f1e89a5b71365da60c05c1d7a85ab6894e264fcb Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 16:04:13 -0500
Subject: [PATCH 14/27] perf: rank small-block text index values over every
 token read

---
 .changeset/topk-text-index-values.md          | 19 +++++++++----------
 .../src/__tests__/metadata.test.ts            |  3 ++-
 packages/common-utils/src/core/metadata.ts    |  4 ++--
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index f5dd4380c..9730d039c 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -7,16 +7,15 @@ perf: read text index values in batches of 10 keys, most frequent first
 "All values" lookups that read a text index (Map key=value indexes and text
 indexes on plain columns) read every token of up to 100 keys in one query, so
 one key with a value per row (request IDs, UUIDs) slowed every other key down.
-These lookups now run in parallel queries of at most 10 keys, and keep at most
-20,000 tokens per key. Values come from `topKWeighted(limit)` weighted by each
-token's row count, so a capped list holds the most common values, most common
-first.
+These lookups now run in parallel queries of at most 10 keys. Values come from
+`topKWeighted(limit)` weighted by each token's row count, so a capped list holds
+the most common values, most common first.
 
 Map key=value indexes with large dictionary blocks (4096 or more, e.g.
 `dictionary_block_size = 8192` as the seed schemas now use) read each key
-separately, so a high-cardinality key stops after its 20,000 tokens. Indexes with
-smaller blocks, including ClickHouse's default of 512, read all keys of a batch
-in one pass: every read loads each part's sparse index, which is large with small
-blocks, so one read per key would multiply that cost. The block size comes from
-the index definition, then the table's `text_index_dictionary_block_size`, then
-the server's.
+separately and stop after 20,000 tokens, so a high-cardinality key can't hold up
+the others. Indexes with smaller blocks, including ClickHouse's default of 512,
+read all keys of a batch in one pass: every read loads each part's sparse index,
+which is large with small blocks, so one read per key would multiply that cost.
+The block size comes from the index definition, then the table's
+`text_index_dictionary_block_size`, then the server's.
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e46b78ac2..ceb1f4396 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1461,7 +1461,8 @@ describe('Metadata', () => {
       expect(sql).toContain(
         "startsWith(token, 'requestId=') OR startsWith(token, 'region=')",
       );
-      expect(sql).toContain('LIMIT 20000 BY key');
+      expect(sql).not.toContain('BY key\n');
+      expect(sql).not.toMatch(/LIMIT \d+/);
       expect(sql).toContain('GROUP BY key');
       expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
     });
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 821b828c7..55b6ec8ae 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1835,7 +1835,8 @@ export class Metadata {
           // each key gets its own read whose LIMIT stops it early, and a key
           // with a value per row can't slow down the others. With small blocks
           // (ClickHouse's default 512) the sparse index is large, so all keys
-          // share one read; LIMIT BY bounds their values but not the read.
+          // share one read. That read can't stop early per key, so topKWeighted
+          // sees every token and ranks by the full counts.
           const blockSize = await this.getTextIndexDictionaryBlockSize({
             databaseName,
             tableName,
@@ -1858,7 +1859,6 @@ export class Metadata {
               info.keys.map(k => chSql`startsWith(token, ${keyPrefix(k)})`),
             )})
             AND v != ''
-          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} BY key
         )
         GROUP BY key`);
             continue;

From be8f60fcad9fa737d66d61d668bfe7436c554dfe Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 16:41:07 -0500
Subject: [PATCH 15/27] perf: keep main's batching for small-block Map text
 index values

---
 .changeset/topk-text-index-values.md          | 21 ++++----
 .../src/__tests__/metadata.test.ts            |  4 +-
 packages/common-utils/src/core/metadata.ts    | 50 +++++++++++++++----
 3 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 9730d039c..055b3c83c 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,20 +2,21 @@
 '@hyperdx/common-utils': patch
 ---
 
-perf: read text index values in batches of 10 keys, most frequent first
+perf: show the most frequent text index values, and read large-block Map
+indexes per key
 
 "All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) read every token of up to 100 keys in one query, so
-one key with a value per row (request IDs, UUIDs) slowed every other key down.
-These lookups now run in parallel queries of at most 10 keys. Values come from
-`topKWeighted(limit)` weighted by each token's row count, so a capped list holds
-the most common values, most common first.
+indexes on plain columns) kept the alphabetically first values. Values now come
+from `topKWeighted(limit)` weighted by each token's row count, so a capped list
+holds the most common values, most common first.
 
 Map key=value indexes with large dictionary blocks (4096 or more, e.g.
 `dictionary_block_size = 8192` as the seed schemas now use) read each key
-separately and stop after 20,000 tokens, so a high-cardinality key can't hold up
-the others. Indexes with smaller blocks, including ClickHouse's default of 512,
-read all keys of a batch in one pass: every read loads each part's sparse index,
+separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens
+per key, so a key with a value per row (request IDs, UUIDs) can't hold up the
+others. Indexes with smaller blocks, including ClickHouse's default of 512, keep
+one read for all keys as before: every read loads each part's sparse index,
 which is large with small blocks, so one read per key would multiply that cost.
 The block size comes from the index definition, then the table's
-`text_index_dictionary_block_size`, then the server's.
+`text_index_dictionary_block_size`, then the server's. Text indexes on plain
+columns run in parallel queries of at most 10 columns.
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index ceb1f4396..7ac297588 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1577,9 +1577,9 @@ describe('Metadata', () => {
 
     it.each([
       [8192, [10, 10, 5]],
-      [512, [1, 1, 1]],
+      [512, [1]],
     ])(
-      'splits Map text index values into queries of at most 10 keys (dictionary blocks of %d)',
+      'batches Map text index values by dictionary block size (%d)',
       async (blockSize, expectedBranchCounts) => {
         setupDefaultLogsSchema();
         withDictionaryBlockSize(blockSize);
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 55b6ec8ae..7970d26b5 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -83,7 +83,8 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 // so a key with more distinct values than this gets the most frequent among
 // its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
 const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
-// Keys per text index values query; see dispatchKeyValueQueries.
+// Keys per text index values query when each key gets its own read, and
+// columns per plain-column text index query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 // Smallest dictionary block size that gets one read per key; see
 // getMapTextIndexKeyValues.
@@ -1792,6 +1793,7 @@ export class Metadata {
     tableName,
     connectionId,
     queryOptions,
+    perKey,
     maxParts,
     partSelection,
     dateRange,
@@ -1799,6 +1801,8 @@ export class Metadata {
     signal,
   }: TableConnection & {
     queryOptions: TextIndexMapColumnQueryOptions;
+    /** One read per key instead of one per column; see the comment below. */
+    perKey: boolean;
     /** null reads every part in the time range. */
     maxParts: number | null;
     partSelection: MapKeyIndexPartSelection;
@@ -1807,7 +1811,7 @@ export class Metadata {
     signal?: AbortSignal;
   }): Promise<KeyValues[] | undefined> {
     const queryOptionsHash = objectHash(queryOptions);
-    const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`;
+    const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${perKey}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`;
     return this.cache.getOrFetch(cacheKey, async () => {
       try {
         const sqlBranches: Array<ChSql> = [];
@@ -1837,13 +1841,7 @@ export class Metadata {
           // (ClickHouse's default 512) the sparse index is large, so all keys
           // share one read. That read can't stop early per key, so topKWeighted
           // sees every token and ranks by the full counts.
-          const blockSize = await this.getTextIndexDictionaryBlockSize({
-            databaseName,
-            tableName,
-            connectionId,
-            indexName: info.indexName,
-          });
-          if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) {
+          if (!perKey) {
             sqlBranches.push(chSql`
         SELECT ${{ String: columnName }} AS column,
           key,
@@ -3124,9 +3122,22 @@ export class Metadata {
 
     // fire all the kv fetch queries
     const promises: Array<Promise<KeyValues[] | undefined>> = [];
-    // Text index lookups run in small parallel batches, so a slow,
-    // high-cardinality key holds up only its own batch's values.
+    // Map columns whose index has large dictionary blocks read each key
+    // separately, in small parallel batches, so a slow, high-cardinality key
+    // holds up only its own batch's values. The rest share one read of all
+    // their keys; see getMapTextIndexKeyValues.
+    const sharedReadOptions: TextIndexMapColumnQueryOptions = new Map();
     for (const [column, options] of mapTextIndexQueryOptions) {
+      const blockSize = await this.getTextIndexDictionaryBlockSize({
+        databaseName,
+        tableName,
+        connectionId,
+        indexName: options.indexName,
+      });
+      if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) {
+        sharedReadOptions.set(column, options);
+        continue;
+      }
       for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) {
         promises.push(
           this.getMapTextIndexKeyValues({
@@ -3134,6 +3145,7 @@ export class Metadata {
             tableName,
             connectionId,
             queryOptions: new Map([[column, { ...options, keys }]]),
+            perKey: true,
             maxParts: mapKeyIndexMaxParts,
             partSelection: mapKeyIndexPartSelection,
             dateRange,
@@ -3143,6 +3155,22 @@ export class Metadata {
         );
       }
     }
+    if (sharedReadOptions.size > 0) {
+      promises.push(
+        this.getMapTextIndexKeyValues({
+          databaseName,
+          tableName,
+          connectionId,
+          queryOptions: sharedReadOptions,
+          perKey: false,
+          maxParts: mapKeyIndexMaxParts,
+          partSelection: mapKeyIndexPartSelection,
+          dateRange,
+          timestampValueExpression,
+          signal,
+        }),
+      );
+    }
     for (const entries of chunk(
       [...nativeTextIndexQueryOptions],
       TEXT_INDEX_KEYS_PER_QUERY,

From 265efeed9b18c013c8e86c3e9ff39d4513c1c1b2 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 16:50:26 -0500
Subject: [PATCH 16/27] perf: keep main's query and row cap for small-block Map
 text index values

---
 .changeset/topk-text-index-values.md          | 33 +++++-----
 .../src/__tests__/metadata.test.ts            | 60 +++++++++++--------
 packages/common-utils/src/core/metadata.ts    | 51 +++++++++-------
 3 files changed, 82 insertions(+), 62 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 055b3c83c..99adf0b18 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -2,21 +2,22 @@
 '@hyperdx/common-utils': patch
 ---
 
-perf: show the most frequent text index values, and read large-block Map
-indexes per key
+perf: read large-block Map text index values per key, most frequent first
 
-"All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) kept the alphabetically first values. Values now come
-from `topKWeighted(limit)` weighted by each token's row count, so a capped list
-holds the most common values, most common first.
+"All values" lookups on Map key=value text indexes with large dictionary blocks
+(4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use)
+now read each key separately, in parallel queries of at most 10 keys, and stop
+after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs)
+can't hold up the others. Values come from `topKWeighted(limit)` weighted by
+each token's row count, so a capped list holds the most common values, most
+common first.
 
-Map key=value indexes with large dictionary blocks (4096 or more, e.g.
-`dictionary_block_size = 8192` as the seed schemas now use) read each key
-separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens
-per key, so a key with a value per row (request IDs, UUIDs) can't hold up the
-others. Indexes with smaller blocks, including ClickHouse's default of 512, keep
-one read for all keys as before: every read loads each part's sparse index,
-which is large with small blocks, so one read per key would multiply that cost.
-The block size comes from the index definition, then the table's
-`text_index_dictionary_block_size`, then the server's. Text indexes on plain
-columns run in parallel queries of at most 10 columns.
+Indexes with smaller blocks, including ClickHouse's default of 512, keep the
+previous query: one read for all keys, the alphabetically first values, and the
+`max_rows_to_read` cap. Every read loads each part's sparse index, which is
+large with small blocks, so one read per key would multiply that cost. The block
+size comes from the index definition, then the table's
+`text_index_dictionary_block_size`, then the server's.
+
+Text indexes on plain columns now return the most frequent values with
+`topKWeighted`, in parallel queries of at most 10 columns.
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 7ac297588..66639fd0e 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -2,6 +2,7 @@ import { ColumnMeta } from '@/clickhouse';
 import { ClickhouseClient } from '@/clickhouse/node';
 import {
   COLUMNS_CACHE_TTL_MS,
+  DEFAULT_METADATA_MAX_ROWS_TO_READ,
   GET_ALL_KEY_VALUES_CHUNK_SIZE,
   Metadata,
   MetadataCache,
@@ -1461,10 +1462,12 @@ describe('Metadata', () => {
       expect(sql).toContain(
         "startsWith(token, 'requestId=') OR startsWith(token, 'region=')",
       );
-      expect(sql).not.toContain('BY key\n');
-      expect(sql).not.toMatch(/LIMIT \d+/);
-      expect(sql).toContain('GROUP BY key');
-      expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
+      expect(sql).toContain('groupUniqArray(');
+      expect(sql).toContain('GROUP BY column, key');
+      expect(call.clickhouse_settings).toMatchObject({
+        max_rows_to_read: String(DEFAULT_METADATA_MAX_ROWS_TO_READ),
+        read_overflow_mode: 'break',
+      });
     });
 
     it.each([
@@ -1474,6 +1477,8 @@ describe('Metadata', () => {
       'reads the %s text index without a row cap, even when one is configured',
       async (_, keyExpression) => {
         setupDefaultLogsSchema();
+        // Small-block Map indexes keep the cap; see the one-pass test.
+        withDictionaryBlockSize(8192);
         jest
           .spyOn(metadata, 'getClickHouseSettings')
           .mockReturnValue({ max_rows_to_read: '1000' });
@@ -1926,27 +1931,34 @@ describe('Metadata', () => {
       // param dump when a test fails.
       const MAX_VALUES = 7777;
 
-      it('threads maxValuesPerKey into the map text-index query', async () => {
-        setupDefaultLogsSchema();
-
-        await metadata.getAllKeyValues({
-          ...baseArgs,
-          keyExpressions: ["LogAttributes['requestId']"],
-          maxValuesPerKey: MAX_VALUES,
-        });
+      it.each([
+        [8192, 'topKWeighted'],
+        [512, 'groupUniqArray'],
+      ])(
+        'threads maxValuesPerKey into the map text-index query (dictionary blocks of %d)',
+        async (blockSize, aggregate) => {
+          setupDefaultLogsSchema();
+          withDictionaryBlockSize(blockSize);
+
+          await metadata.getAllKeyValues({
+            ...baseArgs,
+            keyExpressions: ["LogAttributes['requestId']"],
+            maxValuesPerKey: MAX_VALUES,
+          });
 
-        const mapTextIndexCall = (
-          mockClickhouseClient.query as jest.Mock
-        ).mock.calls.find(
-          (c: any[]) =>
-            typeof c[0].query === 'string' &&
-            c[0].query.includes('startsWith(token,'),
-        );
-        expect(mapTextIndexCall).toBeDefined();
-        expect(mapTextIndexCall![0].query).toContain(
-          `topKWeighted(${MAX_VALUES})(`,
-        );
-      });
+          const mapTextIndexCall = (
+            mockClickhouseClient.query as jest.Mock
+          ).mock.calls.find(
+            (c: any[]) =>
+              typeof c[0].query === 'string' &&
+              c[0].query.includes('startsWith(token,'),
+          );
+          expect(mapTextIndexCall).toBeDefined();
+          expect(mapTextIndexCall![0].query).toContain(
+            `${aggregate}(${MAX_VALUES})(`,
+          );
+        },
+      );
 
       it('threads maxValuesPerKey into the native text-index query', async () => {
         setupDefaultLogsSchema();
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 7970d26b5..dc9cb2352 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1831,34 +1831,28 @@ export class Metadata {
           // may reject. Keys are ingest-controlled, hence SqlString.escape.
           const keyPrefix = (k: string) =>
             chSql`${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}`;
-          // `cardinality` is the token's row count in the part, so
-          // topKWeighted keeps the most frequent values among the tokens read.
-          //
           // Every read of the index loads each part's sparse index, which has
           // one entry per dictionary block. With large blocks that is cheap, so
           // each key gets its own read whose LIMIT stops it early, and a key
-          // with a value per row can't slow down the others. With small blocks
+          // with a value per row can't slow down the others; `cardinality` is
+          // the token's row count in the part, so topKWeighted keeps the most
+          // frequent values among the tokens read. With small blocks
           // (ClickHouse's default 512) the sparse index is large, so all keys
-          // share one read. That read can't stop early per key, so topKWeighted
-          // sees every token and ranks by the full counts.
+          // share one read, as on main.
           if (!perKey) {
             sqlBranches.push(chSql`
-        SELECT ${{ String: columnName }} AS column,
-          key,
-          topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value
-        FROM (
-          SELECT substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
-            ${valueSql} AS v,
-            cardinality
+          SELECT ${{ String: columnName }} AS column,
+            substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key,
+            groupUniqArray(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}) AS value
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND (${concatChSql(
               ' OR ',
               info.keys.map(k => chSql`startsWith(token, ${keyPrefix(k)})`),
             )})
-            AND v != ''
-        )
-        GROUP BY key`);
+            AND ${valueSql} != ''
+          GROUP BY column, key
+          `);
             continue;
           }
           for (const k of info.keys) {
@@ -1883,12 +1877,25 @@ export class Metadata {
             query: sql.sql,
             query_params: sql.params,
             connectionId,
-            clickhouse_settings: {
-              ...this.getClickHouseSettings(),
-              // A row cap would stop the sorted dictionary read partway and
-              // leave the keys that sort after it with no values.
-              max_rows_to_read: '0',
-            },
+            clickhouse_settings: perKey
+              ? {
+                  ...this.getClickHouseSettings(),
+                  // The per-key LIMIT bounds each read. A row cap would stop
+                  // the sorted dictionary read partway and leave the keys that
+                  // sort after it with no values.
+                  max_rows_to_read: '0',
+                }
+              : {
+                  // shortcut: bounds the shared read, but keys that sort after
+                  // a high-cardinality key can come back with no values; drop
+                  // it if that matters more than the read's cost.
+                  max_rows_to_read: String(
+                    this.getClickHouseSettings().max_rows_to_read ??
+                      DEFAULT_METADATA_MAX_ROWS_TO_READ,
+                  ),
+                  read_overflow_mode: 'break',
+                  ...this.getClickHouseSettings(),
+                },
             abort_signal: signal,
           })
           .then(res =>

From 6f823028de18e5c9369c3307b2f27998b7195dc2 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 16:51:57 -0500
Subject: [PATCH 17/27] perf: batch small-block Map text index values by 10
 keys

---
 .changeset/topk-text-index-values.md          |  8 +--
 .../src/__tests__/metadata.test.ts            |  2 +-
 packages/common-utils/src/core/metadata.ts    | 51 ++++++-------------
 3 files changed, 21 insertions(+), 40 deletions(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 99adf0b18..11d50a120 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -13,9 +13,11 @@ each token's row count, so a capped list holds the most common values, most
 common first.
 
 Indexes with smaller blocks, including ClickHouse's default of 512, keep the
-previous query: one read for all keys, the alphabetically first values, and the
-`max_rows_to_read` cap. Every read loads each part's sparse index, which is
-large with small blocks, so one read per key would multiply that cost. The block
+previous query shape (one read for all of a query's keys, the alphabetically
+first values, and the `max_rows_to_read` cap), now in parallel queries of at
+most 10 keys, so the cap's cut-off affects at most 9 other keys. Every read loads
+each part's sparse index, which is large with small blocks, so one read per key
+would multiply that cost. The block
 size comes from the index definition, then the table's
 `text_index_dictionary_block_size`, then the server's.
 
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 66639fd0e..a1c8ef012 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1582,7 +1582,7 @@ describe('Metadata', () => {
 
     it.each([
       [8192, [10, 10, 5]],
-      [512, [1]],
+      [512, [1, 1, 1]],
     ])(
       'batches Map text index values by dictionary block size (%d)',
       async (blockSize, expectedBranchCounts) => {
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index dc9cb2352..afc21020f 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -83,8 +83,7 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 // so a key with more distinct values than this gets the most frequent among
 // its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
 const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
-// Keys per text index values query when each key gets its own read, and
-// columns per plain-column text index query; see dispatchKeyValueQueries.
+// Keys or columns per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 // Smallest dictionary block size that gets one read per key; see
 // getMapTextIndexKeyValues.
@@ -1837,8 +1836,8 @@ export class Metadata {
           // with a value per row can't slow down the others; `cardinality` is
           // the token's row count in the part, so topKWeighted keeps the most
           // frequent values among the tokens read. With small blocks
-          // (ClickHouse's default 512) the sparse index is large, so all keys
-          // share one read, as on main.
+          // (ClickHouse's default 512) the sparse index is large, so the
+          // batch's keys share one read.
           if (!perKey) {
             sqlBranches.push(chSql`
           SELECT ${{ String: columnName }} AS column,
@@ -3129,22 +3128,18 @@ export class Metadata {
 
     // fire all the kv fetch queries
     const promises: Array<Promise<KeyValues[] | undefined>> = [];
-    // Map columns whose index has large dictionary blocks read each key
-    // separately, in small parallel batches, so a slow, high-cardinality key
-    // holds up only its own batch's values. The rest share one read of all
-    // their keys; see getMapTextIndexKeyValues.
-    const sharedReadOptions: TextIndexMapColumnQueryOptions = new Map();
+    // Text index lookups run in small parallel batches, so a slow,
+    // high-cardinality key holds up only its own batch's values. Within a
+    // batch, large dictionary blocks get one read per key; see
+    // getMapTextIndexKeyValues.
     for (const [column, options] of mapTextIndexQueryOptions) {
-      const blockSize = await this.getTextIndexDictionaryBlockSize({
-        databaseName,
-        tableName,
-        connectionId,
-        indexName: options.indexName,
-      });
-      if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) {
-        sharedReadOptions.set(column, options);
-        continue;
-      }
+      const perKey =
+        (await this.getTextIndexDictionaryBlockSize({
+          databaseName,
+          tableName,
+          connectionId,
+          indexName: options.indexName,
+        })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE;
       for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) {
         promises.push(
           this.getMapTextIndexKeyValues({
@@ -3152,7 +3147,7 @@ export class Metadata {
             tableName,
             connectionId,
             queryOptions: new Map([[column, { ...options, keys }]]),
-            perKey: true,
+            perKey,
             maxParts: mapKeyIndexMaxParts,
             partSelection: mapKeyIndexPartSelection,
             dateRange,
@@ -3162,22 +3157,6 @@ export class Metadata {
         );
       }
     }
-    if (sharedReadOptions.size > 0) {
-      promises.push(
-        this.getMapTextIndexKeyValues({
-          databaseName,
-          tableName,
-          connectionId,
-          queryOptions: sharedReadOptions,
-          perKey: false,
-          maxParts: mapKeyIndexMaxParts,
-          partSelection: mapKeyIndexPartSelection,
-          dateRange,
-          timestampValueExpression,
-          signal,
-        }),
-      );
-    }
     for (const entries of chunk(
       [...nativeTextIndexQueryOptions],
       TEXT_INDEX_KEYS_PER_QUERY,

From 78b4b008d7eff12e3dd744d76ebe2fa27d2ea452 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 12 Nov 2026 20:03:57 -0500
Subject: [PATCH 18/27] docs: fold the row cap change into the text index
 values changeset

---
 .changeset/text-index-values-no-row-cap.md | 11 -----------
 .changeset/topk-text-index-values.md       | 11 ++++++-----
 2 files changed, 6 insertions(+), 16 deletions(-)
 delete mode 100644 .changeset/text-index-values-no-row-cap.md

diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md
deleted file mode 100644
index 103ed81b0..000000000
--- a/.changeset/text-index-values-no-row-cap.md
+++ /dev/null
@@ -1,11 +0,0 @@
----
-'@hyperdx/common-utils': patch
----
-
-fix: show values for every key in text index value lookups
-
-"All values" lookups that read a text index (Map key=value indexes and text
-indexes on plain columns) stopped at `max_rows_to_read` with
-`read_overflow_mode = 'break'`. The index is read in sorted token order, so once
-a high-cardinality key used up the budget, every key that sorts after it came
-back with no values and no error. These lookups no longer apply a row cap.
diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 11d50a120..86398f1f3 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -10,16 +10,17 @@ now read each key separately, in parallel queries of at most 10 keys, and stop
 after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs)
 can't hold up the others. Values come from `topKWeighted(limit)` weighted by
 each token's row count, so a capped list holds the most common values, most
-common first.
+common first. These reads no longer apply `max_rows_to_read`: the index is read
+in sorted token order, so the cap left every key that sorts after a
+high-cardinality key with no values.
 
 Indexes with smaller blocks, including ClickHouse's default of 512, keep the
 previous query shape (one read for all of a query's keys, the alphabetically
 first values, and the `max_rows_to_read` cap), now in parallel queries of at
 most 10 keys, so the cap's cut-off affects at most 9 other keys. Every read loads
 each part's sparse index, which is large with small blocks, so one read per key
-would multiply that cost. The block
-size comes from the index definition, then the table's
-`text_index_dictionary_block_size`, then the server's.
+would multiply that cost. The block size comes from the index definition, then
+the table's `text_index_dictionary_block_size`, then the server's.
 
 Text indexes on plain columns now return the most frequent values with
-`topKWeighted`, in parallel queries of at most 10 columns.
+`topKWeighted`, in parallel queries of at most 10 columns, without the row cap.

From 60d29fad860457084d87febc89c1f1a30f70d076 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:02:41 -0500
Subject: [PATCH 19/27] perf: set the text index block size as a table setting
 after seeding

---
 .../text-index-dictionary-block-size.md       | 25 ++++-----
 .changeset/topk-text-index-values.md          |  3 +-
 docker/clickhouse/local/init-db-e2e.sh        | 24 ++++-----
 .../schema/seed/00002_otel_logs.sql           | 18 +++----
 .../schema/seed/00004_hyperdx_sessions.sql    |  4 +-
 .../schema/seed/00005_otel_traces.sql         | 14 ++---
 .../src/__tests__/kvItemsRewrite.int.test.ts  |  5 +-
 .../src/__tests__/metadata.int.test.ts        | 11 ++--
 packages/otel-collector/cmd/migrate/main.go   | 51 +++++++++++++++++++
 .../otel-collector/cmd/migrate/main_test.go   | 29 +++++++++++
 10 files changed, 135 insertions(+), 49 deletions(-)

diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md
index da6a7e61b..35c8d3f1a 100644
--- a/.changeset/text-index-dictionary-block-size.md
+++ b/.changeset/text-index-dictionary-block-size.md
@@ -2,17 +2,18 @@
 '@hyperdx/otel-collector': patch
 ---
 
-perf: use 8,192-token dictionary blocks for every text index in the seed
-schemas
+perf: use 8,192-token text index dictionary blocks on the seeded tables
 
-Every text index on `otel_logs`, `otel_traces` and `hyperdx_sessions` now sets
-`dictionary_block_size = 8192` (the default is 512). Each part's sparse index
-is 16x smaller. Reading filter values with `mergeTreeTextIndex()` loads that
-index for every part on every read, and on a 537M-row table 4,096-token blocks
-already made those reads about 10x cheaper per part, made searches for a rare
-attribute value 3-7x faster, and made searches for a common value 10-30%
-faster. The setting is a per-index argument rather than the table setting
-`text_index_dictionary_block_size`, which ClickHouse only accepts from 26.6.
+On ClickHouse 26.6 and later, the collector's schema migration now sets
+`text_index_dictionary_block_size = 8192` (the default is 512) on `otel_logs`,
+`otel_traces` and `hyperdx_sessions`, with `ALTER TABLE … MODIFY SETTING` after
+the seed. Each part's text index sparse index is 16x smaller. Reading filter
+values with `mergeTreeTextIndex()` loads that index for every part on every
+read, and on a 537M-row table 4,096-token blocks already made those reads about
+10x cheaper per part, made searches for a rare attribute value 3-7x faster, and
+made searches for a common value 10-30% faster.
 
-Only new installs get the new schema. Existing tables keep their indexes until
-the indexes are cleared and materialized again.
+Because it's an `ALTER`, existing installs get it too: new parts use 8,192-token
+blocks, and existing parts keep theirs until they're merged. A table that
+already sets `text_index_dictionary_block_size` is left alone. ClickHouse
+26.2–26.5 doesn't have the setting, so those servers keep 512-token blocks.
diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 86398f1f3..7ab43daeb 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -5,7 +5,8 @@
 perf: read large-block Map text index values per key, most frequent first
 
 "All values" lookups on Map key=value text indexes with large dictionary blocks
-(4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use)
+(4096 or more, e.g. `text_index_dictionary_block_size = 8192`, which the
+collector's migration now sets on ClickHouse 26.6+)
 now read each key separately, in parallel queries of at most 10 keys, and stop
 after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs)
 can't hold up the others. Values come from `topKWeighted(limit)` weighted by
diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh
index 4ce993493..846566110 100755
--- a/docker/clickhouse/local/init-db-e2e.sh
+++ b/docker/clickhouse/local/init-db-e2e.sh
@@ -40,20 +40,20 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs
   \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)),
   \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)),
   \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)),
-  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192)
+  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
 ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)
 TTL toDateTime(Timestamp) + toIntervalDay(30)
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, text_index_dictionary_block_size = 8192;
 
 -- Distributed table over the e2e logs table whose DDL declares a column
 -- (\`ColumnMissingFromShard\`) that its underlying MergeTree table does NOT have.
@@ -118,10 +118,10 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces
     INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1,
     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
     INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
 )
@@ -129,7 +129,7 @@ ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
 ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
 TTL toDate(Timestamp) + toIntervalDay(30)
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, text_index_dictionary_block_size = 8192;
 
 CREATE TABLE ${DATABASE}.e2e_hyperdx_sessions
 (
diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql
index fc046fdcb..f555e7865 100644
--- a/docker/otel-collector/schema/seed/00002_otel_logs.sql
+++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql
@@ -32,15 +32,15 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
   -- part's sparse index is 16x smaller, which makes reading filter values
   -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
-  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192)
+  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
+  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'),
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
+  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'),
+  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
index 9f106e9fe..ac44c11f5 100644
--- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
+++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
@@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
   -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
   -- part's sparse index is 16x smaller, which makes reading filter values
   -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
-  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-  INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
+  INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql
index 17d040434..c019d050f 100644
--- a/docker/otel-collector/schema/seed/00005_otel_traces.sql
+++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql
@@ -30,14 +30,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
     -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
     -- part's sparse index is 16x smaller, which makes reading filter values
     -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
-    INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
+    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
+    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
-    INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName), dictionary_block_size = 8192)
+    INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName))
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
diff --git a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
index 7a59955b6..fd75a190a 100644
--- a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
+++ b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
@@ -70,10 +70,11 @@ describe('KV items rewrite integration', () => {
         LogAttributeItems Array(String) MATERIALIZED
           arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
           CODEC(ZSTD(1)),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
       )
       ENGINE = MergeTree
-      ORDER BY (Timestamp)`,
+      ORDER BY (Timestamp)
+      SETTINGS text_index_dictionary_block_size = 8192`,
     });
 
     await client.insert({
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 915703f1c..1fd29056c 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -697,10 +697,11 @@ describe('Metadata Integration Tests', () => {
             ResourceAttributeItems Array(String) MATERIALIZED
               arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))
               CODEC(ZSTD(1)),
-            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
+          SETTINGS text_index_dictionary_block_size = 8192
         `,
       });
 
@@ -712,10 +713,11 @@ describe('Metadata Integration Tests', () => {
             LogAttributeItems Array(String) MATERIALIZED
               arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
               CODEC(ZSTD(1)),
-            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
+            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
+          SETTINGS text_index_dictionary_block_size = 8192
         `,
       });
     });
@@ -1657,14 +1659,14 @@ describe('Metadata Integration Tests', () => {
     {
       path: 'key',
       tableName: 'test_map_keys_key_text_index_order',
-      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
+      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`,
     },
     {
       path: 'kv',
       tableName: 'test_map_keys_kv_text_index_order',
       indexDefinition: `LogAttributeItems Array(String) MATERIALIZED
           arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`,
     },
   ])(
     'getMapKeys - $path text index ordering',
@@ -1688,6 +1690,7 @@ describe('Metadata Integration Tests', () => {
             ENGINE = MergeTree()
             PARTITION BY toDate(Timestamp)
             ORDER BY Timestamp
+            SETTINGS text_index_dictionary_block_size = 8192
           `,
         });
 
diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go
index b67e37309..7dcc39a84 100644
--- a/packages/otel-collector/cmd/migrate/main.go
+++ b/packages/otel-collector/cmd/migrate/main.go
@@ -9,6 +9,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
+	"errors"
 	"fmt"
 	"log"
 	"net/url"
@@ -644,6 +645,49 @@ func supportsFullTextSearch(major, minor int) bool {
 	return major > 26 || (major == 26 && minor >= 2)
 }
 
+// textIndexTables are the seeded tables with text indexes.
+var textIndexTables = []string{"otel_logs", "otel_traces", "hyperdx_sessions"}
+
+// supportsTextIndexTableSettings returns true if the ClickHouse version has the
+// text_index_dictionary_block_size MergeTree setting. This requires ClickHouse
+// >= 26.6.
+func supportsTextIndexTableSettings(major, minor int) bool {
+	return major > 26 || (major == 26 && minor >= 6)
+}
+
+// setTextIndexBlockSize gives the seeded tables 8,192-token text index
+// dictionary blocks (the default is 512), so filter value reads load a 16x
+// smaller sparse index per part. It runs after the seed so existing tables
+// get it too; new parts use it, existing parts keep theirs until merged. A
+// table that already sets the value is left alone.
+func setTextIndexBlockSize(ctx context.Context, db *sql.DB, database string) error {
+	var firstErr error
+	for _, table := range textIndexTables {
+		var engineFull string
+		err := db.QueryRowContext(ctx,
+			"SELECT engine_full FROM system.tables WHERE database = ? AND name = ?",
+			database, table).Scan(&engineFull)
+		if errors.Is(err, sql.ErrNoRows) {
+			continue
+		}
+		if err == nil && strings.Contains(engineFull, "text_index_dictionary_block_size") {
+			continue
+		}
+		if err == nil {
+			_, err = db.ExecContext(ctx, fmt.Sprintf(
+				"ALTER TABLE `%s`.`%s` MODIFY SETTING text_index_dictionary_block_size = 8192",
+				database, table))
+		}
+		if err != nil {
+			log.Printf("WARNING: could not set the text index block size on %s: %v", table, err)
+			if firstErr == nil {
+				firstErr = err
+			}
+		}
+	}
+	return firstErr
+}
+
 // swapLogsSchemaForCompat replaces the full-text-search logs schema with the
 // compatibility variant (bloom_filter indexes) in the processed temp directory.
 // It removes 00002_otel_logs.sql and renames 00002_otel_logs_compat.sql to
@@ -875,6 +919,13 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Non-fatal so it can't block startup; older servers keep 512-token blocks.
+	if supportsTextIndexTableSettings(chMajor, chMinor) {
+		if err := setTextIndexBlockSize(ctx, db, cfg.Database); err != nil {
+			log.Printf("WARNING: text index block size: %v", err)
+		}
+	}
+
 	// CREATE TABLE IF NOT EXISTS won't update TTL on existing tables; reconcile
 	// (opt-in) applies it to them too. Non-fatal so it can't block startup.
 	if cfg.ReconcileTableTTL {
diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go
index 84e42687d..605f039e7 100644
--- a/packages/otel-collector/cmd/migrate/main_test.go
+++ b/packages/otel-collector/cmd/migrate/main_test.go
@@ -958,6 +958,35 @@ func TestSupportsFullTextSearch(t *testing.T) {
 	}
 }
 
+// ---------------------------------------------------------------------------
+// supportsTextIndexTableSettings
+// ---------------------------------------------------------------------------
+
+func TestSupportsTextIndexTableSettings(t *testing.T) {
+	tests := []struct {
+		name     string
+		major    int
+		minor    int
+		expected bool
+	}{
+		{"26.6 exact threshold", 26, 6, true},
+		{"26.10 above minor", 26, 10, true},
+		{"27.0 above major", 27, 0, true},
+		{"26.5 below minor", 26, 5, false},
+		{"26.2 text indexes without the setting", 26, 2, false},
+		{"25.9 below major", 25, 9, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := supportsTextIndexTableSettings(tt.major, tt.minor)
+			if got != tt.expected {
+				t.Errorf("supportsTextIndexTableSettings(%d, %d) = %v, want %v", tt.major, tt.minor, got, tt.expected)
+			}
+		})
+	}
+}
+
 // ---------------------------------------------------------------------------
 // swapLogsSchemaForCompat
 // ---------------------------------------------------------------------------

From 83df0485d9a173b618269d14e953bdbb4b37ea1c Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:18:12 -0500
Subject: [PATCH 20/27] remove unnecessary comments

---
 docker/otel-collector/schema/seed/00002_otel_logs.sql        | 3 ---
 docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql | 3 ---
 docker/otel-collector/schema/seed/00005_otel_traces.sql      | 3 ---
 3 files changed, 9 deletions(-)

diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql
index f555e7865..12ec3fbe3 100644
--- a/docker/otel-collector/schema/seed/00002_otel_logs.sql
+++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql
@@ -29,9 +29,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))),
   `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))),
   `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))),
-  -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
-  -- part's sparse index is 16x smaller, which makes reading filter values
-  -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
   INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
   INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
index ac44c11f5..e14258ce0 100644
--- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
+++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql
@@ -20,9 +20,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
   `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)),
   `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)),
   INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-  -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
-  -- part's sparse index is 16x smaller, which makes reading filter values
-  -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
   INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
   INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql
index c019d050f..b3ad5b3a9 100644
--- a/docker/otel-collector/schema/seed/00005_otel_traces.sql
+++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql
@@ -27,9 +27,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
     `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(3)),
     `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))),
     `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))),
-    -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
-    -- part's sparse index is 16x smaller, which makes reading filter values
-    -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
     INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
     INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),

From 3f464697e9a5fdbc865ce6255a7848cdd6626899 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:28:39 -0500
Subject: [PATCH 21/27] change min dictionary block size

---
 packages/common-utils/src/core/metadata.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index afc21020f..de029f01f 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -87,7 +87,7 @@ const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 // Smallest dictionary block size that gets one read per key; see
 // getMapTextIndexKeyValues.
-const PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE = 4096;
+const PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE = 2048;
 
 /**
  * The index read could not be scoped to the requested date range, so it was

From 001b1122ce284766ca4de15ca58fcc3359266d28 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:41:38 -0500
Subject: [PATCH 22/27] fix: scale the text index token cap by the parts read

---
 .../src/__tests__/metadata.int.test.ts        | 95 +++++++++++++++++++
 .../src/__tests__/metadata.test.ts            | 31 +++++-
 packages/common-utils/src/core/metadata.ts    | 19 +++-
 3 files changed, 138 insertions(+), 7 deletions(-)

diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 1fd29056c..29bdd15cf 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -1757,4 +1757,99 @@ describe('Metadata Integration Tests', () => {
       });
     },
   );
+
+  describe('getAllKeyValues - values repeated across parts', () => {
+    // mergeTreeTextIndex returns one row per part and token, so a token cap
+    // that ignores the part count stops before the later parts' values. Each
+    // part holds the same SHARED values plus OWN values no other part has.
+    const tableName = 'test_text_index_values_across_parts';
+    const PARTS = 25;
+    const SHARED = 1000;
+    const OWN = 100;
+    const dateRange = (): [Date, Date] => [
+      new Date(Date.now() - 60 * 60 * 1000),
+      new Date(Date.now() + 60 * 1000),
+    ];
+    let textIndexSupported = false;
+
+    beforeAll(async () => {
+      const probe = new Metadata(hdxClient, new MetadataCache());
+      textIndexSupported = supportsMergeTreeTextIndex(
+        await probe.getServerVersion({ connectionId: 'test_connection' }),
+      );
+      if (!textIndexSupported) return;
+
+      await client.command({
+        query: `CREATE OR REPLACE TABLE default.${tableName} (
+            Timestamp DateTime64(9),
+            ResourceAttributes Map(LowCardinality(String), String),
+            ResourceAttributeItems Array(String) MATERIALIZED
+              arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')),
+            Tags Array(String),
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1,
+            INDEX idx_tags Tags TYPE text(tokenizer = 'array') GRANULARITY 1
+          )
+          ENGINE = MergeTree()
+          PARTITION BY toDate(Timestamp)
+          ORDER BY Timestamp
+          SETTINGS text_index_dictionary_block_size = 8192
+        `,
+      });
+      await client.command({
+        query: `SYSTEM STOP MERGES default.${tableName}`,
+      });
+      for (let i = 0; i < PARTS; i++) {
+        await client.command({
+          query: `INSERT INTO default.${tableName} (Timestamp, ResourceAttributes, Tags)
+            SELECT now64(9) - INTERVAL 10 MINUTE, map('pod', v), [v]
+            FROM (
+              SELECT if(number < ${SHARED}, concat('shared-', toString(number)), concat('own-${i}-', toString(number))) AS v
+              FROM numbers(${SHARED + OWN})
+            )`,
+        });
+      }
+    });
+
+    afterAll(async () => {
+      await client.command({
+        query: `DROP TABLE IF EXISTS default.${tableName}`,
+      });
+    });
+
+    it.each([
+      ['Map key=value', "ResourceAttributes['pod']"],
+      ['plain-column', 'Tags'],
+    ])(
+      'returns every distinct value from the %s text index, with and without a part limit',
+      async (_, keyExpression) => {
+        if (!textIndexSupported) {
+          console.warn(
+            'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
+          );
+          return;
+        }
+
+        // PARTS is above the default limit of 20, so unset reads 20 parts and
+        // null reads all of them.
+        for (const [mapKeyIndexMaxParts, partsRead] of [
+          [undefined, 20],
+          [null, PARTS],
+        ] as const) {
+          const metadata = new Metadata(hdxClient, new MetadataCache());
+          const result = await metadata.getAllKeyValues({
+            databaseName: 'default',
+            tableName,
+            keyExpressions: [keyExpression],
+            maxValuesPerKey: 10000,
+            mapKeyIndexMaxParts,
+            connectionId: 'test_connection',
+            dateRange: dateRange(),
+            timestampValueExpression: 'Timestamp',
+          });
+          expect(result).toHaveLength(1);
+          expect(result[0].value).toHaveLength(SHARED + OWN * partsRead);
+        }
+      },
+    );
+  });
 });
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index a1c8ef012..70145ef44 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1382,7 +1382,8 @@ describe('Metadata', () => {
 
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('topKWeighted(');
-      expect(sql).toContain('LIMIT 20000');
+      // 20,000 tokens for each of the default 20 parts.
+      expect(sql).toContain('LIMIT 400000');
       expect(sql).not.toContain('startsWith(token,');
     });
 
@@ -1436,12 +1437,38 @@ describe('Metadata', () => {
       expect(branches[0]).toContain("'requestId' AS key");
       expect(branches[1]).toContain("startsWith(token, 'region=')");
       for (const branch of branches) {
-        expect(branch).toContain('LIMIT 20000');
+        // 20,000 tokens for each of the default 20 parts.
+        expect(branch).toContain('LIMIT 400000');
       }
       expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
       expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
     });
 
+    it.each([
+      ['map KV', "LogAttributes['requestId']"],
+      ['native', 'TraceId'],
+    ])(
+      'scales the %s text index token cap by the parts read, and drops it when every part is read',
+      async (_, keyExpression) => {
+        setupDefaultLogsSchema();
+        withDictionaryBlockSize(8192);
+
+        for (const mapKeyIndexMaxParts of [5, null]) {
+          await metadata.getAllKeyValues({
+            ...baseArgs,
+            keyExpressions: [keyExpression],
+            mapKeyIndexMaxParts,
+          });
+        }
+
+        const tokenLimits = (mockClickhouseClient.query as jest.Mock).mock.calls
+          .map(([opts]) => opts.query as string)
+          .filter(q => q.includes('mergeTreeTextIndex('))
+          .map(q => q.match(/LIMIT (\d+)/)?.[1]);
+        expect(tokenLimits).toEqual(['100000', undefined]);
+      },
+    );
+
     it('reads all Map keys in one pass when the index has small dictionary blocks', async () => {
       setupDefaultLogsSchema();
       withDictionaryBlockSize(512);
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index de029f01f..be0d51d79 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -79,10 +79,19 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20;
 // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle.
 export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 
-// Tokens a text index values lookup reads per key. The dictionary is sorted,
-// so a key with more distinct values than this gets the most frequent among
-// its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
+// Distinct values a text index values lookup reads per key. The dictionary is
+// sorted, so a key with more than this gets the most frequent among its
+// alphabetically first ones. Keep it above "Load more"'s 10,000 values.
 const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
+
+// mergeTreeTextIndex returns one row per part and token, so a value in every
+// part read takes maxParts rows; scaling the LIMIT by maxParts keeps room for
+// TEXT_INDEX_TOKENS_PER_KEY distinct values. A read of every part ("Load
+// more") isn't capped.
+const textIndexTokenLimit = (maxParts: number | null): ChSql =>
+  maxParts == null
+    ? chSql``
+    : chSql`LIMIT ${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(TEXT_INDEX_TOKENS_PER_KEY * maxParts, 'maxParts') }}`;
 // Keys or columns per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 // Smallest dictionary block size that gets one read per key; see
@@ -1865,7 +1874,7 @@ export class Metadata {
           WHERE ${partsFilter}
             AND startsWith(token, ${keyPrefix(k)})
             AND v != ''
-          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
+          ${textIndexTokenLimit(maxParts)}
         )`);
           }
         }
@@ -1961,7 +1970,7 @@ export class Metadata {
           FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
           WHERE ${partsFilter}
             AND token != ''
-          LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
+          ${textIndexTokenLimit(maxParts)}
         )`;
           sqlBranches.push(sql);
         }

From 46f0eadc54e2b4034441dad6be338cc63b9e8d5d Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:44:36 -0500
Subject: [PATCH 23/27] feat: let operators opt out of the text index block
 size ALTER

---
 .changeset/text-index-dictionary-block-size.md  |  6 ++++--
 docker-compose.dev.yml                          |  2 ++
 packages/otel-collector/cmd/migrate/main.go     | 10 ++++++++--
 .../otel-collector/cmd/migrate/main_test.go     | 17 +++++++++++++++++
 4 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md
index 35c8d3f1a..e6331b58b 100644
--- a/.changeset/text-index-dictionary-block-size.md
+++ b/.changeset/text-index-dictionary-block-size.md
@@ -15,5 +15,7 @@ made searches for a common value 10-30% faster.
 
 Because it's an `ALTER`, existing installs get it too: new parts use 8,192-token
 blocks, and existing parts keep theirs until they're merged. A table that
-already sets `text_index_dictionary_block_size` is left alone. ClickHouse
-26.2–26.5 doesn't have the setting, so those servers keep 512-token blocks.
+already sets `text_index_dictionary_block_size` is left alone, and operators who
+manage these tables themselves can turn the change off with
+`HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false`. ClickHouse 26.2–26.5
+doesn't have the setting, so those servers keep 512-token blocks.
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 335054d47..b1bba4f79 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -45,6 +45,8 @@ services:
       #   HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL
       # Apply TTL changes to already-existing tables too (default false):
       #   HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true'
+      # Leave text index dictionary block sizes alone (default: set 8192 on 26.6+):
+      #   HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE: 'false'
       ENABLE_PROMQL: 'true'
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go
index 7dcc39a84..6d680adf6 100644
--- a/packages/otel-collector/cmd/migrate/main.go
+++ b/packages/otel-collector/cmd/migrate/main.go
@@ -43,6 +43,9 @@ type Config struct {
 	// Reconcile TTL on already-existing tables, not just newly created ones.
 	ReconcileTableTTL bool
 
+	// Set 8,192-token text index dictionary blocks on the seeded tables.
+	SetTextIndexBlockSize bool
+
 	// TLS settings
 	TLSCAFile             string
 	TLSCertFile           string
@@ -69,6 +72,7 @@ func loadConfig() (*Config, error) {
 		MetricsTTL:            getEnv("HYPERDX_OTEL_EXPORTER_METRICS_TTL", tablesTTL),
 		SessionsTTL:           getEnv("HYPERDX_OTEL_EXPORTER_SESSIONS_TTL", tablesTTL),
 		ReconcileTableTTL:     getEnv("HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL", "false") == "true",
+		SetTextIndexBlockSize: getEnv("HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE", "true") != "false",
 		TLSCAFile:             getEnv("CLICKHOUSE_TLS_CA_FILE", ""),
 		TLSCertFile:           getEnv("CLICKHOUSE_TLS_CERT_FILE", ""),
 		TLSKeyFile:            getEnv("CLICKHOUSE_TLS_KEY_FILE", ""),
@@ -919,8 +923,10 @@ func main() {
 		os.Exit(1)
 	}
 
-	// Non-fatal so it can't block startup; older servers keep 512-token blocks.
-	if supportsTextIndexTableSettings(chMajor, chMinor) {
+	// On by default; HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false lets
+	// operators who manage these tables opt out. Non-fatal so it can't block
+	// startup; older servers keep 512-token blocks.
+	if cfg.SetTextIndexBlockSize && supportsTextIndexTableSettings(chMajor, chMinor) {
 		if err := setTextIndexBlockSize(ctx, db, cfg.Database); err != nil {
 			log.Printf("WARNING: text index block size: %v", err)
 		}
diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go
index 605f039e7..6e2fea462 100644
--- a/packages/otel-collector/cmd/migrate/main_test.go
+++ b/packages/otel-collector/cmd/migrate/main_test.go
@@ -111,6 +111,9 @@ func TestLoadConfig(t *testing.T) {
 		if cfg.ReconcileTableTTL {
 			t.Error("ReconcileTableTTL should default to false")
 		}
+		if !cfg.SetTextIndexBlockSize {
+			t.Error("SetTextIndexBlockSize should default to true")
+		}
 		if cfg.SchemaDir != dir {
 			t.Errorf("SchemaDir: got %q, want %q", cfg.SchemaDir, dir)
 		}
@@ -629,6 +632,20 @@ func TestProcessSchemaDir_NonexistentDir(t *testing.T) {
 // per-signal TTL resolution + reconcile helpers
 // ---------------------------------------------------------------------------
 
+func TestLoadConfig_TextIndexBlockSizeOptOut(t *testing.T) {
+	dir := t.TempDir()
+	os.Args = []string{"migrate", dir}
+	t.Setenv("HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE", "false")
+
+	cfg, err := loadConfig()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if cfg.SetTextIndexBlockSize {
+		t.Error("SetTextIndexBlockSize should be false")
+	}
+}
+
 func TestLoadConfig_PerSignalTTL(t *testing.T) {
 	dir := t.TempDir()
 	os.Args = []string{"migrate", dir}

From 797e28f2b1a8a083b5f39b59a7cd8f877fa79161 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:45:38 -0500
Subject: [PATCH 24/27] docs: say 2048 for the per-key text index threshold

---
 .changeset/topk-text-index-values.md                 | 2 +-
 packages/common-utils/src/__tests__/metadata.test.ts | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md
index 7ab43daeb..3565b3d1c 100644
--- a/.changeset/topk-text-index-values.md
+++ b/.changeset/topk-text-index-values.md
@@ -5,7 +5,7 @@
 perf: read large-block Map text index values per key, most frequent first
 
 "All values" lookups on Map key=value text indexes with large dictionary blocks
-(4096 or more, e.g. `text_index_dictionary_block_size = 8192`, which the
+(2048 or more, e.g. `text_index_dictionary_block_size = 8192`, which the
 collector's migration now sets on ClickHouse 26.6+)
 now read each key separately, in parallel queries of at most 10 keys, and stop
 after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs)
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 70145ef44..e6fb60280 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1609,6 +1609,9 @@ describe('Metadata', () => {
 
     it.each([
       [8192, [10, 10, 5]],
+      // The per-key threshold is 2048.
+      [2048, [10, 10, 5]],
+      [1024, [1, 1, 1]],
       [512, [1, 1, 1]],
     ])(
       'batches Map text index values by dictionary block size (%d)',

From 757f2ea1e60f7e593cd11b5f22fffcb5b4bc0590 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 14:53:35 -0500
Subject: [PATCH 25/27] perf: keep the text index token cap at 20,000 outside
 "Load more"

---
 .../src/__tests__/metadata.int.test.ts        | 40 ++++++++-----------
 .../src/__tests__/metadata.test.ts            | 10 ++---
 packages/common-utils/src/core/metadata.ts    | 14 +++----
 3 files changed, 28 insertions(+), 36 deletions(-)

diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 29bdd15cf..f5004bae7 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -1760,8 +1760,8 @@ describe('Metadata Integration Tests', () => {
 
   describe('getAllKeyValues - values repeated across parts', () => {
     // mergeTreeTextIndex returns one row per part and token, so a token cap
-    // that ignores the part count stops before the later parts' values. Each
-    // part holds the same SHARED values plus OWN values no other part has.
+    // stops before the later parts' values. Each part holds the same SHARED
+    // values plus OWN values no other part has.
     const tableName = 'test_text_index_values_across_parts';
     const PARTS = 25;
     const SHARED = 1000;
@@ -1820,7 +1820,7 @@ describe('Metadata Integration Tests', () => {
       ['Map key=value', "ResourceAttributes['pod']"],
       ['plain-column', 'Tags'],
     ])(
-      'returns every distinct value from the %s text index, with and without a part limit',
+      'returns every distinct value from the %s text index for "Load more"',
       async (_, keyExpression) => {
         if (!textIndexSupported) {
           console.warn(
@@ -1829,26 +1829,20 @@ describe('Metadata Integration Tests', () => {
           return;
         }
 
-        // PARTS is above the default limit of 20, so unset reads 20 parts and
-        // null reads all of them.
-        for (const [mapKeyIndexMaxParts, partsRead] of [
-          [undefined, 20],
-          [null, PARTS],
-        ] as const) {
-          const metadata = new Metadata(hdxClient, new MetadataCache());
-          const result = await metadata.getAllKeyValues({
-            databaseName: 'default',
-            tableName,
-            keyExpressions: [keyExpression],
-            maxValuesPerKey: 10000,
-            mapKeyIndexMaxParts,
-            connectionId: 'test_connection',
-            dateRange: dateRange(),
-            timestampValueExpression: 'Timestamp',
-          });
-          expect(result).toHaveLength(1);
-          expect(result[0].value).toHaveLength(SHARED + OWN * partsRead);
-        }
+        // "Load more" passes null: every part, no token cap.
+        const metadata = new Metadata(hdxClient, new MetadataCache());
+        const result = await metadata.getAllKeyValues({
+          databaseName: 'default',
+          tableName,
+          keyExpressions: [keyExpression],
+          maxValuesPerKey: 10000,
+          mapKeyIndexMaxParts: null,
+          connectionId: 'test_connection',
+          dateRange: dateRange(),
+          timestampValueExpression: 'Timestamp',
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0].value).toHaveLength(SHARED + OWN * PARTS);
       },
     );
   });
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e6fb60280..185282670 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1382,8 +1382,7 @@ describe('Metadata', () => {
 
       expect(sql).toContain('mergeTreeTextIndex(');
       expect(sql).toContain('topKWeighted(');
-      // 20,000 tokens for each of the default 20 parts.
-      expect(sql).toContain('LIMIT 400000');
+      expect(sql).toContain('LIMIT 20000');
       expect(sql).not.toContain('startsWith(token,');
     });
 
@@ -1437,8 +1436,7 @@ describe('Metadata', () => {
       expect(branches[0]).toContain("'requestId' AS key");
       expect(branches[1]).toContain("startsWith(token, 'region=')");
       for (const branch of branches) {
-        // 20,000 tokens for each of the default 20 parts.
-        expect(branch).toContain('LIMIT 400000');
+        expect(branch).toContain('LIMIT 20000');
       }
       expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
       expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
@@ -1448,7 +1446,7 @@ describe('Metadata', () => {
       ['map KV', "LogAttributes['requestId']"],
       ['native', 'TraceId'],
     ])(
-      'scales the %s text index token cap by the parts read, and drops it when every part is read',
+      'caps the %s text index read per key, and drops the cap when every part is read',
       async (_, keyExpression) => {
         setupDefaultLogsSchema();
         withDictionaryBlockSize(8192);
@@ -1465,7 +1463,7 @@ describe('Metadata', () => {
           .map(([opts]) => opts.query as string)
           .filter(q => q.includes('mergeTreeTextIndex('))
           .map(q => q.match(/LIMIT (\d+)/)?.[1]);
-        expect(tokenLimits).toEqual(['100000', undefined]);
+        expect(tokenLimits).toEqual(['20000', undefined]);
       },
     );
 
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index be0d51d79..f09979418 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -79,19 +79,19 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20;
 // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle.
 export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
 
-// Distinct values a text index values lookup reads per key. The dictionary is
+// Index rows a text index values lookup reads per key. The dictionary is
 // sorted, so a key with more than this gets the most frequent among its
-// alphabetically first ones. Keep it above "Load more"'s 10,000 values.
+// alphabetically first ones.
 const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
 
-// mergeTreeTextIndex returns one row per part and token, so a value in every
-// part read takes maxParts rows; scaling the LIMIT by maxParts keeps room for
-// TEXT_INDEX_TOKENS_PER_KEY distinct values. A read of every part ("Load
-// more") isn't capped.
+// mergeTreeTextIndex returns one row per part and token, so the cap counts a
+// value once per part it's in, and values only in parts read after it are
+// missed. That sample is enough for the top values of a filter, but "Load
+// more" (no part limit) wants every value, so its read isn't capped.
 const textIndexTokenLimit = (maxParts: number | null): ChSql =>
   maxParts == null
     ? chSql``
-    : chSql`LIMIT ${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(TEXT_INDEX_TOKENS_PER_KEY * maxParts, 'maxParts') }}`;
+    : chSql`LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}`;
 // Keys or columns per text index values query; see dispatchKeyValueQueries.
 const TEXT_INDEX_KEYS_PER_QUERY = 10;
 // Smallest dictionary block size that gets one read per key; see

From cb34879d6bfd2071872cc2fceeefcd9a2151ef10 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 15:24:57 -0500
Subject: [PATCH 26/27] perf: look up Map text index block sizes in parallel

---
 .../src/__tests__/metadata.test.ts            | 46 +++++++++++++++++++
 packages/common-utils/src/core/metadata.ts    | 23 ++++++----
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 185282670..08e640d14 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -2217,6 +2217,52 @@ describe('Metadata', () => {
         value: ['r1'],
       };
 
+      it("looks up every Map column's dictionary block size at once", async () => {
+        setupDefaultLogsSchema();
+        const kvIndex = (mapColumn: string, indexName: string) => ({
+          kv: {
+            columnName: mapColumn,
+            mapColumn,
+            indexName,
+            separator: '=',
+            useHasAny: false,
+          },
+        });
+        jest.spyOn(metadata, 'getMapColumnTextIndexes').mockResolvedValue(
+          new Map([
+            ['LogAttributes', kvIndex('LogAttributes', 'idx_log_attr_items')],
+            [
+              'ResourceAttributes',
+              kvIndex('ResourceAttributes', 'idx_res_attr_items'),
+            ],
+          ]),
+        );
+        let releaseSettings: () => void = () => {};
+        const settings = jest
+          .spyOn(metadata, 'getMergeTreeSettings')
+          .mockReturnValue(
+            new Promise(resolve => {
+              releaseSettings = () => resolve(undefined);
+            }),
+          );
+
+        const result = metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: [
+            "LogAttributes['requestId']",
+            "ResourceAttributes['service.name']",
+          ],
+        });
+        // Both lookups start before either finishes.
+        for (let i = 0; i < 50 && settings.mock.calls.length < 2; i++) {
+          await new Promise(resolve => setImmediate(resolve));
+        }
+        expect(settings).toHaveBeenCalledTimes(2);
+
+        releaseSettings();
+        await result;
+      });
+
       it('yields each query as it finishes, without waiting for the slowest', async () => {
         setupDefaultLogsSchema();
         let releaseMapTextIndex: () => void = () => {};
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index f09979418..b77e667d3 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -3141,14 +3141,21 @@ export class Metadata {
     // high-cardinality key holds up only its own batch's values. Within a
     // batch, large dictionary blocks get one read per key; see
     // getMapTextIndexKeyValues.
-    for (const [column, options] of mapTextIndexQueryOptions) {
-      const perKey =
-        (await this.getTextIndexDictionaryBlockSize({
-          databaseName,
-          tableName,
-          connectionId,
-          indexName: options.indexName,
-        })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE;
+    // Every Map column's block size is looked up at once, not one at a time.
+    const mapColumns = await Promise.all(
+      [...mapTextIndexQueryOptions].map(async ([column, options]) => ({
+        column,
+        options,
+        perKey:
+          (await this.getTextIndexDictionaryBlockSize({
+            databaseName,
+            tableName,
+            connectionId,
+            indexName: options.indexName,
+          })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE,
+      })),
+    );
+    for (const { column, options, perKey } of mapColumns) {
       for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) {
         promises.push(
           this.getMapTextIndexKeyValues({

From 46be94f625ffc92b27dd59cd98e0234430af1b6d Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 13 Nov 2026 15:30:19 -0500
Subject: [PATCH 27/27] test: set the block size per index so fixtures run
 before ClickHouse 26.6

---
 .../src/__tests__/kvItemsRewrite.int.test.ts     |  5 ++---
 .../src/__tests__/metadata.int.test.ts           | 16 ++++++----------
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
index fd75a190a..7a59955b6 100644
--- a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
+++ b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts
@@ -70,11 +70,10 @@ describe('KV items rewrite integration', () => {
         LogAttributeItems Array(String) MATERIALIZED
           arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
           CODEC(ZSTD(1)),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
       )
       ENGINE = MergeTree
-      ORDER BY (Timestamp)
-      SETTINGS text_index_dictionary_block_size = 8192`,
+      ORDER BY (Timestamp)`,
     });
 
     await client.insert({
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index f5004bae7..5d782bc87 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -697,11 +697,10 @@ describe('Metadata Integration Tests', () => {
             ResourceAttributeItems Array(String) MATERIALIZED
               arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))
               CODEC(ZSTD(1)),
-            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
-          SETTINGS text_index_dictionary_block_size = 8192
         `,
       });
 
@@ -713,11 +712,10 @@ describe('Metadata Integration Tests', () => {
             LogAttributeItems Array(String) MATERIALIZED
               arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
               CODEC(ZSTD(1)),
-            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
-          SETTINGS text_index_dictionary_block_size = 8192
         `,
       });
     });
@@ -1659,14 +1657,14 @@ describe('Metadata Integration Tests', () => {
     {
       path: 'key',
       tableName: 'test_map_keys_key_text_index_order',
-      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`,
+      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
     {
       path: 'kv',
       tableName: 'test_map_keys_kv_text_index_order',
       indexDefinition: `LogAttributeItems Array(String) MATERIALIZED
           arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`,
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
   ])(
     'getMapKeys - $path text index ordering',
@@ -1690,7 +1688,6 @@ describe('Metadata Integration Tests', () => {
             ENGINE = MergeTree()
             PARTITION BY toDate(Timestamp)
             ORDER BY Timestamp
-            SETTINGS text_index_dictionary_block_size = 8192
           `,
         });
 
@@ -1786,13 +1783,12 @@ describe('Metadata Integration Tests', () => {
             ResourceAttributeItems Array(String) MATERIALIZED
               arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')),
             Tags Array(String),
-            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1,
-            INDEX idx_tags Tags TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1,
+            INDEX idx_tags Tags TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           PARTITION BY toDate(Timestamp)
           ORDER BY Timestamp
-          SETTINGS text_index_dictionary_block_size = 8192
         `,
       });
       await client.command({