From efa31bc81be2300ab317beac6e1bf760f5d8b255 Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Wed, 4 Nov 2026 18:09:29 +0300 Subject: [PATCH 1/3] docs(mcp): recommend quoted Lucene values for exact matches and bare terms for body text Lucene field:value is a substring match and can't use the primary key; the quoted form field:"value" already compiles to =. Body:term and Body LIKE read the whole body column, while a bare term or phrase uses the token/text index. Say both in the query tool descriptions and the query guide. Refs #3538. --- .changeset/mcp-lucene-exact-match.md | 7 +++++++ .../src/mcp/__tests__/dashboards/prompts.test.ts | 7 ++++++- .../api/src/mcp/prompts/dashboards/content.ts | 15 +++++++++++---- packages/api/src/mcp/tools/query/schemas.ts | 9 +++++++-- 4 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 .changeset/mcp-lucene-exact-match.md diff --git a/.changeset/mcp-lucene-exact-match.md b/.changeset/mcp-lucene-exact-match.md new file mode 100644 index 0000000000..853bb32a84 --- /dev/null +++ b/.changeset/mcp-lucene-exact-match.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +docs: MCP tool descriptions and the query guide now recommend the quoted Lucene +form `field:"value"` for exact matches, and bare terms or phrases for log body +text, so agents keep the primary key and the text index diff --git a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts index 124e020cd6..00daf1c76b 100644 --- a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts @@ -175,8 +175,13 @@ describe('MCP Dashboard Prompts', () => { const luceneBody = prompt.slice(luceneIdx); expect(luceneBody).toMatch(/SUBSTRING MATCHING/); expect(luceneBody).toMatch(/ilike\(field, '%value%'\)/); - // The canonical fix is SQL with =. + // The canonical fix is SQL with =, or a quoted Lucene value. expect(luceneBody).toMatch(/SpanKind = 'Server'/); + expect(luceneBody).toMatch( + /SpanKind:"Server" compiles to SpanKind = 'Server'/, + ); + // Body text goes through the token/text index as a bare term, not Body:term. + expect(luceneBody).toMatch(/bare Lucene term or phrase/); // Wildcard limitation is documented alongside substring matching. expect(luceneBody).toMatch(/prefix-within-substring/); expect(luceneBody).toMatch(/LIKE 'api%'/); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index a45d9fcf89..c5ec1267f1 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -883,13 +883,19 @@ Lucene field:value does NOT mean exact equality. It translates to ilike(field, ' ServiceName:api* does NOT mean "starts with api". It means "contains a substring starting with api". In practice this often works for prefix matching, but it is NOT equivalent to SQL LIKE 'api%'. - For exact matching or true prefix/suffix/contains patterns, use SQL instead: + For exact matching in Lucene, quote the value: SpanKind:"Server" compiles to SpanKind = 'Server'. + + For true prefix/suffix/contains patterns, use SQL instead: Exact: whereLanguage: "sql", where: "SpanKind = 'Server'" Prefix: whereLanguage: "sql", where: "ServiceName LIKE 'api%'" - Contains: whereLanguage: "sql", where: "Body LIKE '%timeout%'" + Contains: whereLanguage: "sql", where: "StatusMessage LIKE '%timeout%'" Regex: whereLanguage: "sql", where: "match(ServiceName, '^api-v[0-9]+')" - When Lucene substring matching is FINE: free-text columns like Body, StatusMessage where substring is the intent. + For text in the log body, write a bare Lucene term or phrase (timeout, "connection refused"). It uses the + source's token/text index on the body; Body:timeout, Body LIKE and Body ILIKE read the whole column, and + Body:"..." is exact equality on the whole body. + + When Lucene substring matching is FINE: free-text columns like StatusMessage where substring is the intent. When Lucene substring matching is WRONG: enum-like columns (SpanKind, SeverityText, StatusCode, ServiceName) where you want exact or true prefix match. *** MAP-ATTRIBUTE GOTCHA *** @@ -1562,8 +1568,9 @@ Example: find top patterns for production services over the last 4 hours: Wrong: where: "SpanKind:Server" (whereLanguage: "lucene"; translates to ilike(SpanKind, '%Server%') and matches "ServerStreaming", "InternalServer", etc.) Wrong: where: "ServiceName:api*" (whereLanguage: "lucene"; NOT a true prefix match -- it is prefix-within-substring) Correct: where: "SpanKind = 'Server'" (whereLanguage: "sql"; exact match) + Correct: where: 'SpanKind:"Server"' (whereLanguage: "lucene"; a quoted value is an exact match) Correct: where: "ServiceName LIKE 'api%'" (whereLanguage: "sql"; true prefix match) - Lucene field:value is a substring match (ilike with surrounding wildcards), not exact equality. Lucene field:value* is prefix-within-substring, not SQL LIKE 'value%'. Fine for free-text columns (Body, StatusMessage); wrong for enum-like columns where exact match matters (SpanKind, SeverityText, StatusCode) and wrong when true prefix matching is needed. Use SQL for exact match (=), prefix (LIKE 'x%'), contains (LIKE '%x%'), or regex (match(col, 'pattern')). + Lucene field:value is a substring match (ilike with surrounding wildcards), not exact equality. Lucene field:value* is prefix-within-substring, not SQL LIKE 'value%'. Fine for free-text columns (StatusMessage; for Body use a bare term or phrase, which uses the index); wrong for enum-like columns where exact match matters (SpanKind, SeverityText, StatusCode) and wrong when true prefix matching is needed. Use SQL for exact match (=), prefix (LIKE 'x%'), contains (LIKE '%x%'), or regex (match(col, 'pattern')). 16. Forgetting that map-attribute values are often empty strings Wrong: groupBy on SpanAttributes['db.collection.name'] without filtering empty values; the result has an empty-string bucket alongside the real ones, which renders as a blank row in tables and an unlabelled slice in pies. diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index e3e70664b7..5f8fb9b463 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -20,7 +20,11 @@ const WHERE_DESCRIPTION = 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")\n\n' + 'SUBSTRING TRAP: Lucene field:value matches ANY row containing "value" as a substring, not exact equality.\n' + ' SpanKind:Server matches "Server", "ServerStreaming", "InternalServer", etc.\n' + - " For exact match, use SQL: SpanKind = 'Server'"; + ' For exact match, quote the value: SpanKind:"Server" (compiles to =, so a sort-key column\n' + + " such as ServiceName can use the primary key), or use SQL: SpanKind = 'Server'\n\n" + + 'BODY TEXT: write the term or phrase without a field name, e.g. timeout or "connection refused".\n' + + " That uses the source's token/text index on the body. Body:timeout, Body LIKE and Body ILIKE\n" + + ' read the whole column, and Body:"..." is exact equality on the whole body.'; const WHERE_LANGUAGE_DESCRIPTION = 'Language of the "where" filter. The syntax in "where" MUST match this setting.\n' + @@ -30,7 +34,8 @@ const WHERE_LANGUAGE_DESCRIPTION = 'and map attributes via dot notation. Use "sql" for IN(...) lists, complex expressions, or function calls.\n' + 'IMPORTANT: Lucene field:value is a SUBSTRING match (ilike), not exact equality. ' + 'field:val* is prefix-within-substring, not a true prefix match. ' + - "For exact matching or reliable wildcards, use SQL: WHERE field = 'value' or WHERE field LIKE 'val%'.\n\n" + + 'For exact matching, quote the value (field:"value") or use SQL: WHERE field = \'value\'. ' + + "For reliable wildcards, use SQL: WHERE field LIKE 'val%'.\n\n" + 'Common mistake: writing Column:value (Lucene) but setting whereLanguage to "sql". ' + 'If your filter uses colon syntax, use "lucene".'; From b2db87508b33ec0c10c2848948df4268d1b57a7d Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Wed, 4 Nov 2026 18:35:24 +0300 Subject: [PATCH 2/3] docs(mcp): fix the Lucene phrase and LIKE examples in the query guide, test the where descriptions The cheat sheet still showed Body:"connection refused" (exact equality on the whole body) and Body LIKE as examples. Use a bare phrase and StatusMessage LIKE, and assert the key phrases of the where / whereLanguage tool descriptions. --- packages/api/src/mcp/__tests__/query.test.ts | 16 ++++++++++++++++ .../api/src/mcp/prompts/dashboards/content.ts | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index 0855c40bc7..f8b572a277 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -28,6 +28,8 @@ import { getMetricSelectIssues, mcpQuantileLevelSchema, validateMetricSelectItems, + whereLanguageSchema, + whereSchema, } from '@/mcp/tools/query/schemas'; import { resolveOrderBy } from '@/mcp/tools/query/table'; import { getErrorCategory } from '@/mcp/utils/errors'; @@ -1414,3 +1416,17 @@ describe('mcpQuantileLevelSchema', () => { expect(optional.safeParse('0.97').success).toBe(false); }); }); + +describe('where / whereLanguage descriptions', () => { + it('steer exact matches to the quoted Lucene form and body text to bare terms', () => { + expect(whereSchema.description).toContain( + 'quote the value: SpanKind:"Server"', + ); + expect(whereSchema.description).toContain( + 'BODY TEXT: write the term or phrase without a field name', + ); + expect(whereLanguageSchema.description).toContain( + 'quote the value (field:"value")', + ); + }); +}); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index c5ec1267f1..f6179e0b5e 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -862,7 +862,7 @@ Used in the "where" field of select items and search tiles. OR: level:error OR level:fatal NOT: NOT level:debug Wildcard: ServiceName:front* - Phrase: Body:"connection refused" + Phrase: "connection refused" (body text, uses the text index) Exists: _exists_:http.route Range (numeric): Duration:>1000000000 Range (inclusive): http.status_code:[400 TO 499] @@ -929,7 +929,7 @@ Alternative to Lucene. Set whereLanguage: "sql" when using SQL syntax. Basic: SeverityText = 'error' AND/OR: ServiceName = 'api' AND StatusCode = 'STATUS_CODE_ERROR' IN: ServiceName IN ('api', 'web', 'worker') - LIKE: Body LIKE '%timeout%' + LIKE: StatusMessage LIKE '%timeout%' Comparison: Duration > 1000000000 Map access: SpanAttributes['http.status_code'] = '500' From d70c51f6dc56477ae68744ed24d915afbfcecd4e Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Wed, 4 Nov 2026 18:53:54 +0300 Subject: [PATCH 3/3] docs(mcp): say bare terms search the default column, match whole tokens with an index, and quoted values are case-sensitive --- packages/api/src/mcp/__tests__/query.test.ts | 2 +- packages/api/src/mcp/prompts/dashboards/content.ts | 13 ++++++++----- packages/api/src/mcp/tools/query/schemas.ts | 13 ++++++++----- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index f8b572a277..8d4ea7a421 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -1426,7 +1426,7 @@ describe('where / whereLanguage descriptions', () => { 'BODY TEXT: write the term or phrase without a field name', ); expect(whereLanguageSchema.description).toContain( - 'quote the value (field:"value")', + 'quote the value (field:"value", case-sensitive)', ); }); }); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index f6179e0b5e..159e391db6 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -862,7 +862,7 @@ Used in the "where" field of select items and search tiles. OR: level:error OR level:fatal NOT: NOT level:debug Wildcard: ServiceName:front* - Phrase: "connection refused" (body text, uses the text index) + Phrase: "connection refused" (default search column, e.g. the log body) Exists: _exists_:http.route Range (numeric): Duration:>1000000000 Range (inclusive): http.status_code:[400 TO 499] @@ -883,7 +883,8 @@ Lucene field:value does NOT mean exact equality. It translates to ilike(field, ' ServiceName:api* does NOT mean "starts with api". It means "contains a substring starting with api". In practice this often works for prefix matching, but it is NOT equivalent to SQL LIKE 'api%'. - For exact matching in Lucene, quote the value: SpanKind:"Server" compiles to SpanKind = 'Server'. + For exact matching in Lucene, quote the value: SpanKind:"Server" compiles to SpanKind = 'Server' + (case-sensitive, unlike the unquoted form). For true prefix/suffix/contains patterns, use SQL instead: Exact: whereLanguage: "sql", where: "SpanKind = 'Server'" @@ -891,9 +892,11 @@ Lucene field:value does NOT mean exact equality. It translates to ilike(field, ' Contains: whereLanguage: "sql", where: "StatusMessage LIKE '%timeout%'" Regex: whereLanguage: "sql", where: "match(ServiceName, '^api-v[0-9]+')" - For text in the log body, write a bare Lucene term or phrase (timeout, "connection refused"). It uses the - source's token/text index on the body; Body:timeout, Body LIKE and Body ILIKE read the whole column, and - Body:"..." is exact equality on the whole body. + For text in the log body, write a bare Lucene term or phrase (timeout, "connection refused"). It searches the + source's default search column (the body for logs) and uses its text index when the source has one. With an + index it matches whole tokens (timeout does not match ConnectionTimeoutException); for a substring inside a word, use + SQL ILIKE. Body:timeout, Body LIKE and Body ILIKE read the whole column, and Body:"..." is exact equality on + the whole body. When Lucene substring matching is FINE: free-text columns like StatusMessage where substring is the intent. When Lucene substring matching is WRONG: enum-like columns (SpanKind, SeverityText, StatusCode, ServiceName) where you want exact or true prefix match. diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 5f8fb9b463..b48dbe4603 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -20,11 +20,14 @@ const WHERE_DESCRIPTION = 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")\n\n' + 'SUBSTRING TRAP: Lucene field:value matches ANY row containing "value" as a substring, not exact equality.\n' + ' SpanKind:Server matches "Server", "ServerStreaming", "InternalServer", etc.\n' + - ' For exact match, quote the value: SpanKind:"Server" (compiles to =, so a sort-key column\n' + - " such as ServiceName can use the primary key), or use SQL: SpanKind = 'Server'\n\n" + + ' For exact match, quote the value: SpanKind:"Server" (compiles to a case-sensitive =, so a sort-key\n' + + " column such as ServiceName can use the primary key), or use SQL: SpanKind = 'Server'\n\n" + 'BODY TEXT: write the term or phrase without a field name, e.g. timeout or "connection refused".\n' + - " That uses the source's token/text index on the body. Body:timeout, Body LIKE and Body ILIKE\n" + - ' read the whole column, and Body:"..." is exact equality on the whole body.'; + " That searches the source's default search column (the body for logs) and uses its text index\n" + + ' when the source has one. With an index it matches whole tokens: timeout does not match ConnectionTimeoutException.\n' + + " For a substring inside a word, use SQL ILIKE '%timeout%' (reads the whole column).\n" + + ' Body:timeout, Body LIKE and Body ILIKE read the whole column, and Body:"..." is exact equality on the\n' + + ' whole body.'; const WHERE_LANGUAGE_DESCRIPTION = 'Language of the "where" filter. The syntax in "where" MUST match this setting.\n' + @@ -34,7 +37,7 @@ const WHERE_LANGUAGE_DESCRIPTION = 'and map attributes via dot notation. Use "sql" for IN(...) lists, complex expressions, or function calls.\n' + 'IMPORTANT: Lucene field:value is a SUBSTRING match (ilike), not exact equality. ' + 'field:val* is prefix-within-substring, not a true prefix match. ' + - 'For exact matching, quote the value (field:"value") or use SQL: WHERE field = \'value\'. ' + + 'For exact matching, quote the value (field:"value", case-sensitive) or use SQL: WHERE field = \'value\'. ' + "For reliable wildcards, use SQL: WHERE field LIKE 'val%'.\n\n" + 'Common mistake: writing Column:value (Lucene) but setting whereLanguage to "sql". ' + 'If your filter uses colon syntax, use "lucene".';