From 92a33756cc52a1da4bcc94cb3cd2d060151096f4 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:28:08 -0500 Subject: [PATCH] feat(api): support ad hoc dashboard filters in the external API The v2 dashboards API now creates, reads, and updates ADHOC filters (SQL and PromQL source types) and their adhoc saved conditions. Sources and appliesToSourceIds must exist and match the filter's sourceType. MCP keeps hiding ad hoc filters via convertToExternalDashboard's hideAdhoc option. --- .changeset/adhoc-filters-external-api.md | 10 + packages/api/openapi.json | 178 +++++++++++++++++- .../src/mcp/tools/dashboards/getDashboard.ts | 1 + .../mcp/tools/dashboards/getDashboardTile.ts | 1 + .../mcp/tools/dashboards/patchDashboard.ts | 6 +- .../api/src/mcp/tools/dashboards/queryTile.ts | 1 + .../src/mcp/tools/dashboards/queryTiles.ts | 1 + .../src/mcp/tools/dashboards/saveDashboard.ts | 2 + .../src/mcp/tools/dashboards/validation.ts | 5 +- .../__tests__/dashboards.int.test.ts | 161 +++++++++++++++- .../src/routers/external-api/v2/dashboards.ts | 141 +++++++++++++- .../v2/utils/__tests__/dashboards.test.ts | 37 ++++ .../external-api/v2/utils/dashboards.ts | 68 +++++-- packages/api/src/utils/externalApi.ts | 28 ++- packages/api/src/utils/zod.ts | 18 +- 15 files changed, 608 insertions(+), 50 deletions(-) create mode 100644 .changeset/adhoc-filters-external-api.md diff --git a/.changeset/adhoc-filters-external-api.md b/.changeset/adhoc-filters-external-api.md new file mode 100644 index 0000000000..9edb98ee56 --- /dev/null +++ b/.changeset/adhoc-filters-external-api.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': minor +--- + +feat: Create, read, and update ad hoc dashboard filters through the external API + +`/api/v2/dashboards` now accepts and returns `ADHOC` filters, with SQL and +PromQL source types, and their `adhoc` entries in `savedFilterValues`. Each +filter's sources must exist and be a kind its `sourceType` supports. MCP still +hides ad hoc filters. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3faea..55f996c55a 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -959,13 +959,16 @@ "description": "Query language for the where clause." }, "SavedFilterValue": { - "description": "A single saved dashboard filter selection. Either a rendered SQL condition, or a selection addressed by the name of the dashboard variable it belongs to.\n", + "description": "A single saved dashboard filter selection. Either a rendered SQL condition, a selection addressed by the name of the dashboard variable it belongs to, or the conditions of an ADHOC filter.\n", "oneOf": [ { "$ref": "#/components/schemas/SqlSavedFilterValue" }, { "$ref": "#/components/schemas/VariableSavedFilterValue" + }, + { + "$ref": "#/components/schemas/AdhocSavedFilterValue" } ] }, @@ -1029,6 +1032,78 @@ } } }, + "AdhocSavedFilterValue": { + "type": "object", + "required": [ + "type", + "name", + "conditions" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "adhoc" + ], + "description": "Filter type.", + "example": "adhoc" + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "description": "The variableName of the ADHOC filter these conditions belong to.", + "example": "conditions" + }, + "conditions": { + "type": "array", + "maxItems": 100, + "description": "The filter's conditions.", + "items": { + "$ref": "#/components/schemas/AdhocFilterCondition" + } + } + } + }, + "AdhocFilterCondition": { + "type": "object", + "description": "One \"key operator value\" condition. Conditions are AND'd, except that IN (sql) and =| (promql) conditions sharing a key are OR'd into one.\n", + "required": [ + "key", + "operator", + "value" + ], + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "A SQL expression (column or map access) for sql filters, or a label name for promql filters.", + "example": "ServiceName" + }, + "operator": { + "type": "string", + "enum": [ + "=", + "!=", + "=~", + "!~", + "LIKE", + "NOT LIKE", + "IN", + "=|" + ], + "description": "Comparison operator. =~ and !~ match a regular expression. LIKE, NOT LIKE, and IN apply to sql filters only, and =| to promql filters only; a condition its filter's sourceType can't express is not applied.\n", + "example": "=" + }, + "value": { + "type": "string", + "maxLength": 10000, + "description": "Value, pattern, or regular expression to compare against.", + "example": "api" + } + } + }, "MetricDataType": { "type": "string", "enum": [ @@ -3951,6 +4026,9 @@ }, { "$ref": "#/components/schemas/PrometheusLabelFilterInput" + }, + { + "$ref": "#/components/schemas/AdhocFilterInput" } ], "discriminator": { @@ -3958,7 +4036,8 @@ "mapping": { "QUERY_EXPRESSION": "#/components/schemas/QueryExpressionFilterInput", "STATIC_LIST": "#/components/schemas/StaticListFilterInput", - "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput" + "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", + "ADHOC": "#/components/schemas/AdhocFilterInput" } } }, @@ -4262,6 +4341,101 @@ } } }, + "AdhocFilterInput": { + "type": "object", + "description": "A filter holding free-form \"key operator value\" conditions, with keys\nand values suggested from its sources: columns and map keys for sql\nsources, labels for promql sources. Its conditions are always\nbroadcast to the tiles it applies to: as WHERE conditions on builder\ntiles and raw sql tiles using the $__filters macro, or as label\nmatchers on PromQL tiles. Its conditions are saved in\nsavedFilterValues as an \"adhoc\" entry. Ad hoc filters are a preview\nfeature.\n", + "required": [ + "type", + "name", + "sourceType", + "sources" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "ADHOC" + ], + "description": "Discriminator. Must be \"ADHOC\".", + "example": "ADHOC" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for the dashboard filter", + "example": "Conditions" + }, + "sourceType": { + "type": "string", + "enum": [ + "sql", + "promql" + ], + "description": "The kind of sources the filter reads keys from and applies to: Log,\nTrace, or Session sources for sql, PromQL sources for promql.\n", + "example": "sql" + }, + "sources": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + }, + "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType.", + "example": [ + "65f5e4a3b9e77c001a111111" + ] + }, + "where": { + "type": "string", + "description": "Optional condition narrowing the rows keys and values are suggested\nfrom. sql filters only; never applied to tiles. May reference the\ndashboard's variables.\n", + "example": "ServiceName:api" + }, + "whereLanguage": { + "type": "string", + "enum": [ + "sql", + "lucene" + ], + "description": "Language of the where condition", + "example": "lucene" + }, + "appliesToSourceIds": { + "type": "array", + "items": { + "type": "string" + }, + "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType. Omit or send an empty array to apply to the tiles of\nthe filter's sources.\n", + "example": [ + "65f5e4a3b9e77c001a111111" + ] + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true or omitted.", + "example": true + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true or omitted.", + "example": true + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Unique name the filter's conditions are saved under, in\nsavedFilterValues and the dashboard URL. Tiles can't reference it\nas a variable. Defaults to the display name with whitespace\nreplaced by underscores and remaining illegal characters removed.\nMust be unique across a dashboard's variable-enabled filters.\n", + "example": "conditions" + } + } + }, "Filter": { "allOf": [ { diff --git a/packages/api/src/mcp/tools/dashboards/getDashboard.ts b/packages/api/src/mcp/tools/dashboards/getDashboard.ts index a18f09952b..2133ec09b3 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboard.ts @@ -63,6 +63,7 @@ export function registerGetDashboard({ } const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts index f9a4d2632f..7d331d965c 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts @@ -51,6 +51,7 @@ export function registerGetDashboardTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const tile = externalDashboard.tiles.find(t => t.id === tileId); if (!tile) { diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index aeea54d7bd..4488ca1c7b 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -15,7 +15,7 @@ import { isConfigTile, validateDashboardTiles, } from '@/routers/external-api/v2/utils/dashboards'; -import { getExternalCompatibleFilterValues } from '@/utils/externalApi'; +import { getMcpCompatibleFilterValues } from '@/utils/externalApi'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; @@ -127,7 +127,7 @@ export function registerPatchDashboard({ // Convert only for the error message (read-only, no write-back). const externalDashboard = convertToExternalDashboard( existingDashboard, - { hidePromql: !promqlEnabled }, + { hidePromql: !promqlEnabled, hideAdhoc: true }, ); return mcpUserError( `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => `${t.id} (${t.name})`).join(', ')}`, @@ -271,7 +271,7 @@ export function registerPatchDashboard({ tags: updatedDashboard.tags, ...(savedFilterValues !== undefined ? { - savedFilterValues: getExternalCompatibleFilterValues( + savedFilterValues: getMcpCompatibleFilterValues( updatedDashboard.savedFilterValues, ), } diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index b6cd385b57..0f9df6b116 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -78,6 +78,7 @@ export function registerQueryTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const tile = externalDashboard.tiles.find(t => t.id === tileId); if (!tile) { diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 44af35c950..9860ad768c 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -184,6 +184,7 @@ export function registerQueryTiles({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const allTiles = externalDashboard.tiles; diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index ba0c0d21e3..bde01373e8 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -236,6 +236,7 @@ async function createDashboard({ const externalDashboard = convertToExternalDashboard(newDashboard, { hidePromql: !promqlEnabled, + hideAdhoc: true, }); return { content: [ @@ -452,6 +453,7 @@ async function updateDashboard({ const externalDashboard = convertToExternalDashboard(updatedDashboard, { hidePromql: !promqlEnabled, + hideAdhoc: true, }); return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 987ab6ad2f..58d82da8de 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -18,6 +18,7 @@ import { TIME_RANGE_MACROS, } from '@hyperdx/common-utils/dist/macros'; import { + DashboardFilterValue, DisplayType, QueryExpressionDashboardFilter, SavedChartConfig, @@ -36,8 +37,6 @@ import { } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import type { McpSavedFilterValues } from './schemas'; - /** * Returns one entry per raw SQL tile that uses a source-dependent macro but * does not set a `sourceId`, recording which macro(s) triggered it so callers @@ -253,7 +252,7 @@ export function getFilterVariableWarnings( * applies. Warn so the agent can fix the name or add the filter. */ export function getSavedFilterValueWarnings( - savedFilterValues: McpSavedFilterValues | undefined, + savedFilterValues: DashboardFilterValue[] | undefined, filters: FilterForVariableDeclaration[] | undefined, ): string[] { if (!savedFilterValues?.length) return []; diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 93fea0e4d5..28eb69c859 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8456,9 +8456,16 @@ describe('External API v2 Dashboards - new format', () => { conditions: [{ key: 'ServiceName', operator: '=', value: 'api' }], }; - it('hides ad hoc filters and their saved values from GET', async () => { + const promqlAdhocFilter = () => ({ + ...adhocFilter(), + sourceType: 'promql' as const, + sources: [promqlSource._id.toString()], + }); + + it('returns ad hoc filters and their saved values from GET', async () => { + const filter = adhocFilter(); const dashboard = await createTestDashboard({ - filters: [adhocFilter()], + filters: [filter], savedFilterValues: [adhocValue], }); @@ -8467,25 +8474,163 @@ describe('External API v2 Dashboards - new format', () => { `${BASE_URL}/${dashboard._id}`, ).expect(200); - expect(response.body.data.filters).toEqual([]); - expect(response.body.data.savedFilterValues).toEqual([]); + expect(response.body.data.filters).toEqual([filter]); + expect(response.body.data.savedFilterValues).toEqual([adhocValue]); + }); + + it('creates SQL and PromQL ad hoc filters with saved conditions', async () => { + const promqlValue = { + type: 'adhoc' as const, + name: 'prom', + conditions: [{ key: 'job', operator: '=|', value: 'api' }], + }; + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [ + omit( + { ...adhocFilter(), appliesToSourceIds: [traceSource._id] }, + 'id', + ), + // The mode flags default when omitted. + omit( + { ...promqlAdhocFilter(), name: 'Prom', variableName: 'prom' }, + 'id', + 'isBroadcastEnabled', + 'isVariableEnabled', + ), + ], + savedFilterValues: [adhocValue, promqlValue], + }), + ) + .expect(200); + + expect(response.body.data.filters).toEqual([ + { + ...adhocFilter(), + id: expect.any(String), + appliesToSourceIds: [traceSource._id.toString()], + }, + { + ...promqlAdhocFilter(), + id: expect.any(String), + name: 'Prom', + variableName: 'prom', + }, + ]); + expect(response.body.data.savedFilterValues).toEqual([ + adhocValue, + promqlValue, + ]); + + const stored = await Dashboard.findById(response.body.data.id).lean(); + expect(stored?.filters?.[1]).toMatchObject({ + type: 'ADHOC', + sourceType: 'promql', + sources: [promqlSource._id.toString()], + isBroadcastEnabled: true, + }); + }); + + it('round-trips a GET response through PUT', async () => { + const dashboard = await createTestDashboard({ + filters: [adhocFilter()], + savedFilterValues: [adhocValue], + }); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + + const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send(omit(data, 'id')) + .expect(200); + + expect(response.body.data.filters).toEqual(data.filters); + expect(response.body.data.savedFilterValues).toEqual( + data.savedFilterValues, + ); }); - it('rejects an ad hoc filter in a POST', async () => { + it.each([ + ['broadcast disabled', { isBroadcastEnabled: false }], + ['variables disabled', { isVariableEnabled: false }], + ['no sources', { sources: [] }], + ['an invalid source ID', { sources: ['not-an-id'] }], + ['an unknown source type', { sourceType: 'lucene' }], + ['a selection limit', { maxSelections: 1 }], + ['the external sourceId field', { sourceId: new ObjectId().toString() }], + ])('rejects an ad hoc filter with %s', async (_label, overrides) => { await authRequest('post', BASE_URL) .send( createMockDashboard(traceSource._id.toString(), { - filters: [omit(adhocFilter(), 'id')], + filters: [omit({ ...adhocFilter(), ...overrides }, 'id')], }), ) .expect(400); }); - it('rejects ad hoc saved values in a POST', async () => { + it('rejects an ad hoc filter over a missing source', async () => { + const missing = new ObjectId().toString(); + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...adhocFilter(), sources: [missing] }, 'id')], + }), + ) + .expect(400); + + expect(response.body.message).toContain(missing); + }); + + it.each([ + ['a SQL filter over a PromQL source', () => adhocFilter(), 'promql'], + ['a SQL filter over a Metric source', () => adhocFilter(), 'metric'], + ['a PromQL filter over a Trace source', promqlAdhocFilter, 'trace'], + ] as const)('rejects %s', async (_label, makeFilter, kind) => { + const sourceId = { + promql: promqlSource, + metric: metricSource, + trace: traceSource, + }[kind]._id.toString(); + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...makeFilter(), sources: [sourceId] }, 'id')], + }), + ) + .expect(400); + + expect(response.body.message).toContain(sourceId); + }); + + it('rejects an ad hoc filter applying to a source of the wrong kind', async () => { + await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [ + omit( + { + ...adhocFilter(), + appliesToSourceIds: [promqlSource._id.toString()], + }, + 'id', + ), + ], + }), + ) + .expect(400); + }); + + it('rejects ad hoc saved values with an unknown operator', async () => { await authRequest('post', BASE_URL) .send( createMockDashboard(traceSource._id.toString(), { - savedFilterValues: [adhocValue], + savedFilterValues: [ + { + ...adhocValue, + conditions: [{ key: 'a', operator: '>', value: '1' }], + }, + ], }), ) .expect(400); diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 77d3b31361..5a37bc7fb1 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -57,11 +57,12 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * SavedFilterValue: * description: > * A single saved dashboard filter selection. Either a rendered SQL - * condition, or a selection addressed by the name of the dashboard - * variable it belongs to. + * condition, a selection addressed by the name of the dashboard + * variable it belongs to, or the conditions of an ADHOC filter. * oneOf: * - $ref: '#/components/schemas/SqlSavedFilterValue' * - $ref: '#/components/schemas/VariableSavedFilterValue' + * - $ref: '#/components/schemas/AdhocSavedFilterValue' * SqlSavedFilterValue: * type: object * required: [condition] @@ -102,6 +103,54 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * maxLength: 10000 * example: ["hdx-oss-dev-api"] + * AdhocSavedFilterValue: + * type: object + * required: [type, name, conditions] + * properties: + * type: + * type: string + * enum: [adhoc] + * description: Filter type. + * example: "adhoc" + * name: + * type: string + * minLength: 1 + * maxLength: 1024 + * description: The variableName of the ADHOC filter these conditions belong to. + * example: "conditions" + * conditions: + * type: array + * maxItems: 100 + * description: The filter's conditions. + * items: + * $ref: '#/components/schemas/AdhocFilterCondition' + * AdhocFilterCondition: + * type: object + * description: > + * One "key operator value" condition. Conditions are AND'd, except that + * IN (sql) and =| (promql) conditions sharing a key are OR'd into one. + * required: [key, operator, value] + * properties: + * key: + * type: string + * minLength: 1 + * maxLength: 10000 + * description: A SQL expression (column or map access) for sql filters, or a label name for promql filters. + * example: "ServiceName" + * operator: + * type: string + * enum: ["=", "!=", "=~", "!~", "LIKE", "NOT LIKE", "IN", "=|"] + * description: > + * Comparison operator. =~ and !~ match a regular expression. LIKE, + * NOT LIKE, and IN apply to sql filters only, and =| to promql + * filters only; a condition its filter's sourceType can't express + * is not applied. + * example: "=" + * value: + * type: string + * maxLength: 10000 + * description: Value, pattern, or regular expression to compare against. + * example: "api" * MetricDataType: * type: string * enum: [sum, gauge, histogram, summary, exponential histogram] @@ -2368,12 +2417,14 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/QueryExpressionFilterInput' * - $ref: '#/components/schemas/StaticListFilterInput' * - $ref: '#/components/schemas/PrometheusLabelFilterInput' + * - $ref: '#/components/schemas/AdhocFilterInput' * discriminator: * propertyName: type * mapping: * QUERY_EXPRESSION: '#/components/schemas/QueryExpressionFilterInput' * STATIC_LIST: '#/components/schemas/StaticListFilterInput' * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' + * ADHOC: '#/components/schemas/AdhocFilterInput' * * QueryExpressionFilterInput: * type: object @@ -2684,6 +2735,92 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * behavior. * example: 1 * + * AdhocFilterInput: + * type: object + * description: | + * A filter holding free-form "key operator value" conditions, with keys + * and values suggested from its sources: columns and map keys for sql + * sources, labels for promql sources. Its conditions are always + * broadcast to the tiles it applies to: as WHERE conditions on builder + * tiles and raw sql tiles using the $__filters macro, or as label + * matchers on PromQL tiles. Its conditions are saved in + * savedFilterValues as an "adhoc" entry. Ad hoc filters are a preview + * feature. + * required: + * - type + * - name + * - sourceType + * - sources + * properties: + * type: + * type: string + * enum: [ADHOC] + * description: Discriminator. Must be "ADHOC". + * example: "ADHOC" + * name: + * type: string + * minLength: 1 + * description: Display name for the dashboard filter + * example: "Conditions" + * sourceType: + * type: string + * enum: [sql, promql] + * description: | + * The kind of sources the filter reads keys from and applies to: Log, + * Trace, or Session sources for sql, PromQL sources for promql. + * example: "sql" + * sources: + * type: array + * minItems: 1 + * items: + * type: string + * description: IDs of the sources whose keys and values the filter suggests, all of sourceType. + * example: ["65f5e4a3b9e77c001a111111"] + * where: + * type: string + * description: | + * Optional condition narrowing the rows keys and values are suggested + * from. sql filters only; never applied to tiles. May reference the + * dashboard's variables. + * example: "ServiceName:api" + * whereLanguage: + * type: string + * enum: [sql, lucene] + * description: Language of the where condition + * example: "lucene" + * appliesToSourceIds: + * type: array + * items: + * type: string + * description: | + * IDs of the sources whose tiles the conditions apply to, all of + * sourceType. Omit or send an empty array to apply to the tiles of + * the filter's sources. + * example: ["65f5e4a3b9e77c001a111111"] + * isBroadcastEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true or omitted. + * example: true + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true or omitted. + * example: true + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: | + * Unique name the filter's conditions are saved under, in + * savedFilterValues and the dashboard URL. Tiles can't reference it + * as a variable. Defaults to the display name with whitespace + * replaced by underscores and remaining illegal characters removed. + * Must be unique across a dashboard's variable-enabled filters. + * example: "conditions" + * * Filter: * allOf: * - $ref: '#/components/schemas/FilterInput' diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index dc8882183e..dec4319981 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -508,6 +508,43 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(hidden.filters?.map(f => f.id)).toEqual(['static-filter']); }); + it('hides ad hoc filters and their saved conditions only when asked', () => { + const adhoc = { + id: 'adhoc-filter', + type: 'ADHOC' as const, + name: 'Conditions', + sourceType: 'sql' as const, + sources: [new mongoose.Types.ObjectId().toString()], + isBroadcastEnabled: true as const, + isVariableEnabled: true as const, + variableName: 'conds', + }; + const variableValue = { + type: 'variable' as const, + name: 'env', + values: ['prod'], + }; + const adhocValue = { + type: 'adhoc' as const, + name: 'conds', + conditions: [ + { key: 'ServiceName', operator: '=' as const, value: 'api' }, + ], + }; + const doc = makeDoc({ + filters: [adhoc], + savedFilterValues: [variableValue, adhocValue], + }); + + const shown = convertToExternalDashboard(doc); + expect(shown.filters).toEqual([adhoc]); + expect(shown.savedFilterValues).toEqual([variableValue, adhocValue]); + + const hidden = convertToExternalDashboard(doc, { hideAdhoc: true }); + expect(hidden.filters).toEqual([]); + expect(hidden.savedFilterValues).toEqual([variableValue]); + }); + // seriesLimit is three-state (matching the internal schema): omitted = // default cap, 0 = unlimited, positive N = top-N. All three must survive a // GET->PUT round-trip: 0 and N pass through, null maps to absent (the diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 69e143e950..7e84a6b664 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -14,6 +14,8 @@ import { validateDashboardTileContainerRefs, } from '@hyperdx/common-utils/dist/dashboardValidation'; import { + getAdhocFilterSourceKinds, + isAdhocFilter, isPrometheusLabelFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; @@ -55,8 +57,8 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { - getExternalCompatibleFilters, - getExternalCompatibleFilterValues, + getMcpCompatibleFilters, + getMcpCompatibleFilterValues, translateExternalChartToTileConfig, translateExternalFilterToFilter, translateFilterToExternalFilter, @@ -773,10 +775,13 @@ export function convertToExternalDashboard( dashboard: DashboardDocument, { hidePromql = false, + hideAdhoc = false, }: { // MCP hides PromQL tiles and PROMETHEUS_LABEL filters while PromQL is // disabled on the deployment. hidePromql?: boolean; + // MCP doesn't expose ad hoc filters or their saved conditions yet. + hideAdhoc?: boolean; } = {}, ): ExternalDashboard { const containers = dashboard.containers ?? []; @@ -798,14 +803,17 @@ export function convertToExternalDashboard( convertTileToExternalChart(tile, containerById, dashboardId), ), tags: dashboard.tags || [], - filters: getExternalCompatibleFilters(dashboard.filters) + filters: (hideAdhoc + ? getMcpCompatibleFilters(dashboard.filters) + : (dashboard.filters ?? []) + ) .filter(filter => !(hidePromql && isPrometheusLabelFilter(filter))) .map(translateFilterToExternalFilter), savedQuery: dashboard.savedQuery ?? null, savedQueryLanguage: dashboard.savedQueryLanguage ?? null, - savedFilterValues: getExternalCompatibleFilterValues( - dashboard.savedFilterValues, - ), + savedFilterValues: hideAdhoc + ? getMcpCompatibleFilterValues(dashboard.savedFilterValues) + : (dashboard.savedFilterValues ?? []), // Mongoose persists missing arrays as []. Only emit containers when // the user actually saved one or more, so dashboards without the // organization layer round-trip with the field absent. @@ -1279,6 +1287,39 @@ function getTileOnClick(tile: ExternalDashboardTileWithId) { return tile.config.onClick; } +/** Source IDs a filter reads from or applies to. */ +function getFilterSourceIds( + filter: ExternalDashboardFilter | ExternalDashboardFilterWithId, +): string[] { + if (isStaticListFilter(filter)) return []; + if (isAdhocFilter(filter)) { + return [...filter.sources, ...(filter.appliesToSourceIds ?? [])]; + } + return [filter.sourceId]; +} + +/** + * Returns an error message if an ad hoc filter reads from or applies to a + * source of a kind its `sourceType` can't query, or null if none does. + */ +function getAdhocFilterSourceKindError( + sources: SourceForValidation[], + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], +): string | null { + const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); + const invalid = new Set( + filters.filter(isAdhocFilter).flatMap(filter => { + const kinds = getAdhocFilterSourceKinds(filter.sourceType); + return getFilterSourceIds(filter).filter(id => { + const kind = kindById.get(id); + return kind !== undefined && !kinds.includes(kind); + }); + }), + ); + if (invalid.size === 0) return null; + return `ADHOC filters with sourceType "sql" require Log, Trace, or Session sources, and with sourceType "promql" require PromQL sources. The following source IDs are not a supported kind: ${[...invalid].join(', ')}`; +} + /** Returns source IDs referenced in tiles/filters that do not exist for the team */ function getMissingSources( sources: SourceForValidation[], @@ -1307,13 +1348,8 @@ function getMissingSources( } } - if (filters?.length) { - for (const filter of filters) { - // Every variant but the static one names a source. - if (!isStaticListFilter(filter)) { - sourceIds.add(filter.sourceId); - } - } + for (const filter of filters ?? []) { + for (const sourceId of getFilterSourceIds(filter)) sourceIds.add(sourceId); } const existingSourceIds = new Set( @@ -1832,6 +1868,12 @@ export async function validateDashboardTiles( ); if (promqlLabelFilterError != null) return promqlLabelFilterError; + const adhocFilterError = getAdhocFilterSourceKindError( + sources, + filters ?? [], + ); + if (adhocFilterError != null) return adhocFilterError; + if (missingOnClickDashboards.length > 0) { return `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`; } diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 187abd7a24..b0e2ed35d7 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -240,32 +240,22 @@ export function translateExternalChartToTileConfig( }; } -/** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract; -export type ExternalCompatibleDashboardFilter = Exclude< - DashboardFilter, - InternalOnlyDashboardFilter ->; - -/** The filters the external API and MCP may read. */ -export function getExternalCompatibleFilters( +/** The filters MCP may read; it doesn't expose ad hoc filters yet. */ +export function getMcpCompatibleFilters( filters: DashboardFilter[] | undefined, -): ExternalCompatibleDashboardFilter[] { - return (filters ?? []).filter( - (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter), - ); +): DashboardFilter[] { + return (filters ?? []).filter(filter => !isAdhocFilter(filter)); } -/** Saved filter values the external API and MCP may read. */ -export function getExternalCompatibleFilterValues( +/** Saved filter values MCP may read; it doesn't expose ad hoc filters yet. */ +export function getMcpCompatibleFilterValues( values: IDashboard['savedFilterValues'], ): NonNullable { return (values ?? []).filter(value => !isAdhocFilterValue(value)); } export function translateFilterToExternalFilter( - filter: ExternalCompatibleDashboardFilter, + filter: DashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. @@ -277,6 +267,9 @@ export function translateFilterToExternalFilter( case 'STATIC_LIST': return omit(filter, ...omittedKeys); + case 'ADHOC': + return filter; + case 'PROMETHEUS_LABEL': return { ...omit(filter, 'source', ...omittedKeys), @@ -309,6 +302,7 @@ export function translateExternalFilterToFilter( ): DashboardFilter { switch (filter.type) { case 'STATIC_LIST': + case 'ADHOC': return filter; case 'PROMETHEUS_LABEL': diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index d14d34849d..5a27dccf2b 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,6 +1,8 @@ import { validateChartConfigFormulas } from '@hyperdx/common-utils/dist/dashboardValidation'; import { addDuplicateTileIdIssues, + AdhocDashboardFilterSchema, + AdhocFilterValueSchema, AggregateFunctionSchema, AlertChartConfigSchema, alertDisplayNameSchema, @@ -189,12 +191,21 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ isVariableEnabled: z.literal(true).default(true), }); +// Field names match the stored filter, so it needs no translation. +const externalAdhocFilterShape = AdhocDashboardFilterSchema.extend({ + sources: z.array(objectIdSchema).min(1), + appliesToSourceIds: z.array(objectIdSchema).optional(), + isBroadcastEnabled: z.literal(true).default(true), + isVariableEnabled: z.literal(true).default(true), +}); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ externalQueryExpressionFilterShape.strict(), externalStaticListFilterShape.strict(), externalPromqlLabelFilterShape.strict(), + externalAdhocFilterShape.strict(), ], ); @@ -211,6 +222,7 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalQueryExpressionFilterShape.omit({ id: true }).strict(), externalStaticListFilterShape.omit({ id: true }).strict(), externalPromqlLabelFilterShape.omit({ id: true }).strict(), + externalAdhocFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer< @@ -218,8 +230,9 @@ export type ExternalDashboardFilter = z.infer< >; /** - * One entry in a dashboard's `savedFilterValues`: either a rendered SQL - * predicate, or a selection addressed by the dashboard variable it belongs to. + * One entry in a dashboard's `savedFilterValues`: a rendered SQL predicate, a + * selection addressed by the dashboard variable it belongs to, or an ad hoc + * filter's conditions addressed by its variable name. */ export const externalSqlSavedFilterValueSchema = z.object({ type: z.literal('sql').optional().default('sql'), @@ -232,6 +245,7 @@ export const externalVariableSavedFilterValueSchema = export const externalDashboardSavedFilterValueSchema = z.union([ externalSqlSavedFilterValueSchema, externalVariableSavedFilterValueSchema, + AdhocFilterValueSchema.strict(), ]); // ================================