diff --git a/.changeset/odd-tables-end.md b/.changeset/odd-tables-end.md new file mode 100644 index 0000000000..59764c0964 --- /dev/null +++ b/.changeset/odd-tables-end.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +feat: Add Raw SQL Chart support to external dashboard APIs diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 328dd60df9..1187364f4b 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -144,7 +144,7 @@ }, "tileId": { "type": "string", - "description": "Tile ID for tile-based alerts.", + "description": "Tile ID for tile-based alerts. May not be a Raw-SQL-based tile.", "nullable": true, "example": "65f5e4a3b9e77c001a901234" }, @@ -1002,14 +1002,14 @@ } } }, - "LineChartConfig": { + "LineBuilderChartConfig": { "type": "object", "required": [ "displayType", "sourceId", "select" ], - "description": "Configuration for a line time-series chart.", + "description": "Builder configuration for a line time-series chart.", "properties": { "displayType": { "type": "string", @@ -1063,14 +1063,14 @@ } } }, - "BarChartConfig": { + "BarBuilderChartConfig": { "type": "object", "required": [ "displayType", "sourceId", "select" ], - "description": "Configuration for a stacked-bar time-series chart.", + "description": "Builder configuration for a stacked-bar time-series chart.", "properties": { "displayType": { "type": "string", @@ -1119,14 +1119,14 @@ } } }, - "TableChartConfig": { + "TableBuilderChartConfig": { "type": "object", "required": [ "displayType", "sourceId", "select" ], - "description": "Configuration for a table aggregation chart.", + "description": "Builder configuration for a table aggregation chart.", "properties": { "displayType": { "type": "string", @@ -1177,14 +1177,14 @@ } } }, - "NumberChartConfig": { + "NumberBuilderChartConfig": { "type": "object", "required": [ "displayType", "sourceId", "select" ], - "description": "Configuration for a single big-number chart.", + "description": "Builder configuration for a single big-number chart.", "properties": { "displayType": { "type": "string", @@ -1212,14 +1212,14 @@ } } }, - "PieChartConfig": { + "PieBuilderChartConfig": { "type": "object", "required": [ "displayType", "sourceId", "select" ], - "description": "Configuration for a pie chart tile. Each slice represents one group value.", + "description": "Builder configuration for a pie chart tile. Each slice represents one group value.", "properties": { "displayType": { "type": "string", @@ -1315,8 +1315,265 @@ } } }, + "RawSqlChartConfigBase": { + "type": "object", + "required": [ + "configType", + "connectionId", + "sqlTemplate" + ], + "description": "Shared fields for Raw SQL chart configs. Set configType to \"sql\" and provide connectionId + sqlTemplate instead of sourceId + select.", + "properties": { + "configType": { + "type": "string", + "enum": [ + "sql" + ], + "example": "sql" + }, + "connectionId": { + "type": "string", + "description": "ID of the ClickHouse connection to execute the query against.", + "example": "65f5e4a3b9e77c001a567890" + }, + "sqlTemplate": { + "type": "string", + "maxLength": 100000, + "description": "SQL query template to execute. Supports HyperDX template variables.", + "example": "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR" + }, + "numberFormat": { + "$ref": "#/components/schemas/NumberFormat" + } + } + }, + "LineRawSqlChartConfig": { + "description": "Raw SQL configuration for a line time-series chart.", + "allOf": [ + { + "$ref": "#/components/schemas/RawSqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "line" + ], + "example": "line" + }, + "compareToPreviousPeriod": { + "type": "boolean", + "description": "Overlay the equivalent previous time period for comparison.", + "default": false + }, + "fillNulls": { + "type": "boolean", + "description": "Fill missing time buckets with zero instead of leaving gaps.", + "default": true + }, + "alignDateRangeToGranularity": { + "type": "boolean", + "description": "Expand date range boundaries to the query granularity interval.", + "default": true + } + } + } + ] + }, + "BarRawSqlChartConfig": { + "description": "Raw SQL configuration for a stacked-bar time-series chart.", + "allOf": [ + { + "$ref": "#/components/schemas/RawSqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "stacked_bar" + ], + "example": "stacked_bar" + }, + "fillNulls": { + "type": "boolean", + "description": "Fill missing time buckets with zero instead of leaving gaps.", + "default": true + }, + "alignDateRangeToGranularity": { + "type": "boolean", + "description": "Expand date range boundaries to the query granularity interval.", + "default": true + } + } + } + ] + }, + "TableRawSqlChartConfig": { + "description": "Raw SQL configuration for a table chart.", + "allOf": [ + { + "$ref": "#/components/schemas/RawSqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "table" + ], + "example": "table" + } + } + } + ] + }, + "NumberRawSqlChartConfig": { + "description": "Raw SQL configuration for a single big-number chart.", + "allOf": [ + { + "$ref": "#/components/schemas/RawSqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "number" + ], + "example": "number" + } + } + } + ] + }, + "PieRawSqlChartConfig": { + "description": "Raw SQL configuration for a pie chart.", + "allOf": [ + { + "$ref": "#/components/schemas/RawSqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "pie" + ], + "example": "pie" + } + } + } + ] + }, + "LineChartConfig": { + "description": "Line chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "oneOf": [ + { + "$ref": "#/components/schemas/LineBuilderChartConfig" + }, + { + "$ref": "#/components/schemas/LineRawSqlChartConfig" + } + ], + "discriminator": { + "propertyName": "configType", + "mapping": { + "sql": "#/components/schemas/LineRawSqlChartConfig" + } + } + }, + "BarChartConfig": { + "description": "Stacked-bar chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "oneOf": [ + { + "$ref": "#/components/schemas/BarBuilderChartConfig" + }, + { + "$ref": "#/components/schemas/BarRawSqlChartConfig" + } + ], + "discriminator": { + "propertyName": "configType", + "mapping": { + "sql": "#/components/schemas/BarRawSqlChartConfig" + } + } + }, + "TableChartConfig": { + "description": "Table chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "oneOf": [ + { + "$ref": "#/components/schemas/TableBuilderChartConfig" + }, + { + "$ref": "#/components/schemas/TableRawSqlChartConfig" + } + ], + "discriminator": { + "propertyName": "configType", + "mapping": { + "sql": "#/components/schemas/TableRawSqlChartConfig" + } + } + }, + "NumberChartConfig": { + "description": "Single big-number chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "oneOf": [ + { + "$ref": "#/components/schemas/NumberBuilderChartConfig" + }, + { + "$ref": "#/components/schemas/NumberRawSqlChartConfig" + } + ], + "discriminator": { + "propertyName": "configType", + "mapping": { + "sql": "#/components/schemas/NumberRawSqlChartConfig" + } + } + }, + "PieChartConfig": { + "description": "Pie chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "oneOf": [ + { + "$ref": "#/components/schemas/PieBuilderChartConfig" + }, + { + "$ref": "#/components/schemas/PieRawSqlChartConfig" + } + ], + "discriminator": { + "propertyName": "configType", + "mapping": { + "sql": "#/components/schemas/PieRawSqlChartConfig" + } + } + }, "TileConfig": { - "description": "Tile chart configuration. The displayType field determines which variant is used.\n", + "description": "Tile chart configuration. displayType is the primary discriminant and determines which variant group applies. For displayTypes that support both builder and Raw SQL modes (line, stacked_bar, table, number, pie), configType is the secondary discriminant: omit it for the builder variant or set it to \"sql\" for the Raw SQL variant. The search and markdown displayTypes only have a builder variant.\n", "oneOf": [ { "$ref": "#/components/schemas/LineChartConfig" diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index f26697123b..90512a5fd1 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -1,3 +1,4 @@ +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { sign, verify } from 'jsonwebtoken'; import { groupBy } from 'lodash'; import ms from 'ms'; @@ -74,9 +75,15 @@ export const validateAlertInput = async ( throw new Api400Error('Dashboard not found'); } - if (dashboard.tiles.find(tile => tile.id === alertInput.tileId) == null) { + const tile = dashboard.tiles.find(tile => tile.id === alertInput.tileId); + + if (tile == null) { throw new Api400Error('Tile not found'); } + + if (tile.config != null && isRawSqlSavedChartConfig(tile.config)) { + throw new Api400Error('Cannot create an alert on a raw SQL tile'); + } } if (alertInput.source === AlertSource.SAVED_SEARCH) { diff --git a/packages/api/src/controllers/connection.ts b/packages/api/src/controllers/connection.ts index 422e982f3d..1bff0d6e79 100644 --- a/packages/api/src/controllers/connection.ts +++ b/packages/api/src/controllers/connection.ts @@ -6,6 +6,10 @@ export function getConnections() { return Connection.find({}); } +export function getConnectionsByTeam(team: string) { + return Connection.find({ team }); +} + export function getConnectionById( team: string, connectionId: string, diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 70f15df05b..155550769e 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -2,6 +2,7 @@ import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { BuilderSavedChartConfig, DisplayType, + RawSqlSavedChartConfig, SavedChartConfig, Tile, } from '@hyperdx/common-utils/dist/types'; @@ -501,6 +502,20 @@ export const makeExternalTile = (opts?: { }, }); +export const makeRawSqlTile = (opts?: { id?: string }): Tile => ({ + id: opts?.id ?? randomMongoId(), + x: 1, + y: 1, + w: 1, + h: 1, + config: { + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: 'SELECT 1', + connection: 'test-connection', + } satisfies RawSqlSavedChartConfig, +}); + export const makeAlertInput = ({ dashboardId, interval = '15m', diff --git a/packages/api/src/routers/api/__tests__/alerts.test.ts b/packages/api/src/routers/api/__tests__/alerts.test.ts index ff9d7a1e6d..f1b4995869 100644 --- a/packages/api/src/routers/api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/api/__tests__/alerts.test.ts @@ -2,6 +2,7 @@ import { getLoggedInAgent, getServer, makeAlertInput, + makeRawSqlTile, makeTile, randomMongoId, } from '@/fixtures'; @@ -548,4 +549,62 @@ describe('alerts router', () => { await agent.delete(`/alerts/${fakeId}/silenced`).expect(404); // Should fail }); + + it('rejects creating an alert on a raw SQL tile', async () => { + const rawSqlTile = makeRawSqlTile(); + const dashboard = await agent + .post('/dashboards') + .send({ + name: 'Test Dashboard', + tiles: [rawSqlTile], + tags: [], + }) + .expect(200); + + await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: rawSqlTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(400); + }); + + it('rejects updating an alert to reference a raw SQL tile', async () => { + const regularTile = makeTile(); + const rawSqlTile = makeRawSqlTile(); + const dashboard = await agent + .post('/dashboards') + .send({ + name: 'Test Dashboard', + tiles: [regularTile, rawSqlTile], + tags: [], + }) + .expect(200); + + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: regularTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + + await agent + .put(`/alerts/${alert.body.data._id}`) + .send({ + ...makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: rawSqlTile.id, + webhookId: webhook._id.toString(), + }), + }) + .expect(400); + }); }); diff --git a/packages/api/src/routers/api/__tests__/dashboard.test.ts b/packages/api/src/routers/api/__tests__/dashboard.test.ts index 0539392ea8..22321168ca 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.test.ts @@ -16,6 +16,7 @@ import { getLoggedInAgent, getServer, makeAlertInput, + makeRawSqlTile, makeTile, } from '../../../fixtures'; import Alert from '../../../models/alert'; @@ -250,6 +251,38 @@ describe('dashboard router', () => { ]); }); + it('deletes alert when tile is updated from builder to raw SQL config', async () => { + const builderTile = makeTile(); + const dashboard = await agent + .post('/dashboards') + .send({ name: 'Test Dashboard', tiles: [builderTile], tags: [] }) + .expect(200); + + // Create a standalone alert for the builder tile + await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: builderTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + + expect((await agent.get('/alerts').expect(200)).body.data.length).toBe(1); + + // Update the tile to a raw SQL config (same tile ID) + const rawSqlTile = makeRawSqlTile({ id: builderTile.id }); + await agent + .patch(`/dashboards/${dashboard.body.id}`) + .send({ tiles: [rawSqlTile] }) + .expect(200); + + const alertsAfter = await agent.get('/alerts').expect(200); + expect(alertsAfter.body.data).toEqual([]); + }); + it('deletes attached alerts when deleting tiles', async () => { await agent.post('/dashboards').send(MOCK_DASHBOARD).expect(200); const initialDashboards = await agent.get('/dashboards').expect(200); diff --git a/packages/api/src/routers/external-api/__tests__/alerts.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.test.ts index 5108074ea2..68c2b0825e 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.test.ts @@ -82,6 +82,37 @@ describe('External API Alerts', () => { }).save(); }; + // Helper to create a dashboard with a raw SQL tile for testing + const createTestDashboardWithRawSqlTile = async ( + options: { teamId?: any } = {}, + ) => { + const tileId = new ObjectId().toString(); + const tiles = [ + { + id: tileId, + name: 'Raw SQL Chart', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'line', + sqlTemplate: 'SELECT 1', + connection: 'test-connection', + }, + }, + ]; + + const dashboard = await new Dashboard({ + name: 'Raw SQL Dashboard', + tiles, + team: options.teamId ?? team._id, + }).save(); + + return { dashboard, tileId }; + }; + // Helper to create a saved search for testing const createTestSavedSearch = async (options: { teamId?: any } = {}) => { return new SavedSearch({ @@ -684,6 +715,49 @@ describe('External API Alerts', () => { .send(updatePayload) .expect(400); }); + + it('should reject creating an alert on a raw SQL tile', async () => { + const webhook = await createTestWebhook(); + const { dashboard, tileId } = await createTestDashboardWithRawSqlTile(); + + const alertInput = { + dashboardId: dashboard._id.toString(), + tileId, + threshold: 100, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.ABOVE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }; + + await authRequest('post', ALERTS_BASE_URL).send(alertInput).expect(400); + }); + + it('should reject updating an alert to reference a raw SQL tile', async () => { + const { alert, webhook } = await createTestAlert(); + const { dashboard: rawSqlDashboard, tileId: rawSqlTileId } = + await createTestDashboardWithRawSqlTile(); + + const updatePayload = { + threshold: 200, + interval: '1h', + thresholdType: AlertThresholdType.ABOVE, + source: AlertSource.TILE, + dashboardId: rawSqlDashboard._id.toString(), + tileId: rawSqlTileId, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }; + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send(updatePayload) + .expect(400); + }); }); describe('saved search (SAVED_SEARCH source) validation', () => { diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts index 18109b22a4..da1d841dbd 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts @@ -21,9 +21,11 @@ import { makeExternalChart, makeExternalTile, } from '../../../fixtures'; +import Alert, { AlertSource, AlertThresholdType } from '../../../models/alert'; import Connection from '../../../models/connection'; import Dashboard from '../../../models/dashboard'; import { Source } from '../../../models/source'; +import Webhook, { WebhookService } from '../../../models/webhook'; // Constants const BASE_URL = '/api/v2/dashboards'; @@ -1802,7 +1804,7 @@ describe('External API v2 Dashboards - new format', () => { }); const server = getServer(); - let agent, team, user, traceSource, metricSource; + let agent, team, user, traceSource, metricSource, connection; beforeAll(async () => { await server.start(); @@ -1815,7 +1817,7 @@ describe('External API v2 Dashboards - new format', () => { team = result.team; user = result.user; - const connection = await Connection.create({ + connection = await Connection.create({ team: team._id, name: 'Default', host: config.CLICKHOUSE_HOST, @@ -2307,6 +2309,104 @@ describe('External API v2 Dashboards - new format', () => { expect(omit(response.body.data.tiles[5], ['id'])).toEqual(pieChart); }); + it('can round-trip all raw SQL chart config types', async () => { + const connectionId = connection._id.toString(); + const sqlTemplate = 'SELECT count() FROM otel_logs WHERE {timeFilter}'; + + const lineRawSql: ExternalDashboardTile = { + name: 'Line Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'line', + connectionId, + sqlTemplate, + compareToPreviousPeriod: true, + fillNulls: true, + alignDateRangeToGranularity: true, + numberFormat: { output: 'number', mantissa: 2 }, + }, + }; + + const barRawSql: ExternalDashboardTile = { + name: 'Bar Raw SQL', + x: 6, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'stacked_bar', + connectionId, + sqlTemplate, + fillNulls: false, + alignDateRangeToGranularity: false, + numberFormat: { output: 'byte', decimalBytes: true }, + }, + }; + + const tableRawSql: ExternalDashboardTile = { + name: 'Table Raw SQL', + x: 0, + y: 3, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate, + numberFormat: { output: 'percent', mantissa: 1 }, + }, + }; + + const numberRawSql: ExternalDashboardTile = { + name: 'Number Raw SQL', + x: 6, + y: 3, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'number', + connectionId, + sqlTemplate, + numberFormat: { output: 'currency', currencySymbol: '$' }, + }, + }; + + const pieRawSql: ExternalDashboardTile = { + name: 'Pie Raw SQL', + x: 12, + y: 3, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'pie', + connectionId, + sqlTemplate, + }, + }; + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard with Raw SQL Chart Types', + tiles: [lineRawSql, barRawSql, tableRawSql, numberRawSql, pieRawSql], + tags: ['raw-sql-test'], + }) + .expect(200); + + expect(omit(response.body.data.tiles[0], ['id'])).toEqual(lineRawSql); + expect(omit(response.body.data.tiles[1], ['id'])).toEqual(barRawSql); + expect(omit(response.body.data.tiles[2], ['id'])).toEqual(tableRawSql); + expect(omit(response.body.data.tiles[3], ['id'])).toEqual(numberRawSql); + expect(omit(response.body.data.tiles[4], ['id'])).toEqual(pieRawSql); + }); + it('should return 400 when source IDs do not exist', async () => { const nonExistentSourceId = new ObjectId().toString(); const mockDashboard = createMockDashboard(nonExistentSourceId); @@ -2320,6 +2420,43 @@ describe('External API v2 Dashboards - new format', () => { }); }); + it('should return 400 when connection ID does not belong to the team', async () => { + const otherTeamConnection = await Connection.create({ + team: new ObjectId(), + name: 'Other Team Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const otherConnectionId = otherTeamConnection._id.toString(); + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard with Foreign Connection', + tiles: [ + { + name: 'Raw SQL Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'line', + connectionId: otherConnectionId, + sqlTemplate: 'SELECT count() FROM otel_logs WHERE {timeFilter}', + }, + }, + ], + tags: [], + }) + .expect(400); + + expect(response.body).toEqual({ + message: `Could not find the following connection IDs: ${otherConnectionId}`, + }); + }); + it('should create a dashboard with filters', async () => { const dashboardPayload = { name: 'Dashboard with Filters', @@ -2961,6 +3098,124 @@ describe('External API v2 Dashboards - new format', () => { ); }); + it('can round-trip all raw SQL chart config types', async () => { + const connectionId = connection._id.toString(); + const sqlTemplate = 'SELECT count() FROM otel_logs WHERE {timeFilter}'; + + const lineRawSql: ExternalDashboardTileWithId = { + id: new ObjectId().toString(), + name: 'Line Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'line', + connectionId, + sqlTemplate, + compareToPreviousPeriod: true, + fillNulls: true, + alignDateRangeToGranularity: true, + numberFormat: { output: 'number', mantissa: 2 }, + }, + }; + + const barRawSql: ExternalDashboardTileWithId = { + id: new ObjectId().toString(), + name: 'Bar Raw SQL', + x: 6, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'stacked_bar', + connectionId, + sqlTemplate, + fillNulls: false, + alignDateRangeToGranularity: false, + numberFormat: { output: 'byte', decimalBytes: true }, + }, + }; + + const tableRawSql: ExternalDashboardTileWithId = { + id: new ObjectId().toString(), + name: 'Table Raw SQL', + x: 0, + y: 3, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate, + numberFormat: { output: 'percent', mantissa: 1 }, + }, + }; + + const numberRawSql: ExternalDashboardTileWithId = { + id: new ObjectId().toString(), + name: 'Number Raw SQL', + x: 6, + y: 3, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'number', + connectionId, + sqlTemplate, + numberFormat: { output: 'currency', currencySymbol: '$' }, + }, + }; + + const pieRawSql: ExternalDashboardTileWithId = { + id: new ObjectId().toString(), + name: 'Pie Raw SQL', + x: 12, + y: 3, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'pie', + connectionId, + sqlTemplate, + }, + }; + + const initialDashboard = await createTestDashboard(); + + const response = await authRequest( + 'put', + `${BASE_URL}/${initialDashboard._id}`, + ) + .send({ + name: 'Dashboard with Raw SQL Chart Types', + tiles: [lineRawSql, barRawSql, tableRawSql, numberRawSql, pieRawSql], + tags: ['raw-sql-test'], + }) + .expect(200); + + expect(omit(response.body.data.tiles[0], ['id'])).toEqual( + omit(lineRawSql, ['id']), + ); + expect(omit(response.body.data.tiles[1], ['id'])).toEqual( + omit(barRawSql, ['id']), + ); + expect(omit(response.body.data.tiles[2], ['id'])).toEqual( + omit(tableRawSql, ['id']), + ); + expect(omit(response.body.data.tiles[3], ['id'])).toEqual( + omit(numberRawSql, ['id']), + ); + expect(omit(response.body.data.tiles[4], ['id'])).toEqual( + omit(pieRawSql, ['id']), + ); + }); + it('should return 400 when source IDs do not exist', async () => { const dashboard = await createTestDashboard(); const nonExistentSourceId = new ObjectId().toString(); @@ -2976,6 +3231,229 @@ describe('External API v2 Dashboards - new format', () => { message: `Could not find the following source IDs: ${nonExistentSourceId}`, }); }); + + it('should return 400 when connection ID does not belong to the team', async () => { + const dashboard = await createTestDashboard(); + const otherTeamConnection = await Connection.create({ + team: new ObjectId(), + name: 'Other Team Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const otherConnectionId = otherTeamConnection._id.toString(); + + const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ + name: 'Updated Dashboard with Foreign Connection', + tiles: [ + { + id: new ObjectId().toString(), + name: 'Raw SQL Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'line', + connectionId: otherConnectionId, + sqlTemplate: 'SELECT count() FROM otel_logs WHERE {timeFilter}', + }, + }, + ], + tags: [], + }) + .expect(400); + + expect(response.body).toEqual({ + message: `Could not find the following connection IDs: ${otherConnectionId}`, + }); + }); + + it('should delete alert when tile is updated from builder to raw SQL config', async () => { + const tileId = new ObjectId().toString(); + const dashboard = await createTestDashboard({ + tiles: [ + { + id: tileId, + name: 'Builder Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + source: traceSource._id.toString(), + select: [ + { + aggFn: 'count', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + ], + where: '', + whereLanguage: 'lucene', + granularity: 'auto', + implicitColumnExpression: 'Body', + filters: [], + }, + }, + ], + }); + + const webhook = await Webhook.create({ + name: 'Test Webhook', + service: WebhookService.Slack, + url: 'https://hooks.slack.com/test', + team: team._id, + }); + + // Create a standalone alert for the builder tile + const alert = await Alert.create({ + team: team._id, + dashboard: dashboard._id, + tileId, + source: AlertSource.TILE, + threshold: 100, + interval: '1h', + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + }); + + expect(await Alert.findById(alert._id)).not.toBeNull(); + + // Update the tile to raw SQL config (same tile ID) + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ + name: 'Updated Dashboard', + tags: [], + tiles: [ + { + id: tileId, + name: 'Raw SQL Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'line', + connectionId: connection._id.toString(), + sqlTemplate: 'SELECT count() FROM otel_logs WHERE {timeFilter}', + }, + }, + ], + }) + .expect(200); + + expect(await Alert.findById(alert._id)).toBeNull(); + }); + + it('should delete alert when a tile with an alert is removed from the dashboard', async () => { + const keepTileId = new ObjectId().toString(); + const removeTileId = new ObjectId().toString(); + const dashboard = await createTestDashboard({ + tiles: [ + { + id: keepTileId, + name: 'Keep Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + source: traceSource._id.toString(), + select: [ + { + aggFn: 'count', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + ], + where: '', + whereLanguage: 'lucene', + granularity: 'auto', + implicitColumnExpression: 'Body', + filters: [], + }, + }, + { + id: removeTileId, + name: 'Remove Tile', + x: 6, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + source: traceSource._id.toString(), + select: [ + { + aggFn: 'count', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + ], + where: '', + whereLanguage: 'lucene', + granularity: 'auto', + implicitColumnExpression: 'Body', + filters: [], + }, + }, + ], + }); + + const webhook = await Webhook.create({ + name: 'Test Webhook', + service: WebhookService.Slack, + url: 'https://hooks.slack.com/test', + team: team._id, + }); + + const alert = await Alert.create({ + team: team._id, + dashboard: dashboard._id, + tileId: removeTileId, + source: AlertSource.TILE, + threshold: 100, + interval: '1h', + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + }); + + expect(await Alert.findById(alert._id)).not.toBeNull(); + + // Update the dashboard, omitting the tile that had an alert + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ + name: 'Updated Dashboard', + tags: [], + tiles: [ + { + id: keepTileId, + name: 'Keep Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count', where: '' }], + }, + }, + ], + }) + .expect(200); + + expect(await Alert.findById(alert._id)).toBeNull(); + }); }); describe('DELETE /:id', () => { diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index cf0a12a102..41e959eae6 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -89,7 +89,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * example: "65f5e4a3b9e77c001a567890" * tileId: * type: string - * description: Tile ID for tile-based alerts. + * description: Tile ID for tile-based alerts. May not be a Raw-SQL-based tile. * nullable: true * example: "65f5e4a3b9e77c001a901234" * savedSearchId: diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index a1ae421a2e..e01d4c7303 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -1,3 +1,4 @@ +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { SearchConditionLanguageSchema as whereLanguageSchema } from '@hyperdx/common-utils/dist/types'; import express from 'express'; import { uniq } from 'lodash'; @@ -5,6 +6,8 @@ import { ObjectId } from 'mongodb'; import mongoose from 'mongoose'; import { z } from 'zod'; +import { deleteDashboardAlerts } from '@/controllers/alerts'; +import { getConnectionsByTeam } from '@/controllers/connection'; import { deleteDashboard } from '@/controllers/dashboard'; import { getSources } from '@/controllers/sources'; import Dashboard from '@/models/dashboard'; @@ -13,6 +16,7 @@ import { translateExternalChartToTileConfig, translateExternalFilterToFilter, } from '@/utils/externalApi'; +import logger from '@/utils/logger'; import { ExternalDashboardFilter, externalDashboardFilterSchema, @@ -29,6 +33,7 @@ import { convertToExternalDashboard, convertToInternalTileConfig, isConfigTile, + isRawSqlExternalTileConfig, isSeriesTile, } from './utils/dashboards'; @@ -69,6 +74,31 @@ async function getMissingSources( return [...sourceIds].filter(sourceId => !existingSourceIds.has(sourceId)); } +/** Returns an array of connection IDs that are referenced in the tiles but do not belong to the team */ +async function getMissingConnections( + team: string | mongoose.Types.ObjectId, + tiles: ExternalDashboardTileWithId[], +): Promise { + const connectionIds = new Set(); + + for (const tile of tiles) { + if (isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config)) { + connectionIds.add(tile.config.connectionId); + } + } + + if (connectionIds.size === 0) return []; + + const existingConnections = await getConnectionsByTeam(team.toString()); + const existingConnectionIds = new Set( + existingConnections.map(connection => connection._id.toString()), + ); + + return [...connectionIds].filter( + connectionId => !existingConnectionIds.has(connectionId), + ); +} + type SavedQueryLanguage = z.infer; function resolveSavedQueryLanguage(params: { @@ -488,13 +518,13 @@ const updateDashboardBodySchema = buildDashboardBodySchema( * enum: [delta] * description: Optional period aggregation function for Gauge metrics (e.g., compute the delta over the period). * - * LineChartConfig: + * LineBuilderChartConfig: * type: object * required: * - displayType * - sourceId * - select - * description: Configuration for a line time-series chart. + * description: Builder configuration for a line time-series chart. * properties: * displayType: * type: string @@ -537,13 +567,13 @@ const updateDashboardBodySchema = buildDashboardBodySchema( * description: Overlay the equivalent previous time period for comparison. * default: false * - * BarChartConfig: + * BarBuilderChartConfig: * type: object * required: * - displayType * - sourceId * - select - * description: Configuration for a stacked-bar time-series chart. + * description: Builder configuration for a stacked-bar time-series chart. * properties: * displayType: * type: string @@ -582,13 +612,13 @@ const updateDashboardBodySchema = buildDashboardBodySchema( * numberFormat: * $ref: '#/components/schemas/NumberFormat' * - * TableChartConfig: + * TableBuilderChartConfig: * type: object * required: * - displayType * - sourceId * - select - * description: Configuration for a table aggregation chart. + * description: Builder configuration for a table aggregation chart. * properties: * displayType: * type: string @@ -629,13 +659,13 @@ const updateDashboardBodySchema = buildDashboardBodySchema( * numberFormat: * $ref: '#/components/schemas/NumberFormat' * - * NumberChartConfig: + * NumberBuilderChartConfig: * type: object * required: * - displayType * - sourceId * - select - * description: Configuration for a single big-number chart. + * description: Builder configuration for a single big-number chart. * properties: * displayType: * type: string @@ -655,13 +685,13 @@ const updateDashboardBodySchema = buildDashboardBodySchema( * numberFormat: * $ref: '#/components/schemas/NumberFormat' * - * PieChartConfig: + * PieBuilderChartConfig: * type: object * required: * - displayType * - sourceId * - select - * description: Configuration for a pie chart tile. Each slice represents one group value. + * description: Builder configuration for a pie chart tile. Each slice represents one group value. * properties: * displayType: * type: string @@ -733,10 +763,188 @@ const updateDashboardBodySchema = buildDashboardBodySchema( * description: Markdown content to render inside the tile. * example: "# Dashboard Title\n\nThis is a markdown widget." * + * RawSqlChartConfigBase: + * type: object + * required: + * - configType + * - connectionId + * - sqlTemplate + * description: Shared fields for Raw SQL chart configs. Set configType to "sql" and provide connectionId + sqlTemplate instead of sourceId + select. + * properties: + * configType: + * type: string + * enum: [sql] + * example: "sql" + * connectionId: + * type: string + * description: ID of the ClickHouse connection to execute the query against. + * example: "65f5e4a3b9e77c001a567890" + * sqlTemplate: + * type: string + * maxLength: 100000 + * description: SQL query template to execute. Supports HyperDX template variables. + * example: "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR" + * numberFormat: + * $ref: '#/components/schemas/NumberFormat' + * + * LineRawSqlChartConfig: + * description: Raw SQL configuration for a line time-series chart. + * allOf: + * - $ref: '#/components/schemas/RawSqlChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [line] + * example: "line" + * compareToPreviousPeriod: + * type: boolean + * description: Overlay the equivalent previous time period for comparison. + * default: false + * fillNulls: + * type: boolean + * description: Fill missing time buckets with zero instead of leaving gaps. + * default: true + * alignDateRangeToGranularity: + * type: boolean + * description: Expand date range boundaries to the query granularity interval. + * default: true + * + * BarRawSqlChartConfig: + * description: Raw SQL configuration for a stacked-bar time-series chart. + * allOf: + * - $ref: '#/components/schemas/RawSqlChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [stacked_bar] + * example: "stacked_bar" + * fillNulls: + * type: boolean + * description: Fill missing time buckets with zero instead of leaving gaps. + * default: true + * alignDateRangeToGranularity: + * type: boolean + * description: Expand date range boundaries to the query granularity interval. + * default: true + * + * TableRawSqlChartConfig: + * description: Raw SQL configuration for a table chart. + * allOf: + * - $ref: '#/components/schemas/RawSqlChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [table] + * example: "table" + * + * NumberRawSqlChartConfig: + * description: Raw SQL configuration for a single big-number chart. + * allOf: + * - $ref: '#/components/schemas/RawSqlChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [number] + * example: "number" + * + * PieRawSqlChartConfig: + * description: Raw SQL configuration for a pie chart. + * allOf: + * - $ref: '#/components/schemas/RawSqlChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [pie] + * example: "pie" + * + * LineChartConfig: + * description: > + * Line chart. Omit configType for the builder variant (requires sourceId + * and select). Set configType to "sql" for the Raw SQL variant (requires + * connectionId and sqlTemplate). + * oneOf: + * - $ref: '#/components/schemas/LineBuilderChartConfig' + * - $ref: '#/components/schemas/LineRawSqlChartConfig' + * discriminator: + * propertyName: configType + * mapping: + * sql: '#/components/schemas/LineRawSqlChartConfig' + * + * BarChartConfig: + * description: > + * Stacked-bar chart. Omit configType for the builder variant (requires + * sourceId and select). Set configType to "sql" for the Raw SQL variant + * (requires connectionId and sqlTemplate). + * oneOf: + * - $ref: '#/components/schemas/BarBuilderChartConfig' + * - $ref: '#/components/schemas/BarRawSqlChartConfig' + * discriminator: + * propertyName: configType + * mapping: + * sql: '#/components/schemas/BarRawSqlChartConfig' + * + * TableChartConfig: + * description: > + * Table chart. Omit configType for the builder variant (requires sourceId + * and select). Set configType to "sql" for the Raw SQL variant (requires + * connectionId and sqlTemplate). + * oneOf: + * - $ref: '#/components/schemas/TableBuilderChartConfig' + * - $ref: '#/components/schemas/TableRawSqlChartConfig' + * discriminator: + * propertyName: configType + * mapping: + * sql: '#/components/schemas/TableRawSqlChartConfig' + * + * NumberChartConfig: + * description: > + * Single big-number chart. Omit configType for the builder variant + * (requires sourceId and select). Set configType to "sql" for the Raw + * SQL variant (requires connectionId and sqlTemplate). + * oneOf: + * - $ref: '#/components/schemas/NumberBuilderChartConfig' + * - $ref: '#/components/schemas/NumberRawSqlChartConfig' + * discriminator: + * propertyName: configType + * mapping: + * sql: '#/components/schemas/NumberRawSqlChartConfig' + * + * PieChartConfig: + * description: > + * Pie chart. Omit configType for the builder variant (requires sourceId + * and select). Set configType to "sql" for the Raw SQL variant (requires + * connectionId and sqlTemplate). + * oneOf: + * - $ref: '#/components/schemas/PieBuilderChartConfig' + * - $ref: '#/components/schemas/PieRawSqlChartConfig' + * discriminator: + * propertyName: configType + * mapping: + * sql: '#/components/schemas/PieRawSqlChartConfig' + * * TileConfig: * description: > - * Tile chart configuration. The displayType field determines which - * variant is used. + * Tile chart configuration. displayType is the primary discriminant and + * determines which variant group applies. For displayTypes that support + * both builder and Raw SQL modes (line, stacked_bar, table, number, pie), + * configType is the secondary discriminant: omit it for the builder + * variant or set it to "sql" for the Raw SQL variant. The search and + * markdown displayTypes only have a builder variant. * oneOf: * - $ref: '#/components/schemas/LineChartConfig' * - $ref: '#/components/schemas/BarChartConfig' @@ -1407,7 +1615,10 @@ router.post( savedFilterValues, } = req.body; - const missingSources = await getMissingSources(teamId, tiles, filters); + const [missingSources, missingConnections] = await Promise.all([ + getMissingSources(teamId, tiles, filters), + getMissingConnections(teamId, tiles), + ]); if (missingSources.length > 0) { return res.status(400).json({ message: `Could not find the following source IDs: ${missingSources.join( @@ -1415,6 +1626,13 @@ router.post( )}`, }); } + if (missingConnections.length > 0) { + return res.status(400).json({ + message: `Could not find the following connection IDs: ${missingConnections.join( + ', ', + )}`, + }); + } const internalTiles = tiles.map(tile => { const tileId = new ObjectId().toString(); @@ -1630,7 +1848,10 @@ router.put( savedFilterValues, } = req.body ?? {}; - const missingSources = await getMissingSources(teamId, tiles, filters); + const [missingSources, missingConnections] = await Promise.all([ + getMissingSources(teamId, tiles, filters), + getMissingConnections(teamId, tiles), + ]); if (missingSources.length > 0) { return res.status(400).json({ message: `Could not find the following source IDs: ${missingSources.join( @@ -1638,6 +1859,13 @@ router.put( )}`, }); } + if (missingConnections.length > 0) { + return res.status(400).json({ + message: `Could not find the following connection IDs: ${missingConnections.join( + ', ', + )}`, + }); + } const existingDashboard = await Dashboard.findOne( { _id: dashboardId, team: teamId }, @@ -1702,6 +1930,22 @@ router.put( return res.sendStatus(404); } + // Delete alerts for tiles that are now raw SQL (unsupported) or were removed + const newTileIdSet = new Set(internalTiles.map(t => t.id)); + const tileIdsToDeleteAlerts = [ + ...internalTiles + .filter(tile => isRawSqlSavedChartConfig(tile.config)) + .map(tile => tile.id), + ...[...existingTileIds].filter(id => !newTileIdSet.has(id)), + ]; + if (tileIdsToDeleteAlerts.length > 0) { + logger.info( + { dashboardId, teamId, tileIds: tileIdsToDeleteAlerts }, + `Deleting alerts for tiles with unsupported config or removed tiles`, + ); + await deleteDashboardAlerts(dashboardId, teamId, tileIdsToDeleteAlerts); + } + res.json({ data: convertToExternalDashboard(updatedDashboard), }); diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index a7ac5d1010..a2ff59759e 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -3,6 +3,7 @@ import { AggregateFunctionSchema, BuilderSavedChartConfig, DisplayType, + RawSqlSavedChartConfig, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; import { pick } from 'lodash'; @@ -13,6 +14,7 @@ import { translateFilterToExternalFilter } from '@/utils/externalApi'; import logger from '@/utils/logger'; import { ExternalDashboardFilterWithId, + ExternalDashboardRawSqlTileConfig, ExternalDashboardSelectItem, ExternalDashboardTileConfig, ExternalDashboardTileWithId, @@ -37,6 +39,12 @@ export type ConfigTile = ExternalDashboardTileWithId & { config: Exclude; }; +export function isRawSqlExternalTileConfig( + config: ExternalDashboardTileConfig, +): config is ExternalDashboardRawSqlTileConfig { + return 'configType' in config && config.configType === 'sql'; +} + export function isConfigTile( tile: ExternalDashboardTileWithId, ): tile is ConfigTile { @@ -86,8 +94,66 @@ const convertToExternalSelectItem = ( const convertToExternalTileChartConfig = ( config: SavedChartConfig, ): ExternalDashboardTileConfig | undefined => { - // HDX-3582: Implement this for Raw SQL charts - if (isRawSqlSavedChartConfig(config)) return undefined; + if (isRawSqlSavedChartConfig(config)) { + switch (config.displayType) { + case DisplayType.Line: + return { + configType: 'sql', + displayType: DisplayType.Line, + connectionId: config.connection, + sqlTemplate: config.sqlTemplate, + alignDateRangeToGranularity: config.alignDateRangeToGranularity, + fillNulls: config.fillNulls !== false, + numberFormat: config.numberFormat, + compareToPreviousPeriod: config.compareToPreviousPeriod, + }; + case DisplayType.StackedBar: + return { + configType: 'sql', + displayType: config.displayType, + connectionId: config.connection, + sqlTemplate: config.sqlTemplate, + alignDateRangeToGranularity: config.alignDateRangeToGranularity, + fillNulls: config.fillNulls !== false, + numberFormat: config.numberFormat, + }; + case DisplayType.Table: + return { + configType: 'sql', + displayType: DisplayType.Table, + connectionId: config.connection, + sqlTemplate: config.sqlTemplate, + numberFormat: config.numberFormat, + }; + case DisplayType.Number: + return { + configType: 'sql', + displayType: DisplayType.Number, + connectionId: config.connection, + sqlTemplate: config.sqlTemplate, + numberFormat: config.numberFormat, + }; + case DisplayType.Pie: + return { + configType: 'sql', + displayType: DisplayType.Pie, + connectionId: config.connection, + sqlTemplate: config.sqlTemplate, + numberFormat: config.numberFormat, + }; + case DisplayType.Search: + case DisplayType.Markdown: + case DisplayType.Heatmap: + logger.error( + { config }, + 'Error converting chart config to external chart - unsupported display type for raw SQL config', + ); + return undefined; + } + + config.displayType satisfies never | undefined; + return undefined; + } const sourceId = config.source?.toString() ?? ''; @@ -100,9 +166,25 @@ const convertToExternalTileChartConfig = ( switch (config.displayType) { case DisplayType.Line: + return { + displayType: DisplayType.Line, + sourceId, + asRatio: + config.seriesReturnType === 'ratio' && + Array.isArray(config.select) && + config.select.length == 2, + alignDateRangeToGranularity: config.alignDateRangeToGranularity, + fillNulls: config.fillNulls !== false, + groupBy: stringValueOrDefault(config.groupBy, undefined), + select: Array.isArray(config.select) + ? config.select.map(convertToExternalSelectItem) + : [DEFAULT_SELECT_ITEM], + compareToPreviousPeriod: config.compareToPreviousPeriod, + numberFormat: config.numberFormat, + }; case DisplayType.StackedBar: return { - displayType: config.displayType, + displayType: DisplayType.StackedBar, sourceId, asRatio: config.seriesReturnType === 'ratio' && @@ -114,9 +196,6 @@ const convertToExternalTileChartConfig = ( select: Array.isArray(config.select) ? config.select.map(convertToExternalSelectItem) : [DEFAULT_SELECT_ITEM], - ...(config.displayType === DisplayType.Line - ? { compareToPreviousPeriod: config.compareToPreviousPeriod } - : {}), numberFormat: config.numberFormat, }; case DisplayType.Number: @@ -181,15 +260,20 @@ const convertToExternalTileChartConfig = ( function convertTileToExternalChart( tile: DashboardDocument['tiles'][number], ): ExternalDashboardTileWithId | undefined { - // HDX-3582: Implement this for Raw SQL charts - if (isRawSqlSavedChartConfig(tile.config)) return undefined; - // Returned in case of a failure converting the saved chart config - const defaultTileConfig: ExternalDashboardTileConfig = { - displayType: 'line', - sourceId: tile.config.source?.toString() ?? '', - select: [DEFAULT_SELECT_ITEM], - }; + const defaultTileConfig: ExternalDashboardTileConfig = + isRawSqlSavedChartConfig(tile.config) + ? { + configType: 'sql', + displayType: 'line', + connectionId: tile.config.connection, + sqlTemplate: tile.config.sqlTemplate, + } + : { + displayType: 'line', + sourceId: tile.config.source?.toString() ?? '', + select: [DEFAULT_SELECT_ITEM], + }; return { ...pick(tile, ['id', 'x', 'y', 'w', 'h']), @@ -238,90 +322,139 @@ export function convertToInternalTileConfig( const name = externalTile.name || ''; let internalConfig: SavedChartConfig; - switch (externalConfig.displayType) { - case 'line': - case 'stacked_bar': - internalConfig = { - ...pick(externalConfig, [ - 'groupBy', - 'numberFormat', - 'alignDateRangeToGranularity', - 'compareToPreviousPeriod', - ]), - displayType: - externalConfig.displayType === 'stacked_bar' - ? DisplayType.StackedBar - : DisplayType.Line, - select: externalConfig.select.map(convertToInternalSelectItem), - source: externalConfig.sourceId, - where: '', - fillNulls: externalConfig.fillNulls === false ? false : undefined, - seriesReturnType: externalConfig.asRatio ? 'ratio' : undefined, - name, - }; - break; - case 'table': - internalConfig = { - ...pick(externalConfig, [ - 'groupBy', - 'numberFormat', - 'having', - 'orderBy', - ]), - displayType: DisplayType.Table, - select: externalConfig.select.map(convertToInternalSelectItem), - source: externalConfig.sourceId, - where: '', - seriesReturnType: externalConfig.asRatio ? 'ratio' : undefined, - name, - }; - break; - case 'number': - internalConfig = { - displayType: DisplayType.Number, - select: [convertToInternalSelectItem(externalConfig.select[0])], - source: externalConfig.sourceId, - where: '', - numberFormat: externalConfig.numberFormat, - name, - }; - break; - case 'pie': - internalConfig = { - ...pick(externalConfig, ['groupBy', 'numberFormat']), - displayType: DisplayType.Pie, - select: [convertToInternalSelectItem(externalConfig.select[0])], - source: externalConfig.sourceId, - where: '', - name, - }; - break; - case 'search': - internalConfig = { - ...pick(externalConfig, ['select', 'where']), - displayType: DisplayType.Search, - source: externalConfig.sourceId, - name, - whereLanguage: externalConfig.whereLanguage ?? 'lucene', - }; - break; - case 'markdown': - internalConfig = { - displayType: DisplayType.Markdown, - markdown: externalConfig.markdown, - source: '', - where: '', - select: [], - name, - }; - break; - default: - // Typecheck to ensure all display types are handled - externalConfig satisfies never; - // We should never hit this due to the typecheck above. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - internalConfig = {} as SavedChartConfig; + if (isRawSqlExternalTileConfig(externalConfig)) { + switch (externalConfig.displayType) { + case 'line': + case 'stacked_bar': + internalConfig = { + configType: 'sql', + ...pick(externalConfig, [ + 'numberFormat', + 'alignDateRangeToGranularity', + 'compareToPreviousPeriod', + ]), + displayType: + externalConfig.displayType === 'stacked_bar' + ? DisplayType.StackedBar + : DisplayType.Line, + fillNulls: externalConfig.fillNulls === false ? false : undefined, + name, + connection: externalConfig.connectionId, + sqlTemplate: externalConfig.sqlTemplate, + } satisfies RawSqlSavedChartConfig; + break; + case 'table': + case 'number': + case 'pie': + internalConfig = { + configType: 'sql', + displayType: + externalConfig.displayType === 'table' + ? DisplayType.Table + : externalConfig.displayType === 'number' + ? DisplayType.Number + : DisplayType.Pie, + name, + connection: externalConfig.connectionId, + sqlTemplate: externalConfig.sqlTemplate, + numberFormat: externalConfig.numberFormat, + } satisfies RawSqlSavedChartConfig; + break; + default: + // Typecheck to ensure all display types are handled + externalConfig satisfies never; + + // We should never hit this due to the typecheck above. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + internalConfig = {} as SavedChartConfig; + } + } else { + switch (externalConfig.displayType) { + case 'line': + case 'stacked_bar': + internalConfig = { + ...pick(externalConfig, [ + 'groupBy', + 'numberFormat', + 'alignDateRangeToGranularity', + 'compareToPreviousPeriod', + ]), + displayType: + externalConfig.displayType === 'stacked_bar' + ? DisplayType.StackedBar + : DisplayType.Line, + select: externalConfig.select.map(convertToInternalSelectItem), + source: externalConfig.sourceId, + where: '', + fillNulls: externalConfig.fillNulls === false ? false : undefined, + seriesReturnType: externalConfig.asRatio ? 'ratio' : undefined, + name, + } satisfies BuilderSavedChartConfig; + break; + case 'table': + internalConfig = { + ...pick(externalConfig, [ + 'groupBy', + 'numberFormat', + 'having', + 'orderBy', + ]), + displayType: DisplayType.Table, + select: externalConfig.select.map(convertToInternalSelectItem), + source: externalConfig.sourceId, + where: '', + seriesReturnType: externalConfig.asRatio ? 'ratio' : undefined, + name, + } satisfies BuilderSavedChartConfig; + break; + case 'number': + internalConfig = { + displayType: DisplayType.Number, + select: [convertToInternalSelectItem(externalConfig.select[0])], + source: externalConfig.sourceId, + where: '', + numberFormat: externalConfig.numberFormat, + name, + } satisfies BuilderSavedChartConfig; + break; + case 'pie': + internalConfig = { + ...pick(externalConfig, ['groupBy', 'numberFormat']), + displayType: DisplayType.Pie, + select: [convertToInternalSelectItem(externalConfig.select[0])], + source: externalConfig.sourceId, + where: '', + name, + } satisfies BuilderSavedChartConfig; + break; + case 'search': + internalConfig = { + ...pick(externalConfig, ['select', 'where']), + displayType: DisplayType.Search, + source: externalConfig.sourceId, + name, + whereLanguage: externalConfig.whereLanguage ?? 'lucene', + } satisfies BuilderSavedChartConfig; + break; + case 'markdown': + internalConfig = { + displayType: DisplayType.Markdown, + markdown: externalConfig.markdown, + source: '', + where: '', + select: [], + name, + } satisfies BuilderSavedChartConfig; + break; + default: + // Typecheck to ensure all display types are handled + externalConfig satisfies never; + + // We should never hit this due to the typecheck above. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + internalConfig = {} as SavedChartConfig; + } } // Omit keys that are null/undefined, so that they're not saved as null in Mongo. diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 6fa96b8afd..aed61d9f75 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -207,6 +207,13 @@ export type ExternalDashboardSelectItem = z.infer< typeof externalDashboardSelectItemSchema >; +const externalDashboardRawSqlChartConfigBaseSchema = z.object({ + configType: z.literal('sql'), + connectionId: objectIdSchema, + sqlTemplate: z.string().max(100000), + numberFormat: NumberFormatSchema.optional(), +}); + const externalDashboardTimeChartConfigSchema = z.object({ sourceId: objectIdSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), @@ -223,11 +230,26 @@ const externalDashboardLineChartConfigSchema = compareToPreviousPeriod: z.boolean().optional(), }); +const externalDashboardLineRawSqlChartConfigSchema = + externalDashboardRawSqlChartConfigBaseSchema.extend({ + displayType: z.literal('line'), + compareToPreviousPeriod: z.boolean().optional(), + fillNulls: z.boolean().optional(), + alignDateRangeToGranularity: z.boolean().optional(), + }); + const externalDashboardBarChartConfigSchema = externalDashboardTimeChartConfigSchema.extend({ displayType: z.literal('stacked_bar'), }); +const externalDashboardBarRawSqlChartConfigSchema = + externalDashboardRawSqlChartConfigBaseSchema.extend({ + displayType: z.literal('stacked_bar'), + fillNulls: z.boolean().optional(), + alignDateRangeToGranularity: z.boolean().optional(), + }); + const externalDashboardTableChartConfigSchema = z.object({ displayType: z.literal('table'), sourceId: objectIdSchema, @@ -239,6 +261,21 @@ const externalDashboardTableChartConfigSchema = z.object({ numberFormat: NumberFormatSchema.optional(), }); +const externalDashboardTableRawSqlChartConfigSchema = + externalDashboardRawSqlChartConfigBaseSchema.extend({ + displayType: z.literal('table'), + }); + +const externalDashboardNumberRawSqlChartConfigSchema = + externalDashboardRawSqlChartConfigBaseSchema.extend({ + displayType: z.literal('number'), + }); + +const externalDashboardPieRawSqlChartConfigSchema = + externalDashboardRawSqlChartConfigBaseSchema.extend({ + displayType: z.literal('pie'), + }); + const externalDashboardNumberChartConfigSchema = z.object({ displayType: z.literal('number'), sourceId: objectIdSchema, @@ -267,8 +304,9 @@ const externalDashboardMarkdownChartConfigSchema = z.object({ markdown: z.string().max(50000).optional(), }); -export const externalDashboardTileConfigSchema = z - .discriminatedUnion('displayType', [ +const externalDashboardBuilderTileConfigSchema = z.discriminatedUnion( + 'displayType', + [ externalDashboardLineChartConfigSchema, externalDashboardBarChartConfigSchema, externalDashboardTableChartConfigSchema, @@ -276,8 +314,52 @@ export const externalDashboardTileConfigSchema = z externalDashboardPieChartConfigSchema, externalDashboardMarkdownChartConfigSchema, externalDashboardSearchChartConfigSchema, - ]) + ], +); + +export type ExternalDashboardBuilderTileConfig = z.infer< + typeof externalDashboardBuilderTileConfigSchema +>; + +const externalDashboardRawSqlTileConfigSchema = z.discriminatedUnion( + 'displayType', + [ + externalDashboardLineRawSqlChartConfigSchema, + externalDashboardBarRawSqlChartConfigSchema, + externalDashboardTableRawSqlChartConfigSchema, + externalDashboardNumberRawSqlChartConfigSchema, + externalDashboardPieRawSqlChartConfigSchema, + ], +); + +export type ExternalDashboardRawSqlTileConfig = z.infer< + typeof externalDashboardRawSqlTileConfigSchema +>; + +export const externalDashboardTileConfigSchema = z + .custom< + ExternalDashboardRawSqlTileConfig | ExternalDashboardBuilderTileConfig + >() .superRefine((data, ctx) => { + // Route to the correct sub-schema based on configType so Zod's + // discriminatedUnion can produce targeted field-level errors rather + // than a generic union failure. + const schema = + data !== null && + typeof data === 'object' && + 'configType' in data && + data.configType === 'sql' + ? externalDashboardRawSqlTileConfigSchema + : externalDashboardBuilderTileConfigSchema; + + const result = schema.safeParse(data); + if (!result.success) { + for (const issue of result.error.issues) { + ctx.addIssue(issue); + } + return; + } + if ( 'asRatio' in data && data.asRatio && @@ -288,6 +370,19 @@ export const externalDashboardTileConfigSchema = z message: 'asRatio can only be used with exactly two select items', }); } + }) + .transform(data => { + // Re-parse through the appropriate sub-schema to strip unknown fields. + // Safe to call .parse() here — superRefine already validated the data, + // so this is guaranteed to succeed. + const schema = + data !== null && + typeof data === 'object' && + 'configType' in data && + data.configType === 'sql' + ? externalDashboardRawSqlTileConfigSchema + : externalDashboardBuilderTileConfigSchema; + return schema.parse(data); }); export type ExternalDashboardTileConfig = z.infer<