From 612388f8e03e9bd47020fc532bfa2c18c7c7134e Mon Sep 17 00:00:00 2001 From: Perseus Date: Sat, 26 Jul 2025 08:45:42 +0000 Subject: [PATCH 1/4] (fix): add support for JSON columns in the service dashboard --- packages/app/src/ServicesDashboardPage.tsx | 36 ++++++++-- .../src/__tests__/serviceDashboard.test.ts | 68 +++++++++++++++++++ .../ServiceDashboardDbQuerySidePanel.tsx | 8 ++- ...rviceDashboardEndpointPerformanceChart.tsx | 56 ++++++++++++--- .../ServiceDashboardEndpointSidePanel.tsx | 8 ++- .../ServiceDashboardSlowestEventsTile.tsx | 8 ++- packages/app/src/hooks/useMetadata.tsx | 27 ++++++++ packages/app/src/serviceDashboard.ts | 28 +++++--- packages/common-utils/src/metadata.ts | 20 ++++++ 9 files changed, 231 insertions(+), 28 deletions(-) create mode 100644 packages/app/src/__tests__/serviceDashboard.test.ts diff --git a/packages/app/src/ServicesDashboardPage.tsx b/packages/app/src/ServicesDashboardPage.tsx index c4e1f134e..99f423a37 100644 --- a/packages/app/src/ServicesDashboardPage.tsx +++ b/packages/app/src/ServicesDashboardPage.tsx @@ -43,6 +43,7 @@ import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor'; import { TimePicker } from '@/components/TimePicker'; import WhereLanguageControlled from '@/components/WhereLanguageControlled'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useJsonColumns } from '@/hooks/useMetadata'; import { withAppNav } from '@/layout'; import SearchInputV2 from '@/SearchInputV2'; import { getExpressions } from '@/serviceDashboard'; @@ -90,7 +91,12 @@ function ServiceSelectControlled({ onCreate?: () => void; } & UseControllerProps) { const { data: source } = useSource({ id: sourceId }); - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const queriedConfig = { ...source, @@ -153,7 +159,12 @@ export function EndpointLatencyChart({ appliedConfig?: AppliedConfig; extraFilters?: Filter[]; }) { - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const [latencyChartType, setLatencyChartType] = useState< 'line' | 'histogram' >('line'); @@ -259,7 +270,12 @@ function HttpTab({ appliedConfig: AppliedConfig; }) { const { data: source } = useSource({ id: appliedConfig.source }); - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const [reqChartType, setReqChartType] = useQueryState( 'reqChartType', @@ -529,7 +545,12 @@ function DatabaseTab({ appliedConfig: AppliedConfig; }) { const { data: source } = useSource({ id: appliedConfig.source }); - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const [chartType, setChartType] = useState<'table' | 'list'>('list'); @@ -776,7 +797,12 @@ function ErrorsTab({ appliedConfig: AppliedConfig; }) { const { data: source } = useSource({ id: appliedConfig.source }); - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); return ( diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts new file mode 100644 index 000000000..2e4cb8d58 --- /dev/null +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -0,0 +1,68 @@ +import type { TSource } from '@hyperdx/common-utils/dist/types'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { getExpressions } from '../serviceDashboard'; + +describe('Service Dashboard', () => { + const mockSource: TSource = { + id: 'test-source', + name: 'Test Source', + kind: SourceKind.Trace, + from: { + databaseName: 'test_db', + tableName: 'otel_traces_json', + }, + connection: 'test-connection', + timestampValueExpression: 'Timestamp', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + serviceNameExpression: 'ServiceName', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + severityTextExpression: 'StatusCode', + }; + + describe('getExpressions', () => { + it('should use map syntax for non-JSON columns by default', () => { + const expressions = getExpressions(mockSource, []); + + expect(expressions.k8sResourceName).toBe( + "SpanAttributes['k8s.resource.name']", + ); + expect(expressions.k8sPodName).toBe("SpanAttributes['k8s.pod.name']"); + expect(expressions.httpScheme).toBe("SpanAttributes['http.scheme']"); + expect(expressions.serverAddress).toBe( + "SpanAttributes['server.address']", + ); + expect(expressions.httpHost).toBe("SpanAttributes['http.host']"); + expect(expressions.dbStatement).toBe( + "coalesce(nullif(SpanAttributes['db.query.text'], ''), nullif(SpanAttributes['db.statement'], ''))", + ); + }); + + it('should use backtick syntax when SpanAttributes is a JSON column', () => { + const expressions = getExpressions(mockSource, ['SpanAttributes']); + + expect(expressions.k8sResourceName).toBe( + 'SpanAttributes.`k8s.resource.name`', + ); + expect(expressions.k8sPodName).toBe('SpanAttributes.`k8s.pod.name`'); + expect(expressions.httpScheme).toBe('SpanAttributes.`http.scheme`'); + expect(expressions.serverAddress).toBe('SpanAttributes.`server.address`'); + expect(expressions.httpHost).toBe('SpanAttributes.`http.host`'); + expect(expressions.dbStatement).toBe( + "coalesce(nullif(SpanAttributes.`db.query.text`, ''), nullif(SpanAttributes.`db.statement`, ''))", + ); + }); + + it('should work with empty jsonColumns array', () => { + const expressions = getExpressions(mockSource); + + // Should default to map syntax + expect(expressions.k8sResourceName).toBe( + "SpanAttributes['k8s.resource.name']", + ); + }); + }); +}); diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx index dd24db524..363ef9f50 100644 --- a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx @@ -9,6 +9,7 @@ import { ChartBox } from '@/components/ChartBox'; import { DBTimeChart } from '@/components/DBTimeChart'; import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils'; import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile'; +import { useJsonColumns } from '@/hooks/useMetadata'; import { getExpressions } from '@/serviceDashboard'; import { useSource } from '@/source'; import { useZIndex, ZIndexContext } from '@/zIndex'; @@ -26,7 +27,12 @@ export default function ServiceDashboardDbQuerySidePanel({ searchedTimeRange: [Date, Date]; }) { const { data: source } = useSource({ id: sourceId }); - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const [dbQuery, setDbQuery] = useQueryState('dbquery', parseAsString); const onClose = useCallback(() => { diff --git a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx index 4fb7c2532..cd67c3ac7 100644 --- a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx +++ b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx @@ -4,6 +4,7 @@ import { Group, Text } from '@mantine/core'; import { MS_NUMBER_FORMAT } from '@/ChartUtils'; import { ChartBox } from '@/components/ChartBox'; import DBListBarChart from '@/components/DBListBarChart'; +import { useJsonColumns } from '@/hooks/useMetadata'; import { getExpressions } from '@/serviceDashboard'; const MAX_NUM_GROUPS = 200; @@ -19,7 +20,12 @@ export default function ServiceDashboardEndpointPerformanceChart({ service?: string; endpoint?: string; }) { - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); if (!source) { return null; @@ -42,6 +48,43 @@ export default function ServiceDashboardEndpointPerformanceChart({ WHERE ${parentSpanWhereCondition} `; + let spanNameColSql = ` + concat( + ${expressions.spanName}, ' ', + if( + has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}), + COALESCE( + NULLIF(${expressions.serverAddress}, ''), + NULLIF(${expressions.httpHost}, '') + ), + '' + ));`; + + const spanAttributesExpression = + source.eventAttributesExpression || 'SpanAttributes'; + + // ClickHouse does not support NULLIF(some_dynamic_column) + // so we instead use toString() and an empty string check to check for + // existence of the serverAddress/httpHost to build the span name + if (jsonColumns.includes(spanAttributesExpression)) { + spanNameColSql = ` + concat( + ${expressions.spanName}, ' ', + if( + has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}), + if( + toString(${expressions.serverAddress}) != '', + toString(${expressions.serverAddress}), + if( + toString(${expressions.httpHost}) != '', + toString(${expressions.httpHost}), + '' + ) + ), + '' + ))`; + } + return ( @@ -60,16 +103,7 @@ export default function ServiceDashboardEndpointPerformanceChart({ select: [ { alias: 'group', - valueExpression: `concat( - ${expressions.spanName}, ' ', - if( - has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}), - COALESCE( - NULLIF(${expressions.serverAddress}, ''), - NULLIF(${expressions.httpHost}, '') - ), - '' - ))`, + valueExpression: spanNameColSql, }, { alias: 'Total Time Spent', diff --git a/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx b/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx index b93a994ef..79a93337b 100644 --- a/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx @@ -13,6 +13,7 @@ import { DBTimeChart } from '@/components/DBTimeChart'; import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils'; import ServiceDashboardEndpointPerformanceChart from '@/components/ServiceDashboardEndpointPerformanceChart'; import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile'; +import { useJsonColumns } from '@/hooks/useMetadata'; import { getExpressions } from '@/serviceDashboard'; import { EndpointLatencyChart } from '@/ServicesDashboardPage'; import { useSource } from '@/source'; @@ -31,7 +32,12 @@ export default function ServiceDashboardEndpointSidePanel({ searchedTimeRange: [Date, Date]; }) { const { data: source } = useSource({ id: sourceId }); - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const [endpoint, setEndpoint] = useQueryState('endpoint', parseAsString); const onClose = useCallback(() => { diff --git a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx index f16016288..a4225f253 100644 --- a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx +++ b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx @@ -8,6 +8,7 @@ import { ChartBox } from '@/components/ChartBox'; import DBRowSidePanel from '@/components/DBRowSidePanel'; import { DBSqlRowTable } from '@/components/DBRowTable'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useJsonColumns } from '@/hooks/useMetadata'; import { getExpressions } from '@/serviceDashboard'; import { useSource } from '@/source'; @@ -30,7 +31,12 @@ export default function SlowestEventsTile({ enabled?: boolean; extraFilters?: Filter[]; }) { - const expressions = getExpressions(source); + const { data: jsonColumns = [] } = useJsonColumns({ + databaseName: source?.from?.databaseName || '', + tableName: source?.from?.tableName || '', + connectionId: source?.connection || '', + }); + const expressions = getExpressions(source, jsonColumns); const [rowId, setRowId] = useQueryState('rowId', parseAsString); const [rowSource, setRowSource] = useQueryState('rowSource', parseAsString); diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 7f2e6083f..10e9273af 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -43,6 +43,33 @@ export function useColumns( }); } +export function useJsonColumns( + { + databaseName, + tableName, + connectionId, + }: { + databaseName: string; + tableName: string; + connectionId: string; + }, + options?: Partial>, +) { + return useQuery({ + queryKey: ['useMetadata.useJsonColumns', { databaseName, tableName }], + queryFn: async () => { + const metadata = getMetadata(); + return metadata.getJsonColumns({ + databaseName, + tableName, + connectionId, + }); + }, + enabled: !!databaseName && !!tableName && !!connectionId, + ...options, + }); +} + export function useAllFields( _tableConnections: TableConnection | TableConnection[], options?: Partial>, diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index 61c376031..a3f30deb2 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -1,7 +1,17 @@ import { TSource } from '@hyperdx/common-utils/dist/types'; -function getDefaults() { +function getDefaults(jsonColumns: string[] = []) { const spanAttributeField = 'SpanAttributes'; + const isJsonColumn = jsonColumns.includes(spanAttributeField); + + // Helper function to format field access based on column type + const formatFieldAccess = (field: string, key: string) => { + if (isJsonColumn) { + return `${field}.\`${key}\``; + } else { + return `${field}['${key}']`; + } + }; return { duration: 'Duration', @@ -11,17 +21,17 @@ function getDefaults() { spanName: 'SpanName', spanKind: 'SpanKind', severityText: 'StatusCode', - k8sResourceName: `${spanAttributeField}['k8s.resource.name']`, - k8sPodName: `${spanAttributeField}['k8s.pod.name']`, - httpScheme: `${spanAttributeField}['http.scheme']`, - serverAddress: `${spanAttributeField}['server.address']`, - httpHost: `${spanAttributeField}['http.host']`, - dbStatement: `coalesce(nullif(${spanAttributeField}['db.query.text'], ''), nullif(${spanAttributeField}['db.statement'], ''))`, + k8sResourceName: formatFieldAccess(spanAttributeField, 'k8s.resource.name'), + k8sPodName: formatFieldAccess(spanAttributeField, 'k8s.pod.name'), + httpScheme: formatFieldAccess(spanAttributeField, 'http.scheme'), + serverAddress: formatFieldAccess(spanAttributeField, 'server.address'), + httpHost: formatFieldAccess(spanAttributeField, 'http.host'), + dbStatement: `coalesce(nullif(${formatFieldAccess(spanAttributeField, 'db.query.text')}, ''), nullif(${formatFieldAccess(spanAttributeField, 'db.statement')}, ''))`, }; } -export function getExpressions(source?: TSource) { - const defaults = getDefaults(); +export function getExpressions(source?: TSource, jsonColumns: string[] = []) { + const defaults = getDefaults(jsonColumns); const fieldExpressions = { // General diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index 216b89b21..013a4050f 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -147,6 +147,26 @@ export class Metadata { ); } + async getJsonColumns({ + databaseName, + tableName, + connectionId, + }: { + databaseName: string; + tableName: string; + connectionId: string; + }) { + const columns = await this.getColumns({ + databaseName, + tableName, + connectionId, + }); + + return columns + .filter(column => column.type.startsWith('JSON')) + .map(column => column.name); + } + async getMaterializedColumnsLookupTable({ databaseName, tableName, From 1f6e6f200e9a1f6c896cb79b3288db7fe637bb8d Mon Sep 17 00:00:00 2001 From: Perseus Date: Sat, 26 Jul 2025 09:10:06 +0000 Subject: [PATCH 2/4] (fix): handle JSON columns in db statement expressions --- ...rviceDashboardEndpointPerformanceChart.tsx | 29 ++++++++++--------- packages/app/src/serviceDashboard.ts | 24 +++++++++++++-- packages/common-utils/src/metadata.ts | 1 + 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx index cd67c3ac7..311003c50 100644 --- a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx +++ b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx @@ -68,21 +68,22 @@ export default function ServiceDashboardEndpointPerformanceChart({ // existence of the serverAddress/httpHost to build the span name if (jsonColumns.includes(spanAttributesExpression)) { spanNameColSql = ` - concat( - ${expressions.spanName}, ' ', - if( - has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}), + concat( + ${expressions.spanName}, ' ', if( - toString(${expressions.serverAddress}) != '', - toString(${expressions.serverAddress}), - if( - toString(${expressions.httpHost}) != '', - toString(${expressions.httpHost}), - '' - ) - ), - '' - ))`; + has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}), + if( + toString(${expressions.serverAddress}) != '', + toString(${expressions.serverAddress}), + if( + toString(${expressions.httpHost}) != '', + toString(${expressions.httpHost}), + '' + ) + ), + '' + ) + )`; } return ( diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index a3f30deb2..8fe9b4129 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -4,7 +4,6 @@ function getDefaults(jsonColumns: string[] = []) { const spanAttributeField = 'SpanAttributes'; const isJsonColumn = jsonColumns.includes(spanAttributeField); - // Helper function to format field access based on column type const formatFieldAccess = (field: string, key: string) => { if (isJsonColumn) { return `${field}.\`${key}\``; @@ -13,6 +12,27 @@ function getDefaults(jsonColumns: string[] = []) { } }; + let dbStatement = `coalesce(nullif(${formatFieldAccess(spanAttributeField, 'db.query.text')}, ''), nullif(${formatFieldAccess(spanAttributeField, 'db.statement')}, ''))`; + + // ClickHouse does not support NULLIF(some_dynamic_column) + // so we instead use toString() and an empty string check to check for + // existence of the serverAddress/httpHost to build the span name + if (isJsonColumn) { + dbStatement = ` + coalesce( + if( + toString(${formatFieldAccess(spanAttributeField, 'db.query.text')}) != '', + toString(${formatFieldAccess(spanAttributeField, 'db.query.text')}), + if( + toString(${formatFieldAccess(spanAttributeField, 'db.statement')}) != '', + toString(${formatFieldAccess(spanAttributeField, 'db.statement')}), + '' + ) + ) + ) + `; + } + return { duration: 'Duration', durationPrecision: 9, @@ -26,7 +46,7 @@ function getDefaults(jsonColumns: string[] = []) { httpScheme: formatFieldAccess(spanAttributeField, 'http.scheme'), serverAddress: formatFieldAccess(spanAttributeField, 'server.address'), httpHost: formatFieldAccess(spanAttributeField, 'http.host'), - dbStatement: `coalesce(nullif(${formatFieldAccess(spanAttributeField, 'db.query.text')}, ''), nullif(${formatFieldAccess(spanAttributeField, 'db.statement')}, ''))`, + dbStatement, }; } diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index 013a4050f..3d7472bcc 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -162,6 +162,7 @@ export class Metadata { connectionId, }); + // TODO: should we use .includes() to handle Array(JSON) and other variants? return columns .filter(column => column.type.startsWith('JSON')) .map(column => column.name); From d02a07920b787f4b02e96fb046a536eef17ba668 Mon Sep 17 00:00:00 2001 From: Perseus Date: Sat, 2 Aug 2025 04:25:27 +0000 Subject: [PATCH 3/4] (refactor): use filterColumnMetaByType instead of creating a separate method for filtering json columns (refactor): create generalized method to build coalesced field selection queries (refactor): address minor PR review comments --- .../src/__tests__/serviceDashboard.test.ts | 81 ++++++++- packages/app/src/hooks/useMetadata.tsx | 13 +- packages/app/src/serviceDashboard.ts | 155 ++++++++++++++---- packages/common-utils/src/metadata.ts | 21 --- 4 files changed, 211 insertions(+), 59 deletions(-) diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index 2e4cb8d58..827d944e0 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -1,7 +1,14 @@ import type { TSource } from '@hyperdx/common-utils/dist/types'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; -import { getExpressions } from '../serviceDashboard'; +import { + getExpressions, + makeCoalescedFieldsAccessQuery, +} from '../serviceDashboard'; + +function removeAllWhitespace(str: string) { + return str.replace(/\s|\t|\n/g, ''); +} describe('Service Dashboard', () => { const mockSource: TSource = { @@ -51,8 +58,11 @@ describe('Service Dashboard', () => { expect(expressions.httpScheme).toBe('SpanAttributes.`http.scheme`'); expect(expressions.serverAddress).toBe('SpanAttributes.`server.address`'); expect(expressions.httpHost).toBe('SpanAttributes.`http.host`'); - expect(expressions.dbStatement).toBe( - "coalesce(nullif(SpanAttributes.`db.query.text`, ''), nullif(SpanAttributes.`db.statement`, ''))", + const resultWithWhitespaceStripped = removeAllWhitespace( + expressions.dbStatement, + ); + expect(resultWithWhitespaceStripped).toEqual( + `coalesce(if(toString(SpanAttributes.\`db.query.text\`)!='',toString(SpanAttributes.\`db.query.text\`),if(toString(SpanAttributes.\`db.statement\`)!='',toString(SpanAttributes.\`db.statement\`),'')))`, ); }); @@ -65,4 +75,69 @@ describe('Service Dashboard', () => { ); }); }); + + describe('makeCoalescedFieldsAccessQuery', () => { + it('should throw an error if an empty list of fields is passed', () => { + expect(() => { + makeCoalescedFieldsAccessQuery([], false); + }).toThrowError( + 'Empty fields array passed while trying to build a coalesced field access query', + ); + }); + + it('should throw an error if more than 100 fields are passed', () => { + expect(() => { + makeCoalescedFieldsAccessQuery(Array(101).fill('field'), false); + }).toThrowError( + 'Too many fields (101) passed while trying to build a coalesced field access query. Maximum allowed is 100', + ); + }); + + it('should handle single field for non-JSON columns', () => { + const result = makeCoalescedFieldsAccessQuery(['field1'], false); + expect(result).toBe("nullif(field1, '')"); + }); + + it('should handle single field for JSON columns', () => { + const result = makeCoalescedFieldsAccessQuery(['field1'], true); + expect(result).toBe("if(toString(field1) != '', toString(field1), '')"); + }); + + it('should handle multiple fields for non-JSON columns', () => { + const result = makeCoalescedFieldsAccessQuery( + ['field1', 'field2'], + false, + ); + expect(result).toBe("coalesce(nullif(field1, ''), nullif(field2, ''))"); + }); + + it('should handle multiple fields for JSON columns', () => { + const result = makeCoalescedFieldsAccessQuery(['field1', 'field2'], true); + const resultWithWhitespaceStripped = removeAllWhitespace(result); + expect(resultWithWhitespaceStripped).toEqual( + `coalesce(if(toString(field1)!='',toString(field1),if(toString(field2)!='',toString(field2),'')))`, + ); + }); + + it('should handle three fields for JSON columns', () => { + const result = makeCoalescedFieldsAccessQuery( + ['field1', 'field2', 'field3'], + true, + ); + const resultWithWhitespaceStripped = removeAllWhitespace(result); + expect(resultWithWhitespaceStripped).toEqual( + `coalesce(if(toString(field1)!='',toString(field1),if(toString(field2)!='',toString(field2),if(toString(field3)!='',toString(field3),''))))`, + ); + }); + + it('should handle three fields for non-JSON columns', () => { + const result = makeCoalescedFieldsAccessQuery( + ['field1', 'field2', 'field3'], + false, + ); + expect(result).toBe( + "coalesce(nullif(field1, ''), nullif(field2, ''), nullif(field3, ''))", + ); + }); + }); }); diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 10e9273af..f43b5d0e9 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -1,5 +1,9 @@ import objectHash from 'object-hash'; -import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; +import { + ColumnMeta, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; import { Field, TableConnection, @@ -59,11 +63,16 @@ export function useJsonColumns( queryKey: ['useMetadata.useJsonColumns', { databaseName, tableName }], queryFn: async () => { const metadata = getMetadata(); - return metadata.getJsonColumns({ + const columns = await metadata.getColumns({ databaseName, tableName, connectionId, }); + return ( + filterColumnMetaByType(columns, [JSDataType.JSON])?.map( + column => column.name, + ) ?? [] + ); }, enabled: !!databaseName && !!tableName && !!connectionId, ...options, diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index 8fe9b4129..32b9dc3c3 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -1,37 +1,103 @@ import { TSource } from '@hyperdx/common-utils/dist/types'; -function getDefaults(jsonColumns: string[] = []) { - const spanAttributeField = 'SpanAttributes'; - const isJsonColumn = jsonColumns.includes(spanAttributeField); +const COALESCE_FIELDS_LIMIT = 100; - const formatFieldAccess = (field: string, key: string) => { - if (isJsonColumn) { - return `${field}.\`${key}\``; +// Helper function to format field access based on column type +function formatFieldAccess( + field: string, + key: string, + isJsonColumn: boolean, +): string { + return isJsonColumn ? `${field}.\`${key}\`` : `${field}['${key}']`; +} + +/** + * Creates a 'coalesced' SQL query that checks whether each given field exists + * and returns the first non-empty value. + * + * The list of fields should be ordered from highest precedence to lowest. + * + * @param fields list of fields (in order) to coalesce + * @param isJSONColumn whether the fields are JSON columns + * @returns a SQL query string that coalesces the fields + */ +export function makeCoalescedFieldsAccessQuery( + fields: string[], + isJSONColumn: boolean, +): string { + if (fields.length === 0) { + throw new Error( + 'Empty fields array passed while trying to build a coalesced field access query', + ); + } + + if (fields.length > COALESCE_FIELDS_LIMIT) { + throw new Error( + `Too many fields (${fields.length}) passed while trying to build a coalesced field access query. Maximum allowed is ${COALESCE_FIELDS_LIMIT}`, + ); + } + + if (fields.length === 1) { + if (isJSONColumn) { + return `if(toString(${fields[0]}) != '', toString(${fields[0]}), '')`; } else { - return `${field}['${key}']`; + return `nullif(${fields[0]}, '')`; } - }; + } - let dbStatement = `coalesce(nullif(${formatFieldAccess(spanAttributeField, 'db.query.text')}, ''), nullif(${formatFieldAccess(spanAttributeField, 'db.statement')}, ''))`; - - // ClickHouse does not support NULLIF(some_dynamic_column) - // so we instead use toString() and an empty string check to check for - // existence of the serverAddress/httpHost to build the span name - if (isJsonColumn) { - dbStatement = ` - coalesce( - if( - toString(${formatFieldAccess(spanAttributeField, 'db.query.text')}) != '', - toString(${formatFieldAccess(spanAttributeField, 'db.query.text')}), - if( - toString(${formatFieldAccess(spanAttributeField, 'db.statement')}) != '', - toString(${formatFieldAccess(spanAttributeField, 'db.statement')}), - '' - ) - ) - ) - `; + if (isJSONColumn) { + // For JSON columns, build nested if statements + let query = ''; + for (let i = 0; i < fields.length; i++) { + const field = fields[i]; + const isLast = i === fields.length - 1; + + query += `if( +toString(${field}) != '', +toString(${field}),`; + + if (isLast) { + query += `''\n`; + } else { + query += '\n'; + } + } + + // Close all the if statements + for (let i = 0; i < fields.length; i++) { + query += ')'; + } + + return `coalesce(\n${query}\n)`; + } else { + // For non-JSON columns, use nullif with coalesce + const nullifExpressions = fields.map(field => `nullif(${field}, '')`); + return `coalesce(${nullifExpressions.join(', ')})`; } +} + +function getDefaults({ + spanAttributeField = 'SpanAttributes', + isAttributeFieldJSON = false, +}: { + spanAttributeField?: string; + isAttributeFieldJSON?: boolean; +} = {}) { + const dbStatement = makeCoalescedFieldsAccessQuery( + [ + formatFieldAccess( + spanAttributeField, + 'db.query.text', + isAttributeFieldJSON, + ), + formatFieldAccess( + spanAttributeField, + 'db.statement', + isAttributeFieldJSON, + ), + ], + isAttributeFieldJSON, + ); return { duration: 'Duration', @@ -41,17 +107,40 @@ function getDefaults(jsonColumns: string[] = []) { spanName: 'SpanName', spanKind: 'SpanKind', severityText: 'StatusCode', - k8sResourceName: formatFieldAccess(spanAttributeField, 'k8s.resource.name'), - k8sPodName: formatFieldAccess(spanAttributeField, 'k8s.pod.name'), - httpScheme: formatFieldAccess(spanAttributeField, 'http.scheme'), - serverAddress: formatFieldAccess(spanAttributeField, 'server.address'), - httpHost: formatFieldAccess(spanAttributeField, 'http.host'), + k8sResourceName: formatFieldAccess( + spanAttributeField, + 'k8s.resource.name', + isAttributeFieldJSON, + ), + k8sPodName: formatFieldAccess( + spanAttributeField, + 'k8s.pod.name', + isAttributeFieldJSON, + ), + httpScheme: formatFieldAccess( + spanAttributeField, + 'http.scheme', + isAttributeFieldJSON, + ), + serverAddress: formatFieldAccess( + spanAttributeField, + 'server.address', + isAttributeFieldJSON, + ), + httpHost: formatFieldAccess( + spanAttributeField, + 'http.host', + isAttributeFieldJSON, + ), dbStatement, }; } export function getExpressions(source?: TSource, jsonColumns: string[] = []) { - const defaults = getDefaults(jsonColumns); + const spanAttributeField = + source?.eventAttributesExpression || 'SpanAttributes'; + const isAttributeFieldJSON = jsonColumns.includes(spanAttributeField); + const defaults = getDefaults({ spanAttributeField, isAttributeFieldJSON }); const fieldExpressions = { // General diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index 3d7472bcc..216b89b21 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -147,27 +147,6 @@ export class Metadata { ); } - async getJsonColumns({ - databaseName, - tableName, - connectionId, - }: { - databaseName: string; - tableName: string; - connectionId: string; - }) { - const columns = await this.getColumns({ - databaseName, - tableName, - connectionId, - }); - - // TODO: should we use .includes() to handle Array(JSON) and other variants? - return columns - .filter(column => column.type.startsWith('JSON')) - .map(column => column.name); - } - async getMaterializedColumnsLookupTable({ databaseName, tableName, From bfff2feca53d4a483cd1f08eaf96582164e23108 Mon Sep 17 00:00:00 2001 From: Perseus Date: Sat, 2 Aug 2025 04:34:22 +0000 Subject: [PATCH 4/4] (refactor): use makeCoalescedFieldsAccessQuery in the 'top most time consuming ops' panel --- ...rviceDashboardEndpointPerformanceChart.tsx | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx index 311003c50..9bcd0d462 100644 --- a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx +++ b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx @@ -5,7 +5,10 @@ import { MS_NUMBER_FORMAT } from '@/ChartUtils'; import { ChartBox } from '@/components/ChartBox'; import DBListBarChart from '@/components/DBListBarChart'; import { useJsonColumns } from '@/hooks/useMetadata'; -import { getExpressions } from '@/serviceDashboard'; +import { + getExpressions, + makeCoalescedFieldsAccessQuery, +} from '@/serviceDashboard'; const MAX_NUM_GROUPS = 200; @@ -67,20 +70,16 @@ export default function ServiceDashboardEndpointPerformanceChart({ // so we instead use toString() and an empty string check to check for // existence of the serverAddress/httpHost to build the span name if (jsonColumns.includes(spanAttributesExpression)) { + const coalescedServerAddress = makeCoalescedFieldsAccessQuery( + [expressions.serverAddress, expressions.httpHost], + true, + ); spanNameColSql = ` concat( ${expressions.spanName}, ' ', if( has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}), - if( - toString(${expressions.serverAddress}) != '', - toString(${expressions.serverAddress}), - if( - toString(${expressions.httpHost}) != '', - toString(${expressions.httpHost}), - '' - ) - ), + ${coalescedServerAddress}, '' ) )`;