fix(json-type): add support for JSON columns in the service dashboard

packages/app/src/ServicesDashboardPage.tsx

@@ -43,6 +43,7 @@ import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor';
 import { TimePicker } from '@/components/TimePicker';
 import WhereLanguageControlled from '@/components/WhereLanguageControlled';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { withAppNav } from '@/layout';
 import SearchInputV2 from '@/SearchInputV2';
 import { getExpressions } from '@/serviceDashboard';
@@ -90,7 +91,12 @@ function ServiceSelectControlled({
   onCreate?: () => void;
 } & UseControllerProps<any>) {
   const { data: source } = useSource({ id: sourceId });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const queriedConfig = {
     ...source,
@@ -153,7 +159,12 @@ export function EndpointLatencyChart({
   appliedConfig?: AppliedConfig;
   extraFilters?: Filter[];
 }) {
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
   const [latencyChartType, setLatencyChartType] = useState<
     'line' | 'histogram'
   >('line');
@@ -259,7 +270,12 @@ function HttpTab({
   appliedConfig: AppliedConfig;
 }) {
   const { data: source } = useSource({ id: appliedConfig.source });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [reqChartType, setReqChartType] = useQueryState(
     'reqChartType',
@@ -529,7 +545,12 @@ function DatabaseTab({
   appliedConfig: AppliedConfig;
 }) {
   const { data: source } = useSource({ id: appliedConfig.source });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [chartType, setChartType] = useState<'table' | 'list'>('list');
 
@@ -776,7 +797,12 @@ function ErrorsTab({
   appliedConfig: AppliedConfig;
 }) {
   const { data: source } = useSource({ id: appliedConfig.source });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   return (
     <Grid mt="md" grow={false} w="100%" maw="100%" overflow="hidden">

packages/app/src/__tests__/serviceDashboard.test.ts

@@ -0,0 +1,143 @@
+import type { TSource } from '@hyperdx/common-utils/dist/types';
+import { SourceKind } from '@hyperdx/common-utils/dist/types';
+
+import {
+  getExpressions,
+  makeCoalescedFieldsAccessQuery,
+} from '../serviceDashboard';
+
+function removeAllWhitespace(str: string) {
+  return str.replace(/\s|\t|\n/g, '');
+}
+
+describe('Service Dashboard', () => {
+  const mockSource: TSource = {
+    id: 'test-source',
+    name: 'Test Source',
+    kind: SourceKind.Trace,
+    from: {
+      databaseName: 'test_db',
+      tableName: 'otel_traces_json',
+    },
+    connection: 'test-connection',
+    timestampValueExpression: 'Timestamp',
+    durationExpression: 'Duration',
+    durationPrecision: 9,
+    traceIdExpression: 'TraceId',
+    serviceNameExpression: 'ServiceName',
+    spanNameExpression: 'SpanName',
+    spanKindExpression: 'SpanKind',
+    severityTextExpression: 'StatusCode',
+  };
+
+  describe('getExpressions', () => {
+    it('should use map syntax for non-JSON columns by default', () => {
+      const expressions = getExpressions(mockSource, []);
+
+      expect(expressions.k8sResourceName).toBe(
+        "SpanAttributes['k8s.resource.name']",
+      );
+      expect(expressions.k8sPodName).toBe("SpanAttributes['k8s.pod.name']");
+      expect(expressions.httpScheme).toBe("SpanAttributes['http.scheme']");
+      expect(expressions.serverAddress).toBe(
+        "SpanAttributes['server.address']",
+      );
+      expect(expressions.httpHost).toBe("SpanAttributes['http.host']");
+      expect(expressions.dbStatement).toBe(
+        "coalesce(nullif(SpanAttributes['db.query.text'], ''), nullif(SpanAttributes['db.statement'], ''))",
+      );
+    });
+
+    it('should use backtick syntax when SpanAttributes is a JSON column', () => {
+      const expressions = getExpressions(mockSource, ['SpanAttributes']);
+
+      expect(expressions.k8sResourceName).toBe(
+        'SpanAttributes.`k8s.resource.name`',
+      );
+      expect(expressions.k8sPodName).toBe('SpanAttributes.`k8s.pod.name`');
+      expect(expressions.httpScheme).toBe('SpanAttributes.`http.scheme`');
+      expect(expressions.serverAddress).toBe('SpanAttributes.`server.address`');
+      expect(expressions.httpHost).toBe('SpanAttributes.`http.host`');
+      const resultWithWhitespaceStripped = removeAllWhitespace(
+        expressions.dbStatement,
+      );
+      expect(resultWithWhitespaceStripped).toEqual(
+        `coalesce(if(toString(SpanAttributes.\`db.query.text\`)!='',toString(SpanAttributes.\`db.query.text\`),if(toString(SpanAttributes.\`db.statement\`)!='',toString(SpanAttributes.\`db.statement\`),'')))`,
+      );
+    });
+
+    it('should work with empty jsonColumns array', () => {
+      const expressions = getExpressions(mockSource);
+
+      // Should default to map syntax
+      expect(expressions.k8sResourceName).toBe(
+        "SpanAttributes['k8s.resource.name']",
+      );
+    });
+  });
+
+  describe('makeCoalescedFieldsAccessQuery', () => {
+    it('should throw an error if an empty list of fields is passed', () => {
+      expect(() => {
+        makeCoalescedFieldsAccessQuery([], false);
+      }).toThrowError(
+        'Empty fields array passed while trying to build a coalesced field access query',
+      );
+    });
+
+    it('should throw an error if more than 100 fields are passed', () => {
+      expect(() => {
+        makeCoalescedFieldsAccessQuery(Array(101).fill('field'), false);
+      }).toThrowError(
+        'Too many fields (101) passed while trying to build a coalesced field access query. Maximum allowed is 100',
+      );
+    });
+
+    it('should handle single field for non-JSON columns', () => {
+      const result = makeCoalescedFieldsAccessQuery(['field1'], false);
+      expect(result).toBe("nullif(field1, '')");
+    });
+
+    it('should handle single field for JSON columns', () => {
+      const result = makeCoalescedFieldsAccessQuery(['field1'], true);
+      expect(result).toBe("if(toString(field1) != '', toString(field1), '')");
+    });
+
+    it('should handle multiple fields for non-JSON columns', () => {
+      const result = makeCoalescedFieldsAccessQuery(
+        ['field1', 'field2'],
+        false,
+      );
+      expect(result).toBe("coalesce(nullif(field1, ''), nullif(field2, ''))");
+    });
+
+    it('should handle multiple fields for JSON columns', () => {
+      const result = makeCoalescedFieldsAccessQuery(['field1', 'field2'], true);
+      const resultWithWhitespaceStripped = removeAllWhitespace(result);
+      expect(resultWithWhitespaceStripped).toEqual(
+        `coalesce(if(toString(field1)!='',toString(field1),if(toString(field2)!='',toString(field2),'')))`,
+      );
+    });
+
+    it('should handle three fields for JSON columns', () => {
+      const result = makeCoalescedFieldsAccessQuery(
+        ['field1', 'field2', 'field3'],
+        true,
+      );
+      const resultWithWhitespaceStripped = removeAllWhitespace(result);
+      expect(resultWithWhitespaceStripped).toEqual(
+        `coalesce(if(toString(field1)!='',toString(field1),if(toString(field2)!='',toString(field2),if(toString(field3)!='',toString(field3),''))))`,
+      );
+    });
+
+    it('should handle three fields for non-JSON columns', () => {
+      const result = makeCoalescedFieldsAccessQuery(
+        ['field1', 'field2', 'field3'],
+        false,
+      );
+      expect(result).toBe(
+        "coalesce(nullif(field1, ''), nullif(field2, ''), nullif(field3, ''))",
+      );
+    });
+  });
+});

packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx

@@ -9,6 +9,7 @@ import { ChartBox } from '@/components/ChartBox';
 import { DBTimeChart } from '@/components/DBTimeChart';
 import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
 import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 import { useSource } from '@/source';
 import { useZIndex, ZIndexContext } from '@/zIndex';
@@ -26,7 +27,12 @@ export default function ServiceDashboardDbQuerySidePanel({
   searchedTimeRange: [Date, Date];
 }) {
   const { data: source } = useSource({ id: sourceId });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [dbQuery, setDbQuery] = useQueryState('dbquery', parseAsString);
   const onClose = useCallback(() => {

packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx

@@ -4,7 +4,11 @@ import { Group, Text } from '@mantine/core';
 import { MS_NUMBER_FORMAT } from '@/ChartUtils';
 import { ChartBox } from '@/components/ChartBox';
 import DBListBarChart from '@/components/DBListBarChart';
-import { getExpressions } from '@/serviceDashboard';
+import { useJsonColumns } from '@/hooks/useMetadata';
+import {
+  getExpressions,
+  makeCoalescedFieldsAccessQuery,
+} from '@/serviceDashboard';
 
 const MAX_NUM_GROUPS = 200;
 
@@ -19,7 +23,12 @@ export default function ServiceDashboardEndpointPerformanceChart({
   service?: string;
   endpoint?: string;
 }) {
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   if (!source) {
     return null;
@@ -42,6 +51,40 @@ export default function ServiceDashboardEndpointPerformanceChart({
     WHERE ${parentSpanWhereCondition}
     `;
 
+  let spanNameColSql = `
+    concat(
+      ${expressions.spanName}, ' ',
+      if(
+        has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}),
+          COALESCE(
+            NULLIF(${expressions.serverAddress}, ''),
+            NULLIF(${expressions.httpHost}, '')
+          ),
+          ''
+    ));`;
+
+  const spanAttributesExpression =
+    source.eventAttributesExpression || 'SpanAttributes';
+
+  // ClickHouse does not support NULLIF(some_dynamic_column)
+  // so we instead use toString() and an empty string check to check for
+  // existence of the serverAddress/httpHost to build the span name
+  if (jsonColumns.includes(spanAttributesExpression)) {
+    const coalescedServerAddress = makeCoalescedFieldsAccessQuery(
+      [expressions.serverAddress, expressions.httpHost],
+      true,
+    );
+    spanNameColSql = `
+      concat(
+        ${expressions.spanName}, ' ',
+        if(
+          has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}),
+          ${coalescedServerAddress},
+          ''
+        )
+      )`;
+  }
+
   return (
     <ChartBox style={{ height: 350, overflow: 'auto' }}>
       <Group justify="space-between" align="center" mb="sm">
@@ -60,16 +103,7 @@ export default function ServiceDashboardEndpointPerformanceChart({
             select: [
               {
                 alias: 'group',
-                valueExpression: `concat(
-                  ${expressions.spanName}, ' ',
-                  if(
-                    has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}),
-                    COALESCE(
-                      NULLIF(${expressions.serverAddress}, ''),
-                      NULLIF(${expressions.httpHost}, '')
-                    ),
-                    ''
-                  ))`,
+                valueExpression: spanNameColSql,
               },
               {
                 alias: 'Total Time Spent',

packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx

@@ -13,6 +13,7 @@ import { DBTimeChart } from '@/components/DBTimeChart';
 import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
 import ServiceDashboardEndpointPerformanceChart from '@/components/ServiceDashboardEndpointPerformanceChart';
 import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 import { EndpointLatencyChart } from '@/ServicesDashboardPage';
 import { useSource } from '@/source';
@@ -31,7 +32,12 @@ export default function ServiceDashboardEndpointSidePanel({
   searchedTimeRange: [Date, Date];
 }) {
   const { data: source } = useSource({ id: sourceId });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [endpoint, setEndpoint] = useQueryState('endpoint', parseAsString);
   const onClose = useCallback(() => {

packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx

@@ -8,6 +8,7 @@ import { ChartBox } from '@/components/ChartBox';
 import DBRowSidePanel from '@/components/DBRowSidePanel';
 import { DBSqlRowTable } from '@/components/DBRowTable';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 import { useSource } from '@/source';
 
@@ -30,7 +31,12 @@ export default function SlowestEventsTile({
   enabled?: boolean;
   extraFilters?: Filter[];
 }) {
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [rowId, setRowId] = useQueryState('rowId', parseAsString);
   const [rowSource, setRowSource] = useQueryState('rowSource', parseAsString);