Add benchmark tests

Author: amortemousque

.claude/commands/new-benchmark.md

@@ -0,0 +1,15 @@
+# CPU Benchmark using Chrome DevTools Protocol
+
+Run a CPU usage benchmark test using the Chrome DevTools Protocol.
+
+This command will:
+1. Launch a browser with CDP enabled
+2. Navigate to a test page with SDK loaded
+3. Profile CPU usage during user interactions
+4. Report detailed CPU metrics including:
+   - Total CPU time
+   - SDK-specific CPU time
+   - Per-function CPU breakdown
+   - Timeline visualization
+
+The benchmark uses the Profiler domain of CDP to capture precise CPU measurements.

benchmark/helpers.ts

@@ -0,0 +1,79 @@
+import { test } from '@playwright/test'
+import type { Page } from '@playwright/test'
+import type { BrowserWindow, Metrics } from 'profiling.type'
+import { startProfiling } from './profilers'
+import { reportMetricsToConsole } from './reporters/console'
+
+const bundleUrl = 'https://www.datadoghq-browser-agent.com/us1/v6/datadog-rum.js'
+
+export const scenarioConfigurations = ['none', 'rum', 'rum_replay', 'rum_profiling'] as const
+
+type ScenarioConfiguration = (typeof scenarioConfigurations)[number]
+
+async function injectSDK(page: Page, scenarioConfiguration: ScenarioConfiguration) {
+  await page.addInitScript(`
+    function loadSDK() {
+      (function(h,o,u,n,d) {
+        h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
+        d=o.createElement(u);d.async=1;d.src=n
+        n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
+      })(window,document,'script','${bundleUrl}','DD_RUM')
+
+      window.DD_RUM.onReady(function() {
+        window.DD_RUM.init({
+          clientToken: 'xxx',
+          applicationId: 'xxx',
+          site: 'datadoghq.com',
+          profilingSampleRate: ${scenarioConfiguration === 'rum_replay' ? 100 : 0},
+          sessionReplaySampleRate: ${scenarioConfiguration === 'rum_profiling' ? 100 : 0},
+        })
+      })
+    }
+
+    document.addEventListener("readystatechange", (event) => {
+      if (document.readyState === "interactive") {
+        loadSDK()
+      }
+    });
+  `)
+}
+
+type TestRunner = (page: Page, takeMeasurements: () => Promise<void>) => Promise<void> | void
+
+async function stopSession(page: Page) {
+  await page.evaluate(() => {
+    ;(window as BrowserWindow).DD_RUM?.stopSession()
+  })
+}
+
+export function createBenchmarkTest(scenarioName: string) {
+  return {
+    run(runner: TestRunner) {
+      const metrics: Record<string, Metrics> = {}
+      scenarioConfigurations.forEach((scenarioConfiguration) => {
+        test(`${scenarioName} ${scenarioConfiguration}`, async ({ page }) => {
+          await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' })
+
+          const { stopProfiling, takeMeasurements } = await startProfiling(page)
+
+          if (scenarioConfiguration !== 'none') {
+            await injectSDK(page, scenarioConfiguration)
+          }
+
+          await runner(page, takeMeasurements)
+
+          if (scenarioConfiguration !== 'none') {
+            await stopSession(page) // Flush events
+          }
+
+          metrics[scenarioConfiguration] = await stopProfiling()
+          await page.close()
+        })
+      })
+
+      test.afterAll(() => {
+        reportMetricsToConsole(metrics)
+      })
+    },
+  }
+}

benchmark/playwright.config.ts

@@ -0,0 +1,19 @@
+import { defineConfig, devices } from '@playwright/test'
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  testDir: './scenarios',
+  testMatch: '**/*.scenario.ts',
+  tsconfig: './tsconfig.json',
+  fullyParallel: true,
+  workers: 1,
+  retries: 0,
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+})

benchmark/profilers/index.ts

@@ -0,0 +1,4 @@
+export * from './startProfiling'
+export * from './startMemoryProfiling'
+export * from './startCpuProfiling'
+export * from './startWebVitalsProfiling'

benchmark/profilers/startCpuProfiling.ts

@@ -0,0 +1,37 @@
+import type { CDPSession } from '@playwright/test'
+
+export async function startCPUProfiling(client: CDPSession) {
+  await client.send('Profiler.enable')
+  await client.send('Profiler.start')
+
+  let totalConsumption = 0
+
+  async function stopAndAddProfile() {
+    const { profile } = await client.send('Profiler.stop')
+
+    const timeDeltaForNodeId = new Map<number, number>()
+
+    for (let index = 0; index < profile.samples!.length; index += 1) {
+      const nodeId = profile.samples![index]
+      timeDeltaForNodeId.set(nodeId, (timeDeltaForNodeId.get(nodeId) || 0) + profile.timeDeltas![index])
+    }
+
+    for (const node of profile.nodes) {
+      const consumption = timeDeltaForNodeId.get(node.id) || 0
+      totalConsumption += consumption
+    }
+  }
+
+  return {
+    takeCPUMeasurements: async () => {
+      // We need to restart profiling at each "measurement" because the running profile gets reset
+      // on each navigation.
+      await stopAndAddProfile()
+      await client.send('Profiler.start')
+    },
+    stopCPUProfiling: async () => {
+      await stopAndAddProfile()
+      return totalConsumption
+    },
+  }
+}

benchmark/profilers/startMemoryProfiling.ts

@@ -0,0 +1,48 @@
+import type { CDPSession } from '@playwright/test'
+
+export async function startMemoryProfiling(client: CDPSession) {
+  await client.send('HeapProfiler.enable')
+  await client.send('HeapProfiler.startSampling', {
+    // Set a low sampling interval to have more precise measurement
+    samplingInterval: 100,
+  })
+
+  const measurements: Array<{ totalConsumption: number }> = []
+
+  return {
+    takeMemoryMeasurements: async () => {
+      await client.send('HeapProfiler.collectGarbage')
+      const { profile } = await client.send('HeapProfiler.getSamplingProfile')
+
+      const sizeForNodeId = new Map<number, number>()
+
+      for (const sample of profile.samples) {
+        sizeForNodeId.set(sample.nodeId, (sizeForNodeId.get(sample.nodeId) || 0) + sample.size)
+      }
+
+      let totalConsumption = 0
+      for (const node of iterNodes(profile.head)) {
+        const consumption = sizeForNodeId.get(node.id) || 0
+        totalConsumption += consumption
+      }
+      measurements.push({ totalConsumption })
+    },
+
+    stopMemoryProfiling: async () => {
+      await client.send('HeapProfiler.stopSampling')
+
+      measurements.sort((a, b) => a.totalConsumption - b.totalConsumption)
+      const { totalConsumption } = measurements[Math.floor(measurements.length / 2)]
+      return totalConsumption
+    },
+  }
+}
+
+function* iterNodes<N extends { children?: N[] }>(root: N): Generator<N> {
+  yield root
+  if (root.children) {
+    for (const child of root.children) {
+      yield* iterNodes(child)
+    }
+  }
+}

benchmark/profilers/startNetworkProfiling.ts

@@ -0,0 +1,38 @@
+import type { CDPSession } from '@playwright/test'
+import type { Protocol } from 'playwright-core/types/protocol'
+
+export async function startNetworkProfiling(client: CDPSession) {
+  await client.send('Network.enable')
+  let downloadBytes = 0
+  let uploadBytes = 0
+
+  const requestListener = ({ request }: Protocol.Network.requestWillBeSentPayload) => {
+    const postData = request.postData
+    if (postData) {
+      if (request.headers['accept-encoding']?.includes('deflate')) {
+        uploadBytes += postData.length
+      } else {
+        uploadBytes += new TextEncoder().encode(postData).length
+      }
+    }
+  }
+
+  const loadingFinishedListener = ({ encodedDataLength }: Protocol.Network.loadingFinishedPayload) => {
+    downloadBytes += encodedDataLength
+  }
+
+  client.on('Network.requestWillBeSent', requestListener)
+  client.on('Network.loadingFinished', loadingFinishedListener)
+
+  return {
+    stopNetworkProfiling: () => {
+      client.off('Network.requestWillBeSent', requestListener)
+      client.off('Network.loadingFinished', loadingFinishedListener)
+
+      return {
+        upload: uploadBytes,
+        download: downloadBytes,
+      }
+    },
+  }
+}

benchmark/profilers/startProfiling.ts

@@ -0,0 +1,29 @@
+import type { Page } from '@playwright/test'
+import type { Metrics } from 'profiling.type'
+import { startCPUProfiling } from './startCpuProfiling'
+import { startMemoryProfiling } from './startMemoryProfiling'
+import { startWebVitalsProfiling } from './startWebVitalsProfiling'
+import { startNetworkProfiling } from './startNetworkProfiling'
+
+export async function startProfiling(page: Page) {
+  const context = page.context()
+  const cdpSession = await context.newCDPSession(page)
+
+  const { stopCPUProfiling, takeCPUMeasurements } = await startCPUProfiling(cdpSession)
+  const { stopMemoryProfiling, takeMemoryMeasurements } = await startMemoryProfiling(cdpSession)
+  const { stopNetworkProfiling } = await startNetworkProfiling(cdpSession)
+  const { stopWebVitalsProfiling } = await startWebVitalsProfiling(page)
+
+  return {
+    takeMeasurements: async () => {
+      await takeCPUMeasurements()
+      await takeMemoryMeasurements()
+    },
+    stopProfiling: async (): Promise<Metrics> => ({
+      memory: await stopMemoryProfiling(),
+      cpu: await stopCPUProfiling(),
+      ...stopNetworkProfiling(),
+      ...(await stopWebVitalsProfiling()),
+    }),
+  }
+}

benchmark/profilers/startWebVitalsProfiling.ts

@@ -0,0 +1,39 @@
+import type { Page } from '@playwright/test'
+import type { BrowserWindow, WebVitalsMetrics } from 'profiling.type'
+interface Metric {
+  value: number
+}
+
+type RecordMetric = (callback: (metric: Metric) => void, options?: { reportAllChanges: boolean }) => void
+
+interface WebVitalsModule {
+  onLCP: RecordMetric
+  onCLS: RecordMetric
+  onFCP: RecordMetric
+  onTTFB: RecordMetric
+  onINP: RecordMetric
+}
+
+export async function startWebVitalsProfiling(page: Page) {
+  await page.addInitScript(() => {
+    import('https://unpkg.com/web-vitals@5?module' as string)
+      .then(({ onLCP, onCLS, onFCP, onTTFB, onINP }: WebVitalsModule) => {
+        const metrics: WebVitalsMetrics = {}
+        ;(window as BrowserWindow).__webVitalsMetrics__ = metrics
+        const recordMetric = (name: keyof WebVitalsMetrics) => (metric: Metric) => (metrics[name] = metric.value)
+        onINP(recordMetric('INP'), { reportAllChanges: true })
+        onLCP(recordMetric('LCP'), { reportAllChanges: true })
+        onCLS(recordMetric('CLS'), { reportAllChanges: true })
+        onFCP(recordMetric('FCP'))
+        onTTFB(recordMetric('TTFB'))
+      })
+      .catch((e) => console.error('web-vitals load failed:', e))
+  })
+
+  return {
+    stopWebVitalsProfiling: async (): Promise<WebVitalsMetrics> => {
+      const metrics = await page.evaluate(() => (window as BrowserWindow).__webVitalsMetrics__ || {})
+      return metrics
+    },
+  }
+}

benchmark/profiling.type.ts

@@ -0,0 +1,27 @@
+import type { Page } from '@playwright/test'
+import type { RumPublicApi } from '@datadog/browser-rum-core'
+
+export interface BrowserWindow extends Window {
+  DD_RUM?: RumPublicApi
+  __webVitalsMetrics__?: WebVitalsMetrics
+}
+
+export interface Scenario {
+  description: string
+  run(this: void, page: Page, takeMeasurements: () => Promise<void>): Promise<void>
+}
+
+export interface Metrics extends WebVitalsMetrics {
+  memory: number
+  cpu: number
+  upload: number
+  download: number
+}
+
+export interface WebVitalsMetrics {
+  LCP?: number
+  CLS?: number
+  FCP?: number
+  TTFB?: number
+  INP?: number
+}