[RUM-12600] Continuous Benchmarking

.gitlab-ci.yml

@@ -587,6 +587,21 @@ bump-chrome-version-scheduled-failure:
   dependencies:
     - bump-chrome-version-scheduled
 
+########################################################################################################################
+# Performance benchmark
+########################################################################################################################
+
+performance-benchmark:
+  stage: task
+  extends: .base-configuration
+  only:
+    variables:
+      - $TARGET_TASK_NAME == "performance-benchmark-scheduled"
+  script:
+    - yarn
+    - yarn playwright install
+    - yarn test:performance
+
 ########################################################################################################################
 # Check expired telemetry
 ########################################################################################################################

benchmark/configuration.ts

@@ -0,0 +1,5 @@
+export const TEST_APP_URL = 'https://df1fyr56hap9d.cloudfront.net/'
+export const SDK_BUNDLE_URL = 'https://df1fyr56hap9d.cloudfront.net/datadog-rum.js'
+export const APPLICATION_ID = '9fa62a5b-8a7e-429d-8466-8b111a4d4693'
+export const CLIENT_TOKEN = 'pubab31fd1ab9d01d2b385a8aa3dd403b1d'
+export const DATADOG_SITE = 'datadoghq.com'

benchmark/createBenchmarkTest.ts

@@ -0,0 +1,127 @@
+import { test } from '@playwright/test'
+import type { Page } from '@playwright/test'
+import type { RumInitConfiguration } from '@datadog/browser-rum-core'
+import type { BrowserWindow, Metrics } from './profiling.type'
+import { startProfiling } from './profilers'
+import { reportToConsole } from './reporters/reportToConsole'
+import { reportToDatadog } from './reporters/reportToDatadog'
+import { isContinuousIntegration } from './environment'
+import { CLIENT_TOKEN, APPLICATION_ID, DATADOG_SITE, TEST_APP_URL, SDK_BUNDLE_URL } from './configuration'
+
+
+const SCENARIO_CONFIGURATIONS = ['none', 'rum', 'rum_replay', 'rum_profiling', 'none_with_headers'] as const
+
+type ScenarioConfiguration = (typeof SCENARIO_CONFIGURATIONS)[number]
+type TestRunner = (page: Page, takeMeasurementsAndGoto: (pageUrl: string) => Promise<void>, appUrl: string) => Promise<void> | void
+
+
+export function createBenchmarkTest(scenarioName: string) {
+  return {
+    run(runner: TestRunner) {
+      const metrics: Record<string, Metrics> = {}
+      SCENARIO_CONFIGURATIONS.forEach((scenarioConfiguration) => {
+        test(`${scenarioName} benchmark ${scenarioConfiguration}`, async ({ page }) => {
+          await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' })
+
+          const { stopProfiling, takeMeasurements } = await startProfiling(page)
+
+          if (shouldInjectSDK(scenarioConfiguration)) {
+            await injectSDK(page, scenarioConfiguration, scenarioName)
+          }
+
+          await runner(page, takeMeasurements, buildAppUrl(scenarioConfiguration))
+
+          await flushEvents(page)
+          metrics[scenarioConfiguration] = await stopProfiling()
+        })
+      })
+
+      test.afterAll(async () => {
+        reportToConsole(metrics)
+        if (isContinuousIntegration) {
+          await reportToDatadog(metrics, scenarioName)
+        }
+      })
+    },
+  }
+}
+
+interface PageInitScriptParameters {
+  configuration: Partial<RumInitConfiguration>
+  sdkBundleUrl: string
+  scenarioConfiguration: ScenarioConfiguration
+  scenarioName: string
+}
+
+
+async function injectSDK(page: Page, scenarioConfiguration: ScenarioConfiguration, scenarioName: string) {
+  const configuration: Partial<RumInitConfiguration> = {
+    clientToken: CLIENT_TOKEN,
+    applicationId: APPLICATION_ID,
+    site: DATADOG_SITE,
+    profilingSampleRate: scenarioConfiguration === 'rum_profiling' ? 100 : 0,
+    sessionReplaySampleRate: scenarioConfiguration === 'rum_replay' ? 100 : 0,
+  }
+
+  await page.addInitScript(
+    ({ sdkBundleUrl, scenarioConfiguration, scenarioName, configuration }: PageInitScriptParameters) => {
+      function loadSDK() {
+        ;(function (h: any, o: Document, u: string, n: string, d: string) {
+          h = h[d] = h[d] || {
+            q: [],
+            onReady(c: () => void) {
+              // eslint-disable-next-line
+              h.q.push(c)
+            },
+          }
+          const s = o.createElement(u) as HTMLScriptElement
+          s.async = true
+          s.src = n
+          o.head.appendChild(s)
+        })(window, document, 'script', sdkBundleUrl, 'DD_RUM')
+        ;(window as BrowserWindow).DD_RUM?.onReady(function () {
+          ;(window as BrowserWindow).DD_RUM!.setGlobalContextProperty('scenario', {
+            configuration: scenarioConfiguration,
+            name: scenarioName,
+          })
+          ;(window as BrowserWindow).DD_RUM!.init(configuration as RumInitConfiguration)
+        })
+      }
+
+      // Init scripts run before DOM is ready; wait until "interactive" to append the SDK <script> tag.
+      document.addEventListener('readystatechange', () => {
+        if (document.readyState === 'interactive') {
+          loadSDK()
+        }
+      })
+    },
+    { configuration, sdkBundleUrl: SDK_BUNDLE_URL, scenarioConfiguration, scenarioName }
+  )
+}
+
+function shouldInjectSDK(scenarioConfiguration: ScenarioConfiguration): boolean {
+  return !['none', 'none_with_headers'].includes(scenarioConfiguration)
+}
+
+
+function buildAppUrl(scenarioConfiguration: ScenarioConfiguration): string {
+  const url = new URL(TEST_APP_URL)
+  if (scenarioConfiguration === 'rum_profiling' || scenarioConfiguration === 'none_with_headers') {
+    url.searchParams.set('profiling', 'true')
+  }
+  return url.toString()
+}
+
+/**
+ * Flushes the events of the SDK and Google Web Vitals 
+ * by simulating a `visibilitychange` event with the state set to "hidden".
+ */
+async function flushEvents(page: Page) {
+  await page.evaluate(() => {
+    Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => 'hidden' })
+    const hiddenEvent = new Event('visibilitychange', { bubbles: true })
+    ;(hiddenEvent as unknown as { __ddIsTrusted: boolean }).__ddIsTrusted = true
+    document.dispatchEvent(hiddenEvent)
+  })
+}
+

benchmark/environment.ts

@@ -0,0 +1 @@
+export const isContinuousIntegration = Boolean(process.env.CI)

benchmark/playwright.config.ts

@@ -0,0 +1,29 @@
+import { defineConfig, devices } from '@playwright/test'
+import { isContinuousIntegration } from './environment'
+
+const baseConfig = {
+  testDir: './scenarios',
+  testMatch: '**/*.scenario.ts',
+  tsconfig: './tsconfig.json',
+  fullyParallel: true,
+  retries: 0,
+  workers: 1,
+  timeout: 90000, // 90 seconds to allow time for profile collection
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+}
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  ...baseConfig,
+  ...(isContinuousIntegration && {
+    repeatEach: 15,
+    workers: 4,
+  }),
+})

benchmark/profilers/index.ts

@@ -0,0 +1,4 @@
+export * from './startProfiling'
+export * from './startMemoryProfiling'
+export * from './startCpuProfiling'
+export * from './startWebVitalsProfiling'

benchmark/profilers/startCpuProfiling.ts

@@ -0,0 +1,37 @@
+import type { CDPSession } from '@playwright/test'
+
+export async function startCPUProfiling(client: CDPSession) {
+  await client.send('Profiler.enable')
+  await client.send('Profiler.start')
+
+  let totalConsumption = 0
+
+  async function stopAndAddProfile() {
+    const { profile } = await client.send('Profiler.stop')
+
+    const timeDeltaForNodeId = new Map<number, number>()
+
+    for (let index = 0; index < profile.samples!.length; index += 1) {
+      const nodeId = profile.samples![index]
+      timeDeltaForNodeId.set(nodeId, (timeDeltaForNodeId.get(nodeId) || 0) + profile.timeDeltas![index])
+    }
+
+    for (const node of profile.nodes) {
+      const consumption = timeDeltaForNodeId.get(node.id) || 0
+      totalConsumption += consumption
+    }
+  }
+
+  return {
+    takeCPUMeasurements: async () => {
+      // We need to restart profiling at each "measurement" because the running profile gets reset
+      // on each navigation.
+      await stopAndAddProfile()
+      await client.send('Profiler.start')
+    },
+    stopCPUProfiling: async () => {
+      await stopAndAddProfile()
+      return totalConsumption
+    },
+  }
+}

benchmark/profilers/startMemoryProfiling.ts

@@ -0,0 +1,48 @@
+import type { CDPSession } from '@playwright/test'
+
+export async function startMemoryProfiling(client: CDPSession) {
+  await client.send('HeapProfiler.enable')
+  await client.send('HeapProfiler.startSampling', {
+    // Set a low sampling interval to have more precise measurement
+    samplingInterval: 100,
+  })
+
+  const measurements: Array<{ totalConsumption: number }> = []
+
+  return {
+    takeMemoryMeasurements: async () => {
+      await client.send('HeapProfiler.collectGarbage')
+      const { profile } = await client.send('HeapProfiler.getSamplingProfile')
+
+      const sizeForNodeId = new Map<number, number>()
+
+      for (const sample of profile.samples) {
+        sizeForNodeId.set(sample.nodeId, (sizeForNodeId.get(sample.nodeId) || 0) + sample.size)
+      }
+
+      let totalConsumption = 0
+      for (const node of iterNodes(profile.head)) {
+        const consumption = sizeForNodeId.get(node.id) || 0
+        totalConsumption += consumption
+      }
+      measurements.push({ totalConsumption })
+    },
+
+    stopMemoryProfiling: async () => {
+      await client.send('HeapProfiler.stopSampling')
+
+      measurements.sort((a, b) => a.totalConsumption - b.totalConsumption)
+      const { totalConsumption } = measurements[Math.floor(measurements.length / 2)]
+      return totalConsumption
+    },
+  }
+}
+
+function* iterNodes<N extends { children?: N[] }>(root: N): Generator<N> {
+  yield root
+  if (root.children) {
+    for (const child of root.children) {
+      yield* iterNodes(child)
+    }
+  }
+}

benchmark/profilers/startNetworkProfiling.ts

@@ -0,0 +1,19 @@
+import type { Page } from '@playwright/test'
+
+export  function startNetworkProfiling(page: Page) {
+  let uploadBytes = 0
+  let downloadBytes = 0
+
+  page.on('requestfinished', async (request) => {
+    const sizes = await request.sizes()
+    downloadBytes += sizes.responseBodySize + sizes.responseHeadersSize
+    uploadBytes += sizes.requestBodySize + sizes.requestHeadersSize
+  })
+
+  return {
+    stopNetworkProfiling: () => ({
+        upload: uploadBytes,
+        download: downloadBytes,
+      }),
+  }
+}

benchmark/profilers/startProfiling.ts

@@ -0,0 +1,29 @@
+import type { Page } from '@playwright/test'
+import type { Metrics } from '../profiling.type'
+import { startCPUProfiling } from './startCpuProfiling'
+import { startMemoryProfiling } from './startMemoryProfiling'
+import { startWebVitalsProfiling } from './startWebVitalsProfiling'
+import { startNetworkProfiling } from './startNetworkProfiling'
+
+export async function startProfiling(page: Page) {
+  const context = page.context()
+  const cdpSession = await context.newCDPSession(page)
+
+  const { stopCPUProfiling, takeCPUMeasurements } = await startCPUProfiling(cdpSession)
+  const { stopMemoryProfiling, takeMemoryMeasurements } = await startMemoryProfiling(cdpSession)
+  const { stopNetworkProfiling } = startNetworkProfiling(page)
+  const { stopWebVitalsProfiling } = await startWebVitalsProfiling(page)
+
+  return {
+    takeMeasurements: async () => {
+      await takeCPUMeasurements()
+      await takeMemoryMeasurements()
+    },
+    stopProfiling: async (): Promise<Metrics> => ({
+      memory: await stopMemoryProfiling(),
+      cpu: await stopCPUProfiling(),
+      ...stopNetworkProfiling(),
+      ... await stopWebVitalsProfiling(),
+    }),
+  }
+}

benchmark/profilers/startWebVitalsProfiling.ts

@@ -0,0 +1,61 @@
+import type { Page } from '@playwright/test'
+import type { BrowserWindow, WebVitalsMetrics } from '../profiling.type'
+interface Metric {
+  value: number
+}
+
+type RecordMetric = (callback: (metric: Metric) => void, options?: { reportAllChanges: boolean }) => void
+
+interface WebVitalsModule {
+  onLCP: RecordMetric
+  onCLS: RecordMetric
+  onFCP: RecordMetric
+  onTTFB: RecordMetric
+  onINP: RecordMetric
+}
+
+export async function startWebVitalsProfiling(page: Page) {
+  await page.addInitScript(() => {
+    const metrics: WebVitalsMetrics = {}
+    ;(window as BrowserWindow).__webVitalsMetrics__ = metrics
+    import('https://unpkg.com/web-vitals@5?module' as string)
+      .then(({ onLCP, onCLS, onFCP, onTTFB, onINP }: WebVitalsModule) => {
+        const recordMetric = (name: keyof WebVitalsMetrics) => (metric: Metric) => (metrics[name] = metric.value)
+        onINP(recordMetric('INP'), { reportAllChanges: true })
+        onLCP(recordMetric('LCP'), { reportAllChanges: true })
+        onCLS(recordMetric('CLS'), { reportAllChanges: true })
+        onFCP((metric: Metric) => {
+          recordMetric('FCP')(metric)
+          onTBT(recordMetric('TBT'))
+        })
+        onTTFB(recordMetric('TTFB'))
+      })
+      .catch((e) => console.error('web-vitals load failed:', e))
+
+
+      function onTBT(callback: (metric: Metric) => void) {
+        let tbt = 0
+        const observer = new PerformanceObserver((list) => {
+          for (const entry of list.getEntries()) {
+            // Only count tasks that are longer than 50ms
+            if (entry.startTime >= metrics.FCP! && entry.duration > 50) {
+              // The blocking time is the duration minus 50ms
+              const blockingTime = entry.duration - 50
+              tbt += blockingTime
+              callback({ value: tbt })
+            }
+          }
+        })
+
+        observer.observe({ type: 'longtask', buffered: true })
+      }
+  })
+
+
+  return {
+    stopWebVitalsProfiling: async (): Promise<WebVitalsMetrics> => {
+      const metrics = await page.evaluate(() => (window as BrowserWindow).__webVitalsMetrics__ || {})
+      return metrics
+    },
+  }
+}