Post performance results as PR comment

.github/workflows/metrics-comparison.yml

@@ -17,6 +17,7 @@ jobs:
     runs-on: macos-15
     permissions:
       contents: read
+      pull-requests: write
     env:
       ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -36,7 +37,13 @@ jobs:
 
       - name: Compare performance with trunk
         if: github.event_name == 'pull_request'
-        run: cd scripts/compare-perf && npm ci && npm run compare -- perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA
+        run: cd scripts/compare-perf && npm ci && npm run compare -- perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA --rounds 3
+
+      - name: Post performance results to PR
+        if: github.event_name == 'pull_request' && !cancelled()
+        run: |
+          cd scripts/compare-perf
+          npm run post-to-github -- $GITHUB_TOKEN ${{ github.repository }} ${{ github.event.pull_request.number }} trunk $GITHUB_SHA
 
       - name: Compare performance with base branch
         if: github.event_name == 'push'
@@ -45,7 +52,7 @@ jobs:
         # it needs to be updated every time it becomes unsupported by the current performance tests.
         # It is used as a base comparison point to avoid fluctuation in the performance metrics.
         run: |
-          cd scripts/compare-perf && npm ci && npm run compare -- perf $GITHUB_SHA d1f49275f3e08fb675d5685855c2243b6cd183de --tests-branch $GITHUB_SHA
+          cd scripts/compare-perf && npm ci && npm run compare -- perf $GITHUB_SHA d1f49275f3e08fb675d5685855c2243b6cd183de --tests-branch $GITHUB_SHA --rounds 3
 
       # Log performance metrics to CodeVitals when running on trunk
       - name: Log performance metrics to CodeVitals

scripts/compare-perf/package.json

@@ -8,13 +8,18 @@
 	"repository": "Automattic/studio",
 	"scripts": {
 		"compare": "ts-node index.ts",
-		"log-to-codevitals": "ts-node log-to-codevitals.ts"
+		"log-to-codevitals": "ts-node log-to-codevitals.ts",
+		"post-to-github": "ts-node post-to-github.ts"
 	},
 	"dependencies": {
 		"chalk": "^4.1.2",
 		"commander": "^13.1.0",
 		"inquirer": "^12.5.0",
 		"simple-git": "^3.27.0",
 		"ts-node": "^10.9.2"
+	},
+	"devDependencies": {
+		"@types/node": "^20.0.0",
+		"typescript": "^5.0.0"
 	}
 }

scripts/compare-perf/post-to-github.ts

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+/* eslint-disable no-console */
+import fs from 'fs';
+import https from 'https';
+import path from 'path';
+
+interface PerformanceResults {
+	[ branch: string ]: {
+		[ metric: string ]: number;
+	};
+}
+
+interface GitHubContext {
+	repo: string;
+	owner: string;
+	prNumber: number;
+	token: string;
+}
+
+function formatResultsAsMarkdown(
+	results: Record< string, PerformanceResults >,
+	baseBranch: string,
+	compareBranch: string
+): string {
+	let markdown = `## 📊 Performance Test Results\n\n`;
+	markdown += `Comparing **${ compareBranch }** vs **${ baseBranch }**\n\n`;
+
+	for ( const [ testSuite, suiteResults ] of Object.entries( results ) ) {
+		markdown += `### ${ testSuite }\n\n`;
+		markdown += `| Metric | ${ baseBranch } | ${ compareBranch } | Diff | Change |\n`;
+		markdown += `|--------|${ '-'.repeat( baseBranch.length + 2 ) }|${ '-'.repeat(
+			compareBranch.length + 2
+		) }|------|--------|\n`;
+
+		const baseMetrics = suiteResults[ baseBranch ] || {};
+		const compareMetrics = suiteResults[ compareBranch ] || {};
+
+		for ( const metric of Object.keys( { ...baseMetrics, ...compareMetrics } ) ) {
+			const baseValue = baseMetrics[ metric ] || 0;
+			const compareValue = compareMetrics[ metric ] || 0;
+			const diff = compareValue - baseValue;
+			const percentChange = baseValue !== 0 ? ( ( diff / baseValue ) * 100 ).toFixed( 1 ) : 'N/A';
+
+			const emoji = diff > 0 ? '🔴' : diff < 0 ? '🟢' : '⚪';
+			const sign = diff > 0 ? '+' : '';
+
+			markdown += `| ${ metric } | ${ baseValue.toFixed( 2 ) } ms | ${ compareValue.toFixed(
+				2
+			) } ms | ${ sign }${ diff.toFixed( 2 ) } ms | ${ emoji } ${ percentChange }% |\n`;
+		}
+
+		markdown += `\n`;
+	}
+
+	markdown += `\n---\n`;
+	markdown += `*Results are median values from multiple test runs.*\n\n`;
+	markdown += `*Legend: 🟢 Improvement (faster) | 🔴 Regression (slower) | ⚪ No change*\n`;
+
+	return markdown;
+}
+
+async function findOrCreateComment(
+	context: GitHubContext,
+	commentIdentifier: string
+): Promise< number | null > {
+	return new Promise( ( resolve, reject ) => {
+		const options = {
+			hostname: 'api.github.com',
+			port: 443,
+			path: `/repos/${ context.owner }/${ context.repo }/issues/${ context.prNumber }/comments`,
+			method: 'GET',
+			headers: {
+				Accept: 'application/vnd.github.v3+json',
+				Authorization: `token ${ context.token }`,
+				'User-Agent': 'studio-performance-bot',
+			},
+		};
+
+		const req = https.request( options, ( res ) => {
+			let data = '';
+			res.on( 'data', ( chunk ) => ( data += chunk ) );
+			res.on( 'end', () => {
+				if ( res.statusCode === 200 ) {
+					const comments = JSON.parse( data );
+					const existingComment = comments.find( ( c: { body?: string } ) =>
+						c.body?.includes( commentIdentifier )
+					);
+					resolve( existingComment?.id || null );
+				} else {
+					reject( new Error( `Failed to fetch comments: ${ res.statusCode }` ) );
+				}
+			} );
+		} );
+
+		req.on( 'error', reject );
+		req.end();
+	} );
+}
+
+async function postOrUpdateComment( context: GitHubContext, body: string ): Promise< void > {
+	const commentIdentifier = '<!-- performance-test-results -->';
+	const bodyWithIdentifier = `${ commentIdentifier }\n${ body }`;
+
+	const existingCommentId = await findOrCreateComment( context, commentIdentifier );
+
+	const method = existingCommentId ? 'PATCH' : 'POST';
+	const apiPath = existingCommentId
+		? `/repos/${ context.owner }/${ context.repo }/issues/comments/${ existingCommentId }`
+		: `/repos/${ context.owner }/${ context.repo }/issues/${ context.prNumber }/comments`;
+
+	return new Promise( ( resolve, reject ) => {
+		const postData = JSON.stringify( { body: bodyWithIdentifier } );
+
+		const options = {
+			hostname: 'api.github.com',
+			port: 443,
+			path: apiPath,
+			method,
+			headers: {
+				Accept: 'application/vnd.github.v3+json',
+				Authorization: `token ${ context.token }`,
+				'User-Agent': 'studio-performance-bot',
+				'Content-Type': 'application/json',
+				'Content-Length': Buffer.byteLength( postData ),
+			},
+		};
+
+		const req = https.request( options, ( res ) => {
+			let data = '';
+			res.on( 'data', ( chunk ) => ( data += chunk ) );
+			res.on( 'end', () => {
+				if ( res.statusCode === 200 || res.statusCode === 201 ) {
+					console.log(
+						`✅ Successfully ${ existingCommentId ? 'updated' : 'posted' } PR comment`
+					);
+					resolve();
+				} else {
+					reject( new Error( `Failed to post comment: ${ res.statusCode } ${ data }` ) );
+				}
+			} );
+		} );
+
+		req.on( 'error', reject );
+		req.write( postData );
+		req.end();
+	} );
+}
+
+async function main() {
+	const [ token, repo, prNumber, baseBranch, compareBranch, artifactsPath ] =
+		process.argv.slice( 2 );
+
+	if ( ! token || ! repo || ! prNumber || ! baseBranch || ! compareBranch ) {
+		console.error(
+			'Usage: post-to-github <token> <repo> <pr-number> <base-branch> <compare-branch> [artifacts-path]'
+		);
+		process.exit( 1 );
+	}
+
+	const [ owner, repoName ] = repo.split( '/' );
+	const context: GitHubContext = {
+		repo: repoName,
+		owner,
+		prNumber: parseInt( prNumber, 10 ),
+		token,
+	};
+
+	const resultsPath = artifactsPath || process.env.ARTIFACTS_PATH || 'artifacts';
+	const summaryFiles = fs.readdirSync( resultsPath ).filter( ( f ) => f.endsWith( '.summary.json' ) );
+
+	if ( summaryFiles.length === 0 ) {
+		console.error( `No summary files found in ${ resultsPath }` );
+		process.exit( 1 );
+	}
+
+	const results: Record< string, PerformanceResults > = {};
+	for ( const file of summaryFiles ) {
+		const testSuite = file.replace( '.summary.json', '' );
+		const filePath = path.join( resultsPath, file );
+		results[ testSuite ] = JSON.parse( fs.readFileSync( filePath, 'utf8' ) );
+	}
+
+	const markdown = formatResultsAsMarkdown( results, baseBranch, compareBranch );
+	await postOrUpdateComment( context, markdown );
+}
+
+main().catch( ( error ) => {
+	console.error( 'Error posting to GitHub:', error );
+	process.exit( 1 );
+} );
+

scripts/compare-perf/tsconfig.json

@@ -0,0 +1,17 @@
+{
+	"compilerOptions": {
+		"target": "ES2017",
+		"lib": ["ES2017"],
+		"module": "commonjs",
+		"moduleResolution": "node",
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"strict": true,
+		"resolveJsonModule": true,
+		"types": ["node"]
+	},
+	"include": ["*.ts"],
+	"exclude": ["node_modules"]
+}
+
+