From a6fedfe00525865b7c50b7e219f272e288206eb8 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 10 Sep 2025 11:30:04 -0700 Subject: [PATCH] tc --- .../commit_jobs_query/params.json | 6 ++- .../commit_jobs_query/query.sql | 15 +++++- torchci/components/commit/CommitStatus.tsx | 5 +- torchci/components/commit/WorkflowBox.tsx | 47 ++++++++++++++----- torchci/lib/fetchCommit.ts | 39 +++++++++++---- torchci/lib/types.ts | 1 + .../[repoOwner]/[repoName]/commit/[sha].ts | 11 ++++- 7 files changed, 93 insertions(+), 31 deletions(-) diff --git a/torchci/clickhouse_queries/commit_jobs_query/params.json b/torchci/clickhouse_queries/commit_jobs_query/params.json index 36cf80d8ee..efc029390d 100644 --- a/torchci/clickhouse_queries/commit_jobs_query/params.json +++ b/torchci/clickhouse_queries/commit_jobs_query/params.json @@ -2,13 +2,15 @@ "params": { "repo": "String", "sha": "String", - "workflowId": "Int64" + "workflowId": "Int64", + "runAttempt": "Int64" }, "tests": [ { "repo": "pytorch/pytorch", "sha": "85df746892d9b0e87e7a5dfa78ef81a84aec6de0", - "workflow_id": 0 + "workflow_id": 0, + "run_attempt": 0 } ] } diff --git a/torchci/clickhouse_queries/commit_jobs_query/query.sql b/torchci/clickhouse_queries/commit_jobs_query/query.sql index 5be2ea540b..7f59d4d7b9 100644 --- a/torchci/clickhouse_queries/commit_jobs_query/query.sql +++ b/torchci/clickhouse_queries/commit_jobs_query/query.sql @@ -36,7 +36,8 @@ WITH job AS ( job.torchci_classification_kg.'line_num' as line_num, job.torchci_classification_kg.'context' as context, job.runner_name AS runner_name, - workflow.head_commit. 'author'.'email' AS authorEmail + workflow.head_commit. 'author'.'email' AS authorEmail, + job.run_attempt AS run_attempt FROM workflow_job job final INNER JOIN workflow_run workflow final ON workflow.id = job.run_id @@ -51,6 +52,10 @@ WITH job AS ( {workflowId: Int64} = 0 OR workflow.id = {workflowId: Int64} -- If a specific workflow ID is provided, filter by it ) + AND ( + {runAttempt: Int64} = 0 + OR job.run_attempt = {runAttempt: Int64} -- If a specific run attempt + ) AND job.id in (select id from materialized_views.workflow_job_by_head_sha where head_sha = {sha: String}) AND workflow.repository. 'full_name' = {repo: String } -- UNION AND workflow.name != 'Upload test stats while running' -- Continuously running cron job that cancels itself to avoid running concurrently @@ -82,7 +87,8 @@ WITH job AS ( 0 as line_num, [ ] as context, '' AS runner_name, - workflow.head_commit.author.email AS authorEmail + workflow.head_commit.author.email AS authorEmail, + workflow.run_attempt as run_attempt FROM workflow_run workflow final WHERE @@ -94,6 +100,10 @@ WITH job AS ( {workflowId: Int64} = 0 OR workflow.id = {workflowId: Int64} -- If a specific workflow ID is provided, filter by it ) + AND ( + {runAttempt: Int64} = 0 + OR workflow.run_attempt = {runAttempt: Int64} -- If a specific run attempt is provided, filter by it + ) AND workflow.repository.full_name = {repo: String } AND workflow.name != 'Upload test stats while running' -- Continuously running cron job that cancels itself to avoid running concurrently ) @@ -118,6 +128,7 @@ SELECT runner_name AS runnerName, authorEmail, time, + run_attempt AS runAttempt FROM job ORDER BY diff --git a/torchci/components/commit/CommitStatus.tsx b/torchci/components/commit/CommitStatus.tsx index 7487afe4c3..da58550561 100644 --- a/torchci/components/commit/CommitStatus.tsx +++ b/torchci/components/commit/CommitStatus.tsx @@ -7,6 +7,7 @@ import { import { CommitData, IssueData, JobData } from "lib/types"; import useScrollTo from "lib/useScrollTo"; import _ from "lodash"; +import { WorkflowRunInfo } from "pages/api/[repoOwner]/[repoName]/commit/[sha]"; import { useState } from "react"; import { linkIt, UrlComponent, urlRegex } from "react-linkify-it"; import { getConclusionSeverityForSorting } from "../../lib/JobClassifierUtil"; @@ -59,7 +60,7 @@ function WorkflowsContainer({ }: { jobs: JobData[]; unstableIssues: IssueData[]; - workflowIdsByName: Record; + workflowIdsByName: Record; repoFullName: string; }) { useScrollTo(); @@ -117,7 +118,7 @@ export default function CommitStatus({ repoName: string; commit: CommitData; jobs: JobData[]; - workflowIdsByName: Record; + workflowIdsByName: Record; isCommitPage: boolean; unstableIssues: IssueData[]; }) { diff --git a/torchci/components/commit/WorkflowBox.tsx b/torchci/components/commit/WorkflowBox.tsx index 4e81b185d1..198ae5282f 100644 --- a/torchci/components/commit/WorkflowBox.tsx +++ b/torchci/components/commit/WorkflowBox.tsx @@ -14,7 +14,10 @@ import { ListUtilizationMetadataInfoAPIResponse, UtilizationMetadataInfo, } from "lib/utilization/types"; -import { CommitApiResponse } from "pages/api/[repoOwner]/[repoName]/commit/[sha]"; +import { + CommitApiResponse, + WorkflowRunInfo, +} from "pages/api/[repoOwner]/[repoName]/commit/[sha]"; import React, { useEffect, useState } from "react"; import { FaInfoCircle } from "react-icons/fa"; import useSWR from "swr"; @@ -154,18 +157,18 @@ export default function WorkflowBox({ jobs: JobData[]; unstableIssues: IssueData[]; wide: boolean; - allWorkflowIds: number[]; + allWorkflowIds: [WorkflowRunInfo]; setWide: any; repoFullName: string; }) { const [selectedWorkflowId, setSelectedWorkflowId] = useState< - string | undefined + WorkflowRunInfo | undefined >(undefined); - const workflowId = selectedWorkflowId || jobs[0].workflowId; + const workflowId = selectedWorkflowId?.id || jobs[0].workflowId; const { data: jobsFromSelectedWorkflowId } = useSWR( selectedWorkflowId && - `/api/${repoFullName}/commit/${jobs[0].sha}?workflowId=${selectedWorkflowId}`, + `/api/${repoFullName}/commit/${jobs[0].sha}?workflowId=${selectedWorkflowId.id}&runAttempt=${selectedWorkflowId.attempt}`, fetcher ); @@ -180,7 +183,7 @@ export default function WorkflowBox({ const anchorName = encodeURIComponent(workflowName.toLowerCase()); - const { utilMetadataList } = useUtilMetadata(workflowId); + const { utilMetadataList } = useUtilMetadata(workflowId?.toString()); const groupUtilMetadataList = groupMetadataByJobId(utilMetadataList); const { artifacts, error } = useArtifacts(jobs.map((job) => job.workflowId)); @@ -219,20 +222,30 @@ export default function WorkflowBox({ Job Status - + {" "} @@ -275,7 +288,15 @@ export default function WorkflowBox({ {wide && ( - + )} <> {jobs.sort(sortJobsByConclusion).map((job) => ( diff --git a/torchci/lib/fetchCommit.ts b/torchci/lib/fetchCommit.ts index 3434218468..6bb09b4168 100644 --- a/torchci/lib/fetchCommit.ts +++ b/torchci/lib/fetchCommit.ts @@ -1,6 +1,9 @@ import _ from "lodash"; import { Octokit } from "octokit"; -import { CommitApiResponse } from "pages/api/[repoOwner]/[repoName]/commit/[sha]"; +import { + CommitApiResponse, + WorkflowRunInfo, +} from "pages/api/[repoOwner]/[repoName]/commit/[sha]"; import { queryClickhouseSaved } from "./clickhouse"; import { commitDataFromResponse, getOctokit } from "./github"; import { removeCancelledJobAfterRetry } from "./jobUtils"; @@ -10,12 +13,14 @@ async function fetchDatabaseInfo( owner: string, repo: string, sha: string, - workflowId: number + workflowId: number, + runAttempt: number ) { const response = await queryClickhouseSaved("commit_jobs_query", { repo: `${owner}/${repo}`, sha: sha, workflowId, + runAttempt, }); for (const row of response) { @@ -32,16 +37,29 @@ async function fetchDatabaseInfo( * @param jobs * @returns */ -function getWorkflowIdsByName(jobs: JobData[]): Record { +function getWorkflowIdsByName( + jobs: JobData[] +): Record { return _(jobs) .groupBy((job) => job.workflowName) .map((jobs, key) => { - const workflowIds = _(jobs) - .map((job) => job.workflowId) - .filter((id) => id !== null && id !== undefined) - .uniq() + const idAndAttempts = _(jobs) + .map((job) => { + return { + id: job.workflowId, + attempt: job.runAttempt, + }; + }) + .filter( + (id) => + id.id !== null && + id.id !== undefined && + id.attempt !== null && + id.attempt !== undefined + ) + .uniqBy((id) => `${id.id}-${id.attempt}`) .value(); - return [key, workflowIds]; + return [key, idAndAttempts]; }) .fromPairs() .value(); @@ -60,14 +78,15 @@ export default async function fetchCommit( owner: string, repo: string, sha: string, - workflowId: number = 0 + workflowId: number = 0, + runAttempt: number = 0 ): Promise { // Retrieve commit data from GitHub const octokit = await getOctokit(owner, repo); const [githubResponse, response] = await Promise.all([ octokit.rest.repos.getCommit({ owner, repo, ref: sha }), - await fetchDatabaseInfo(owner, repo, sha, workflowId), + await fetchDatabaseInfo(owner, repo, sha, workflowId, runAttempt), ]); let jobs = response as any[]; diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts index 80a97acba6..2cfeb29ef0 100644 --- a/torchci/lib/types.ts +++ b/torchci/lib/types.ts @@ -30,6 +30,7 @@ export interface JobData extends BasicJobData { repo?: string; failureAnnotation?: string; failedPreviousRun?: boolean; + runAttempt?: number; } // Used by Dr.CI diff --git a/torchci/pages/api/[repoOwner]/[repoName]/commit/[sha].ts b/torchci/pages/api/[repoOwner]/[repoName]/commit/[sha].ts index 6d3acaa014..b56be00074 100644 --- a/torchci/pages/api/[repoOwner]/[repoName]/commit/[sha].ts +++ b/torchci/pages/api/[repoOwner]/[repoName]/commit/[sha].ts @@ -2,10 +2,15 @@ import fetchCommit from "lib/fetchCommit"; import { CommitData, JobData } from "lib/types"; import type { NextApiRequest, NextApiResponse } from "next"; +export type WorkflowRunInfo = { + id: number; + attempt: number; +}; + export type CommitApiResponse = { commit: CommitData; jobs: JobData[]; - workflowIdsByName: Record; + workflowIdsByName: Record; }; export default async function handler( @@ -13,6 +18,7 @@ export default async function handler( res: NextApiResponse ) { const workflowId = parseInt(req.query.workflowId as string, 10) || 0; + const runAttempt = parseInt(req.query.runAttempt as string, 10) || 0; res .status(200) .json( @@ -20,7 +26,8 @@ export default async function handler( req.query.repoOwner as string, req.query.repoName as string, req.query.sha as string, - workflowId + workflowId, + runAttempt ) ); }