Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions torchci/clickhouse_queries/commit_jobs_query/params.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
"params": {
"repo": "String",
"sha": "String",
"workflowId": "Int64"
"workflowId": "Int64",
"runAttempt": "Int64"
},
"tests": [
{
"repo": "pytorch/pytorch",
"sha": "85df746892d9b0e87e7a5dfa78ef81a84aec6de0",
"workflow_id": 0
"workflow_id": 0,
"run_attempt": 0
}
]
}
15 changes: 13 additions & 2 deletions torchci/clickhouse_queries/commit_jobs_query/query.sql
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ WITH job AS (
job.torchci_classification_kg.'line_num' as line_num,
job.torchci_classification_kg.'context' as context,
job.runner_name AS runner_name,
workflow.head_commit. 'author'.'email' AS authorEmail
workflow.head_commit. 'author'.'email' AS authorEmail,
job.run_attempt AS run_attempt
FROM
workflow_job job final
INNER JOIN workflow_run workflow final ON workflow.id = job.run_id
Expand All @@ -51,6 +52,10 @@ WITH job AS (
{workflowId: Int64} = 0
OR workflow.id = {workflowId: Int64} -- If a specific workflow ID is provided, filter by it
)
AND (
{runAttempt: Int64} = 0
OR job.run_attempt = {runAttempt: Int64} -- If a specific run attempt
)
AND job.id in (select id from materialized_views.workflow_job_by_head_sha where head_sha = {sha: String})
AND workflow.repository. 'full_name' = {repo: String } -- UNION
AND workflow.name != 'Upload test stats while running' -- Continuously running cron job that cancels itself to avoid running concurrently
Expand Down Expand Up @@ -82,7 +87,8 @@ WITH job AS (
0 as line_num,
[ ] as context,
'' AS runner_name,
workflow.head_commit.author.email AS authorEmail
workflow.head_commit.author.email AS authorEmail,
workflow.run_attempt as run_attempt
FROM
workflow_run workflow final
WHERE
Expand All @@ -94,6 +100,10 @@ WITH job AS (
{workflowId: Int64} = 0
OR workflow.id = {workflowId: Int64} -- If a specific workflow ID is provided, filter by it
)
AND (
{runAttempt: Int64} = 0
OR workflow.run_attempt = {runAttempt: Int64} -- If a specific run attempt is provided, filter by it
)
AND workflow.repository.full_name = {repo: String }
AND workflow.name != 'Upload test stats while running' -- Continuously running cron job that cancels itself to avoid running concurrently
)
Expand All @@ -118,6 +128,7 @@ SELECT
runner_name AS runnerName,
authorEmail,
time,
run_attempt AS runAttempt
FROM
job
ORDER BY
Expand Down
5 changes: 3 additions & 2 deletions torchci/components/commit/CommitStatus.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
import { CommitData, IssueData, JobData } from "lib/types";
import useScrollTo from "lib/useScrollTo";
import _ from "lodash";
import { WorkflowRunInfo } from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
import { useState } from "react";
import { linkIt, UrlComponent, urlRegex } from "react-linkify-it";
import { getConclusionSeverityForSorting } from "../../lib/JobClassifierUtil";
Expand Down Expand Up @@ -59,7 +60,7 @@ function WorkflowsContainer({
}: {
jobs: JobData[];
unstableIssues: IssueData[];
workflowIdsByName: Record<string, number[]>;
workflowIdsByName: Record<string, [WorkflowRunInfo]>;
repoFullName: string;
}) {
useScrollTo();
Expand Down Expand Up @@ -117,7 +118,7 @@ export default function CommitStatus({
repoName: string;
commit: CommitData;
jobs: JobData[];
workflowIdsByName: Record<string, number[]>;
workflowIdsByName: Record<string, [WorkflowRunInfo]>;
isCommitPage: boolean;
unstableIssues: IssueData[];
}) {
Expand Down
47 changes: 34 additions & 13 deletions torchci/components/commit/WorkflowBox.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ import {
ListUtilizationMetadataInfoAPIResponse,
UtilizationMetadataInfo,
} from "lib/utilization/types";
import { CommitApiResponse } from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
import {
CommitApiResponse,
WorkflowRunInfo,
} from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
import React, { useEffect, useState } from "react";
import { FaInfoCircle } from "react-icons/fa";
import useSWR from "swr";
Expand Down Expand Up @@ -154,18 +157,18 @@ export default function WorkflowBox({
jobs: JobData[];
unstableIssues: IssueData[];
wide: boolean;
allWorkflowIds: number[];
allWorkflowIds: [WorkflowRunInfo];
setWide: any;
repoFullName: string;
}) {
const [selectedWorkflowId, setSelectedWorkflowId] = useState<
string | undefined
WorkflowRunInfo | undefined
>(undefined);
const workflowId = selectedWorkflowId || jobs[0].workflowId;
const workflowId = selectedWorkflowId?.id || jobs[0].workflowId;

const { data: jobsFromSelectedWorkflowId } = useSWR<CommitApiResponse>(
selectedWorkflowId &&
`/api/${repoFullName}/commit/${jobs[0].sha}?workflowId=${selectedWorkflowId}`,
`/api/${repoFullName}/commit/${jobs[0].sha}?workflowId=${selectedWorkflowId.id}&runAttempt=${selectedWorkflowId.attempt}`,
fetcher
);

Expand All @@ -180,7 +183,7 @@ export default function WorkflowBox({

const anchorName = encodeURIComponent(workflowName.toLowerCase());

const { utilMetadataList } = useUtilMetadata(workflowId);
const { utilMetadataList } = useUtilMetadata(workflowId?.toString());
const groupUtilMetadataList = groupMetadataByJobId(utilMetadataList);

const { artifacts, error } = useArtifacts(jobs.map((job) => job.workflowId));
Expand Down Expand Up @@ -219,20 +222,30 @@ export default function WorkflowBox({
<Typography fontWeight="bold" paddingBottom={2}>
Job Status
</Typography>
</Stack>
</Stack>{" "}
<Stack direction="column" spacing={1}>
<Stack direction="row" spacing={1}>
<select
value={selectedWorkflowId}
value={
selectedWorkflowId
? `${selectedWorkflowId?.id} ${selectedWorkflowId?.attempt}`
: ""
}
onChange={(e) => {
setSelectedWorkflowId(e.target.value);
const split = e.target.value.split(" ");
setSelectedWorkflowId({
id: parseInt(split[0]),
attempt: parseInt(split[1]),
});
}}
style={{ width: "100%" }}
>
<option value={""}>Select Workflow ID</option>
{allWorkflowIds.sort().map((id) => (
<option key={id} value={id}>
{id}
<option
key={`${id.id} ${id.attempt}`}
value={`${id.id} ${id.attempt}`}
>
{id.id} (Attempt {id.attempt})
</option>
))}
</select>
Expand Down Expand Up @@ -275,7 +288,15 @@ export default function WorkflowBox({
</Stack>
</Stack>
{wide && (
<TestInfo workflowId={workflowId!} runAttempt={"1"} jobs={jobs} />
<TestInfo
workflowId={workflowId!.toString()}
runAttempt={
selectedWorkflowId?.attempt?.toString() ||
jobs[0].runAttempt?.toString() ||
"1"
}
jobs={jobs}
/>
)}
<>
{jobs.sort(sortJobsByConclusion).map((job) => (
Expand Down
39 changes: 29 additions & 10 deletions torchci/lib/fetchCommit.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import _ from "lodash";
import { Octokit } from "octokit";
import { CommitApiResponse } from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
import {
CommitApiResponse,
WorkflowRunInfo,
} from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
import { queryClickhouseSaved } from "./clickhouse";
import { commitDataFromResponse, getOctokit } from "./github";
import { removeCancelledJobAfterRetry } from "./jobUtils";
Expand All @@ -10,12 +13,14 @@ async function fetchDatabaseInfo(
owner: string,
repo: string,
sha: string,
workflowId: number
workflowId: number,
runAttempt: number
) {
const response = await queryClickhouseSaved("commit_jobs_query", {
repo: `${owner}/${repo}`,
sha: sha,
workflowId,
runAttempt,
});

for (const row of response) {
Expand All @@ -32,16 +37,29 @@ async function fetchDatabaseInfo(
* @param jobs
* @returns
*/
function getWorkflowIdsByName(jobs: JobData[]): Record<string, number[]> {
function getWorkflowIdsByName(
jobs: JobData[]
): Record<string, [WorkflowRunInfo]> {
return _(jobs)
.groupBy((job) => job.workflowName)
.map((jobs, key) => {
const workflowIds = _(jobs)
.map((job) => job.workflowId)
.filter((id) => id !== null && id !== undefined)
.uniq()
const idAndAttempts = _(jobs)
.map((job) => {
return {
id: job.workflowId,
attempt: job.runAttempt,
};
})
.filter(
(id) =>
id.id !== null &&
id.id !== undefined &&
id.attempt !== null &&
id.attempt !== undefined
)
.uniqBy((id) => `${id.id}-${id.attempt}`)
.value();
return [key, workflowIds];
return [key, idAndAttempts];
})
.fromPairs()
.value();
Expand All @@ -60,14 +78,15 @@ export default async function fetchCommit(
owner: string,
repo: string,
sha: string,
workflowId: number = 0
workflowId: number = 0,
runAttempt: number = 0
): Promise<CommitApiResponse> {
// Retrieve commit data from GitHub
const octokit = await getOctokit(owner, repo);

const [githubResponse, response] = await Promise.all([
octokit.rest.repos.getCommit({ owner, repo, ref: sha }),
await fetchDatabaseInfo(owner, repo, sha, workflowId),
await fetchDatabaseInfo(owner, repo, sha, workflowId, runAttempt),
]);

let jobs = response as any[];
Expand Down
1 change: 1 addition & 0 deletions torchci/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export interface JobData extends BasicJobData {
repo?: string;
failureAnnotation?: string;
failedPreviousRun?: boolean;
runAttempt?: number;
}

// Used by Dr.CI
Expand Down
11 changes: 9 additions & 2 deletions torchci/pages/api/[repoOwner]/[repoName]/commit/[sha].ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,32 @@ import fetchCommit from "lib/fetchCommit";
import { CommitData, JobData } from "lib/types";
import type { NextApiRequest, NextApiResponse } from "next";

export type WorkflowRunInfo = {
id: number;
attempt: number;
};

export type CommitApiResponse = {
commit: CommitData;
jobs: JobData[];
workflowIdsByName: Record<string, number[]>;
workflowIdsByName: Record<string, [WorkflowRunInfo]>;
};

export default async function handler(
req: NextApiRequest,
res: NextApiResponse<CommitApiResponse>
) {
const workflowId = parseInt(req.query.workflowId as string, 10) || 0;
const runAttempt = parseInt(req.query.runAttempt as string, 10) || 0;
res
.status(200)
.json(
await fetchCommit(
req.query.repoOwner as string,
req.query.repoName as string,
req.query.sha as string,
workflowId
workflowId,
runAttempt
)
);
}