Add step to auto-sync private repo

Author: nhatnghiho

tests/ci/cdk/pipeline/ci_stage.py

@@ -4,7 +4,7 @@
 import re
 import typing
 
-from aws_cdk import Stage, Environment, Duration, pipelines, aws_iam as iam, Stack
+from aws_cdk import Stage, Environment, Duration, Stack, pipelines, aws_iam as iam, aws_codebuild as codebuild
 from constructs import Construct
 
 from cdk.aws_lc_analytics_stack import AwsLcGitHubAnalyticsStack
@@ -13,6 +13,7 @@
 from cdk.aws_lc_github_ci_stack import AwsLcGitHubCIStack
 from cdk.aws_lc_github_fuzz_ci_stack import AwsLcGitHubFuzzCIStack
 from pipeline.codebuild_batch_step import CodeBuildBatchStep
+from util.metadata import PRE_PROD_ACCOUNT
 
 
 class CiStage(Stage):
@@ -173,6 +174,37 @@ def add_stage_to_pipeline(
     ):
         stack_names = [stack.stack_name for stack in self.stacks]
 
+        private_repo_sync_step=None
+
+        if self.stacks[0].account == PRE_PROD_ACCOUNT:
+            private_repo_sync_step = pipelines.CodeBuildStep(
+                "PrivateRepoSync",
+                build_environment=codebuild.BuildEnvironment(
+                    environment_variables={
+                        "GITHUB_PAT":  codebuild.BuildEnvironmentVariable(
+                            type=codebuild.BuildEnvironmentVariableType.SECRETS_MANAGER,
+                            value="aws-lc/ci/github/token",
+                        ),
+                    }
+                ),
+                commands=[
+                    "env",
+                    "curl -H \"Authorization: token ${GITHUB_PAT}\" https://api.github.com/user",
+                    "git clone https://${GITHUB_PAT}@github.com/${STAGING_GITHUB_REPO_OWNER}/${STAGING_GITHUB_REPO_NAME}.git",
+                    "git remote add upstream https://github.com/aws/aws-lc.git",
+                    "git fetch upstream",
+                    "git checkout main",
+                    "git merge upstream/main",
+                    # "git push origin main",
+                ],
+                env={
+                    "STAGING_GITHUB_REPO_OWNER": "aws",
+                    "STAGING_GITHUB_REPO_NAME": "private-aws-lc-staging",
+                },
+                role=role,
+                timeout=Duration.minutes(60),
+            )
+
         env = env or {}
 
         prebuild_check_step = pipelines.CodeBuildStep(
@@ -220,6 +252,7 @@ def add_stage_to_pipeline(
             ],
             role=role,
             timeout=300,
+            project_description=f"Pipeline step AwsLcCiPipeline/{self.stage_name}/StartWait",
             partial_batch_build_spec=batch_build_jobs,
             env={
                 **env,
@@ -230,7 +263,11 @@ def add_stage_to_pipeline(
 
         ci_run_step.add_step_dependency(prebuild_check_step)
 
-        pipeline.add_stage(self, post=[prebuild_check_step, ci_run_step])
+        pipeline.add_stage(
+            self,
+            pre=[private_repo_sync_step] if private_repo_sync_step else None,
+            post=[prebuild_check_step, ci_run_step]
+        )
 
 
 class BatchBuildOptions:

tests/ci/cdk/pipeline/codebuild_batch_step.py

@@ -42,6 +42,7 @@ def __init__(
         partial_batch_build_spec: typing.Mapping[builtins.str, typing.Any],
         role: iam.Role,
         timeout: int = 300,
+        project_description: str = None,
         env: typing.Optional[typing.Mapping[str, str]] = None,
     ):
         super().__init__(id)
@@ -54,6 +55,7 @@ def __init__(
         self.partial_batch_build_spec = partial_batch_build_spec
         self.role = role
         self.timeout = timeout
+        self.project_description = project_description
         self.env = (
             {
                 key: codebuild.BuildEnvironmentVariable(value=value)
@@ -80,6 +82,7 @@ def produce_action(
                 }
             ),
             role=self.role,
+            description=self.project_description,
             timeout=Duration.minutes(self.timeout),
         )
 

tests/ci/cdk/pipeline/pipeline_stack.py

@@ -57,7 +57,11 @@ def __init__(
             iam.PolicyStatement(
                 effect=iam.Effect.ALLOW,
                 resources=["*"],
-                actions=["codepipeline:GetPipelineExecution"],
+                actions=[
+                    "codepipeline:GetPipelineExecution",
+                    "secretsmanager:GetSecretValue",
+                    "kms:Decrypt"
+                ],
             )
         )
 

tests/ci/cdk/pipeline/scripts/build_target.sh

@@ -165,7 +165,7 @@ fi
 
 if [[ ${BUILD_TYPE} == "docker" ]]; then
   if [[ -z "${PLATFORM+x}" || -z "${PLATFORM}" ]]; then
-    echo "When building Docker images, a platform must be specified"
+    echo "When building Docker images, a platform must be specified."
     exit 1
   fi
 
@@ -179,7 +179,7 @@ fi
 
 if [[ ${BUILD_TYPE} == "ci" ]]; then
   if [[ -z "${PROJECT+x}" || -z "${PROJECT}" ]]; then
-    echo "When building CI tests, a project name must be specified"
+    echo "When building CI tests, a project name must be specified."
     exit 1
   fi
 

tests/ci/cdk/pipeline/setup_stage.py

@@ -56,11 +56,11 @@ def __init__(
 
         cross_account_role = iam.Role(
             self,
-            "CrossAccountCodeBuildRole",
-            role_name="CrossAccountCodeBuildRole",
+            "CrossAccountBuildRole",
+            role_name="CrossAccountBuildRole",
             assumed_by=iam.ArnPrincipal(
                 f"arn:aws:iam::{pipeline_environment.account}:role/CrossAccountPipelineRole"
-            ),  # TODO: add a conditional to exclude this in dev env
+            ),
         )
 
         # Grant access to all CodeBuild projects