Add typing and cleanup

Author: nhatnghiho

tests/ci/cdk/README.md

@@ -112,25 +112,6 @@ To deploy production pipeline using default parameters:
 ./run-cdk.sh --action deploy-production-pipeline
 ```
 
-<!-- Bootstrap pipeline account
-```
-AWS_ACCOUNT_ID=183295444613
-PIPELINE_ACCOUNT_ID=774305600158
-cdk bootstrap aws://${PIPELINE_ACCOUNT_ID}/us-west-2
-```
-
-Give pipeline account administrator access to deployment account's CloudFormation
-```
-cdk bootstrap aws://${AWS_ACCOUNT_ID}/us-west-2 --trust ${PIPELINE_ACCOUNT_ID} --trust-for-lookup ${PIPELINE_ACCOUNT_ID} --cloudformation-execution-policies arn:aws:iam::aws:policy/AdministratorAccess
-```
-
-Deploy pipeline
-```
-GITHUB_REPO_OWNER=nhatnghiho
-GITHUB_SOURCE_VERSION=ci-pipeline
-./run-cdk.sh --github-repo-owner ${GITHUB_REPO_OWNER} --github-source-version ${GITHUB_SOURCE_VERSION} --aws-account ${AWS_ACCOUNT_ID} --action invoke --command "cdk deploy AwsLcCiPipeline --require-approval never"
-``` -->
-
 ### CI Commands
 Use these commands if you wish to deploy individual stacks instead of the entire pipeline.
 

tests/ci/cdk/cdk/aws_lc_analytics_stack.py

@@ -20,7 +20,7 @@ def __init__(self,
                  scope: Construct,
                  id: str,
                  spec_file_path: str,
-                 env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+                 env: typing.Union[Environment, typing.Dict[str, typing.Any]],
                  **kwargs) -> None:
         super().__init__(scope, id, env=env, **kwargs)
 

tests/ci/cdk/cdk/aws_lc_android_ci_stack.py

@@ -22,7 +22,7 @@ def __init__(self,
                  scope: Construct,
                  id: str,
                  spec_file_path: str,
-                 env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+                 env: typing.Union[Environment, typing.Dict[str, typing.Any]],
                  **kwargs) -> None:
         super().__init__(scope, id, env=env, **kwargs)
 

tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py

@@ -27,7 +27,7 @@ def __init__(self,
                  scope: Construct,
                  id: str,
                  spec_file_path: str,
-                 env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+                 env: typing.Union[Environment, typing.Dict[str, typing.Any]],
                  **kwargs) -> None:
         super().__init__(scope, id, env=env, **kwargs)
 

tests/ci/cdk/cdk/aws_lc_github_ci_stack.py

@@ -8,7 +8,7 @@
 
 from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_batch_policy_in_json, code_build_publish_metrics_in_json, \
-    code_build_cloudwatch_logs_policy_in_json, s3_read_policy_in_json
+    code_build_cloudwatch_logs_policy_in_json
 from util.metadata import GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, \
     PIPELINE_ACCOUNT, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
@@ -21,7 +21,7 @@ def __init__(self,
                  scope: Construct,
                  id: str,
                  spec_file_path: str,
-                 env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+                 env: typing.Union[Environment, typing.Dict[str, typing.Any]],
                  **kwargs) -> None:
         super().__init__(scope, id, env=env, **kwargs)
 
@@ -52,7 +52,6 @@ def __init__(self,
         code_build_cloudwatch_logs_policy = iam.PolicyDocument.from_json(
             code_build_cloudwatch_logs_policy_in_json([log_group])
         )
-        s3_assets_policy = iam.PolicyDocument.from_json(s3_read_policy_in_json())
         resource_access_role = iam.Role(scope=self,
                                         id="{}-resource-role".format(id),
                                         assumed_by=iam.CompositePrincipal(
@@ -61,7 +60,6 @@ def __init__(self,
                                         ),
                                         inline_policies={
                                             "code_build_cloudwatch_logs_policy": code_build_cloudwatch_logs_policy,
-                                            "s3_assets_policy": s3_assets_policy
                                         })
 
         # Define a IAM role for this stack.

tests/ci/cdk/cdk/aws_lc_github_ci_x509_stack.py

@@ -9,13 +9,12 @@
     GITHUB_REPO_OWNER, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME,
 )
 
-
 class AwsLcGitHubX509CIStack(Stack):
     def __init__(
         self,
         scope: Construct,
         id: str,
-        env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+        env: typing.Union[Environment, typing.Dict[str, typing.Any]],
         **kwargs,
     ) -> None:
         super().__init__(scope, id, env=env, **kwargs)

tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py

@@ -22,7 +22,7 @@ def __init__(self,
                  scope: Construct,
                  id: str,
                  spec_file_path: str,
-                 env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+                 env: typing.Union[Environment, typing.Dict[str, typing.Any]],
                  **kwargs) -> None:
         super().__init__(scope, id, env=env, **kwargs)
 

tests/ci/cdk/cdk/components.py

@@ -1,7 +1,9 @@
 import pathlib
+import typing
 
-from aws_cdk import aws_codebuild as codebuild, aws_lambda as lambda_, aws_ecr_assets as ecr_assets, aws_secretsmanager as sm, \
-    aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration
+from aws_cdk import aws_codebuild as codebuild, aws_lambda as lambda_, aws_ecr_assets as ecr_assets, \
+    aws_secretsmanager as sm, \
+    aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration, Environment
 
 from constructs import Construct
 from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
@@ -14,7 +16,7 @@ def __init__(
             id: str,
             *,
             project: codebuild.IProject,
-            env,
+            env: typing.Union[Environment, typing.Dict[str, typing.Any]],
             ec2_permissions: bool
     ) -> None:
         super().__init__(scope, id)

tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py

@@ -26,7 +26,7 @@ def __init__(
             self,
             scope: Construct,
             id: str,
-            env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+            env: typing.Union[Environment, typing.Dict[str, typing.Any]],
             **kwargs) -> None:
         super().__init__(scope, id, env=env, **kwargs)
 

tests/ci/cdk/cdk/windows_docker_image_build_stack.py

@@ -34,7 +34,7 @@ def __init__(
             self,
             scope: Construct,
             id: str,
-            env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
+            env: typing.Union[Environment, typing.Dict[str, typing.Any]],
             **kwargs) -> None:
         super().__init__(
             scope,

tests/ci/cdk/pipeline/ci_stage.py

@@ -1,5 +1,8 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
+import builtins
+import re
+import typing
 
 from aws_cdk import Stage, Environment, Duration, pipelines, aws_iam as iam, Stack
 from constructs import Construct
@@ -9,16 +12,15 @@
 from cdk.aws_lc_ec2_test_framework_ci_stack import AwsLcEC2TestingCIStack
 from cdk.aws_lc_github_ci_stack import AwsLcGitHubCIStack
 from cdk.aws_lc_github_fuzz_ci_stack import AwsLcGitHubFuzzCIStack
-from pipeline.codebuild_batch_step import BatchBuildTargetOptions, CodeBuildBatchStep
-
+from pipeline.codebuild_batch_step import CodeBuildBatchStep
 
 class CiStage(Stage):
     def __init__(
             self,
             scope: Construct,
-            id,
-            pipeline_environment,
-            deploy_environment,
+            id: str,
+            pipeline_environment: typing.Union[Environment, typing.Dict[str, typing.Any]],
+            deploy_environment: typing.Union[Environment, typing.Dict[str, typing.Any]],
             **kwargs
     ):
         super().__init__(
@@ -28,7 +30,7 @@ def __init__(
             **kwargs,
         )
 
-        self.build_targets = []
+        self.build_options = []
 
         # Define CodeBuild Batch job for testing code.
         x86_build_spec_file = "cdk/codebuild/github_ci_linux_x86_omnibus.yaml"
@@ -39,8 +41,8 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-linux-x86",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-linux-x86",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-linux-x86",
             ignore_failure=False,
         ))
 
@@ -53,8 +55,8 @@ def __init__(
             env=deploy_environment,
             stack_name=arm_stack_name,
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-linux-arm",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-linux-arm",
             ignore_failure=False,
         ))
 
@@ -66,8 +68,8 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-integration",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-integration",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-integration",
             ignore_failure=True,
         ))
 
@@ -79,8 +81,8 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-fuzzing",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-fuzzing",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-fuzzing",
             ignore_failure=False,
         ))
 
@@ -92,8 +94,8 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-analytics",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-analytics",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-analytics",
             ignore_failure=True,
         ))
 
@@ -109,8 +111,8 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-ec2-test-framework",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-ec2-test-framework",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-ec2-test-framework",
             ignore_failure=True,
         ))
 
@@ -122,8 +124,8 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-devicefarm-android",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-devicefarm-android",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-devicefarm-android",
             ignore_failure=False,
         ))
 
@@ -135,25 +137,27 @@ def __init__(
             env=deploy_environment,
             stack_name="aws-lc-ci-windows-x86",
         )
-        self.build_targets.append(BatchBuildTargetOptions(
-            target="aws-lc-ci-windows-x86",
+        self.build_options.append(BatchBuildOptions(
+            project="aws-lc-ci-windows-x86",
             ignore_failure=False,
         ))
 
     @property
-    def stacks(self):
+    def stacks(self) -> typing.List[Stack]:
         return [child for child in self.node.children if isinstance(child, Stack)]
 
     def add_stage_to_pipeline(
             self,
             pipeline: pipelines.CodePipeline,
             input: pipelines.FileSet,
             role: iam.Role,
-            max_retry: int=2,
-            env={},
+            max_retry: typing.Optional[int] = 2,
+            env: typing.Optional[typing.Mapping[str, str]] = None,
     ):
         stack_names = [stack.stack_name for stack in self.stacks]
 
+        env = env or {}
+
         prebuild_check_step = pipelines.CodeBuildStep(
             "PrebuildCheck",
             input=input,
@@ -168,8 +172,7 @@ def add_stage_to_pipeline(
                 "STACKS": " ".join(stack_names),
             },
             role=role,
-            timeout=Duration.minutes(180)
-            # project_name=f"{self.stage_name}-PrebuildCheck"
+            timeout=Duration.minutes(60)
         )
 
         batch_build_jobs = {
@@ -179,13 +182,13 @@ def add_stage_to_pipeline(
                     "ignore-failure": options.ignore_failure,
                     "env": {
                         "variables": {
-                            "PROJECT": options.target,
-                            "TIMEOUT": options.timeout,
+                            "PROJECT": options.project,
+                            "TIMEOUT": str(max_retry * options.timeout),
                             **options.env,
                         }
                     }
                 }
-                for options in self.build_targets
+                for options in self.build_options
             ]
         }
 
@@ -199,32 +202,17 @@ def add_stage_to_pipeline(
                 "./build_target.sh --build-type ci --project ${PROJECT} --max-retry ${MAX_RETRY} --timeout ${TIMEOUT}"
             ],
             role=role,
+            timeout=300,
             partial_batch_buildspec=batch_build_jobs,
             env={
                 **env,
                 "MAX_RETRY": max_retry,
-                "NEED_REBUILD": prebuild_check_step.exported_variable("NEED_REBUILD")
+                "NEED_REBUILD": prebuild_check_step.exported_variable("NEED_REBUILD"),
             },
         )
 
         ci_run_step.add_step_dependency(prebuild_check_step)
 
-        # pipeline.add_stage(
-        #     self,
-        #     post=[
-        #         CodeBuildRunStep(
-        #             f"{self.stage_name}-BuildStep",
-        #             name_prefix=self.stage_name,
-        #             input=input,
-        #             role=role,
-        #             stacks=[stack.stack_name for stack in self.stacks],
-        #             build_targets=self.build_targets,
-        #             max_retry=max_retry,
-        #             env=env,
-        #         )
-        #     ]
-        # )
-
         pipeline.add_stage(
             self,
             post=[
@@ -233,6 +221,17 @@ def add_stage_to_pipeline(
             ]
         )
 
-
-
-
+class BatchBuildOptions:
+    def __init__(
+            self,
+            project: str,
+            identifier: str = None,
+            ignore_failure: bool = False,
+            timeout: int = 120,
+            env: typing.Optional[typing.Mapping[str, str]] = None
+    ):
+        self.project = project
+        self.identifier = identifier or re.sub(r'[^a-zA-Z0-9]', '_', project)
+        self.ignore_failure = ignore_failure
+        self.timeout = timeout
+        self.env = env