Skip to content

Commit 45a1b91

Browse files
nhatnghihonhatnghiho
committed
Create pipeline
1 parent 652f7a9 commit 45a1b91

38 files changed

+2012
-223
lines changed

‎tests/ci/cdk/README.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,26 @@ To setup or update the CI in your account you will need the following IAM permis
6363
* secretsmanager:DeleteSecret
6464
* secretsmanager:GetSecretValue
6565

66+
### Pipeline Commands
67+
Bootstrap pipeline account
68+
```
69+
AWS_ACCOUNT_ID=183295444613
70+
PIPELINE_ACCOUNT_ID=774305600158
71+
cdk bootstrap aws://${PIPELINE_ACCOUNT_ID}/us-west-2
72+
```
73+
74+
Give pipeline account administrator access to deployment account's CloudFormation
75+
```
76+
cdk bootstrap aws://${AWS_ACCOUNT_ID}/us-west-2 --trust ${PIPELINE_ACCOUNT_ID} --trust-for-lookup ${PIPELINE_ACCOUNT_ID} --cloudformation-execution-policies arn:aws:iam::aws:policy/AdministratorAccess
77+
```
78+
79+
Deploy pipeline
80+
```
81+
GITHUB_REPO_OWNER=nhatnghiho
82+
GITHUB_SOURCE_VERSION=ci-pipeline
83+
./run-cdk.sh --github-repo-owner ${GITHUB_REPO_OWNER} --github-source-version ${GITHUB_SOURCE_VERSION} --aws-account ${AWS_ACCOUNT_ID} --action invoke --command "cdk deploy AwsLcCiPipeline --require-approval never"
84+
```
85+
6686
### Commands
6787

6888
These commands are run from `aws-lc/tests/ci/cdk`. \

‎tests/ci/cdk/app.py

Lines changed: 42 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -12,49 +12,54 @@
1212
from cdk.aws_lc_github_fuzz_ci_stack import AwsLcGitHubFuzzCIStack
1313
from cdk.aws_lc_ec2_test_framework_ci_stack import AwsLcEC2TestingCIStack
1414
from cdk.linux_docker_image_batch_build_stack import LinuxDockerImageBatchBuildStack
15+
from pipeline.pipeline_stack import AwsLcCiPipeline
1516
from cdk.windows_docker_image_build_stack import WindowsDockerImageBuildStack
1617
from cdk.aws_lc_github_ci_x509_stack import AwsLcGitHubX509CIStack
1718
from cdk.ecr_stack import EcrStack
18-
from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_X86_ECR_REPO
19+
from util.metadata import LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_X86_ECR_REPO, \
20+
PIPELINE_ACCOUNT, PIPELINE_REGION, DEPLOY_ACCOUNT, DEPLOY_REGION
1921

2022
# Initialize app.
2123
app = App()
2224

23-
# Initialize env.
24-
env = Environment(account=AWS_ACCOUNT, region=AWS_REGION)
25-
26-
# Define AWS ECR stacks.
27-
# ECR holds the docker images, which are pre-built to accelerate the code builds/tests of git pull requests.
28-
EcrStack(app, "aws-lc-ecr-linux-x86", LINUX_X86_ECR_REPO, env=env)
29-
EcrStack(app, "aws-lc-ecr-linux-aarch", LINUX_AARCH_ECR_REPO, env=env)
30-
EcrStack(app, "aws-lc-ecr-windows-x86", WINDOWS_X86_ECR_REPO, env=env)
31-
32-
# Define CodeBuild Batch job for building Docker images.
33-
LinuxDockerImageBatchBuildStack(app, "aws-lc-docker-image-build-linux", env=env)
34-
35-
# AWS CodeBuild cannot build Windows Docker images because DIND (Docker In Docker) is not supported on Windows.
36-
# Windows Docker images are created by running commands in Windows EC2 instance.
37-
WindowsDockerImageBuildStack(app, "aws-lc-docker-image-build-windows", env=env)
38-
39-
# Define CodeBuild Batch job for testing code.
40-
x86_build_spec_file = "cdk/codebuild/github_ci_linux_x86_omnibus.yaml"
41-
AwsLcGitHubCIStack(app, "aws-lc-ci-linux-x86", x86_build_spec_file, env=env)
42-
arm_build_spec_file = "cdk/codebuild/github_ci_linux_arm_omnibus.yaml"
43-
AwsLcGitHubCIStack(app, "aws-lc-ci-linux-arm", arm_build_spec_file, env=env)
44-
integration_build_spec_file = "cdk/codebuild/github_ci_integration_omnibus.yaml"
45-
AwsLcGitHubCIStack(app, "aws-lc-ci-integration", integration_build_spec_file, env=env)
46-
win_x86_build_spec_file = "cdk/codebuild/github_ci_windows_x86_omnibus.yaml"
47-
AwsLcGitHubCIStack(app, "aws-lc-ci-windows-x86", win_x86_build_spec_file, env=env)
48-
fuzz_build_spec_file = "cdk/codebuild/github_ci_fuzzing_omnibus.yaml"
49-
AwsLcGitHubFuzzCIStack(app, "aws-lc-ci-fuzzing", fuzz_build_spec_file, env=env)
50-
analytics_build_spec_file = "cdk/codebuild/github_ci_analytics_omnibus.yaml"
51-
AwsLcGitHubAnalyticsStack(app, "aws-lc-ci-analytics", analytics_build_spec_file, env=env)
52-
# bm_framework_build_spec_file = "cdk/codebuild/bm_framework_omnibus.yaml"
53-
# BmFrameworkStack(app, "aws-lc-ci-bm-framework", bm_framework_build_spec_file, env=env)
54-
ec2_test_framework_build_spec_file = "cdk/codebuild/ec2_test_framework_omnibus.yaml"
55-
AwsLcEC2TestingCIStack(app, "aws-lc-ci-ec2-test-framework", ec2_test_framework_build_spec_file, env=env)
56-
android_build_spec_file = "cdk/codebuild/github_ci_android_omnibus.yaml"
57-
AwsLcAndroidCIStack(app, "aws-lc-ci-devicefarm-android", android_build_spec_file, env=env)
58-
AwsLcGitHubX509CIStack(app, "aws-lc-ci-x509")
25+
AwsLcCiPipeline(app, "AwsLcCiPipeline", env=Environment(account=PIPELINE_ACCOUNT, region=PIPELINE_REGION))
26+
27+
if DEPLOY_ACCOUNT is not None and DEPLOY_REGION is not None:
28+
# Initialize env.
29+
env = Environment(account=DEPLOY_ACCOUNT, region=DEPLOY_REGION)
30+
31+
# Define AWS ECR stacks.
32+
# ECR holds the docker images, which are pre-built to accelerate the code builds/tests of git pull requests.
33+
EcrStack(app, "aws-lc-ecr-linux-x86", LINUX_X86_ECR_REPO, env=env)
34+
EcrStack(app, "aws-lc-ecr-linux-aarch", LINUX_AARCH_ECR_REPO, env=env)
35+
EcrStack(app, "aws-lc-ecr-windows-x86", WINDOWS_X86_ECR_REPO, env=env)
36+
37+
# Define CodeBuild Batch job for building Docker images.
38+
LinuxDockerImageBatchBuildStack(app, "aws-lc-docker-image-build-linux", env=env)
39+
40+
# AWS CodeBuild cannot build Windows Docker images because DIND (Docker In Docker) is not supported on Windows.
41+
# Windows Docker images are created by running commands in Windows EC2 instance.
42+
WindowsDockerImageBuildStack(app, "aws-lc-docker-image-build-windows", env=env)
43+
44+
# Define CodeBuild Batch job for testing code.
45+
x86_build_spec_file = "cdk/codebuild/github_ci_linux_x86_omnibus.yaml"
46+
AwsLcGitHubCIStack(app, "aws-lc-ci-linux-x86", x86_build_spec_file, env=env)
47+
arm_build_spec_file = "cdk/codebuild/github_ci_linux_arm_omnibus.yaml"
48+
AwsLcGitHubCIStack(app, "aws-lc-ci-linux-arm", arm_build_spec_file, env=env)
49+
integration_build_spec_file = "cdk/codebuild/github_ci_integration_omnibus.yaml"
50+
AwsLcGitHubCIStack(app, "aws-lc-ci-integration", integration_build_spec_file, env=env)
51+
win_x86_build_spec_file = "cdk/codebuild/github_ci_windows_x86_omnibus.yaml"
52+
AwsLcGitHubCIStack(app, "aws-lc-ci-windows-x86", win_x86_build_spec_file, env=env)
53+
fuzz_build_spec_file = "cdk/codebuild/github_ci_fuzzing_omnibus.yaml"
54+
AwsLcGitHubFuzzCIStack(app, "aws-lc-ci-fuzzing", fuzz_build_spec_file, env=env)
55+
analytics_build_spec_file = "cdk/codebuild/github_ci_analytics_omnibus.yaml"
56+
AwsLcGitHubAnalyticsStack(app, "aws-lc-ci-analytics", analytics_build_spec_file, env=env)
57+
# bm_framework_build_spec_file = "cdk/codebuild/bm_framework_omnibus.yaml"
58+
# BmFrameworkStack(app, "aws-lc-ci-bm-framework", bm_framework_build_spec_file, env=env)
59+
ec2_test_framework_build_spec_file = "cdk/codebuild/ec2_test_framework_omnibus.yaml"
60+
AwsLcEC2TestingCIStack(app, "aws-lc-ci-ec2-test-framework", ec2_test_framework_build_spec_file, env=env)
61+
android_build_spec_file = "cdk/codebuild/github_ci_android_omnibus.yaml"
62+
AwsLcAndroidCIStack(app, "aws-lc-ci-devicefarm-android", android_build_spec_file, env=env)
63+
AwsLcGitHubX509CIStack(app, "aws-lc-ci-x509", env=env)
5964

6065
app.synth()

‎tests/ci/cdk/cdk/aws_lc_analytics_stack.py

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
3+
import typing
34

4-
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
5+
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs, \
6+
Environment
57
from constructs import Construct
68

79
from cdk.components import PruneStaleGitHubBuilds
810
from util.iam_policies import code_build_publish_metrics_in_json
9-
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
11+
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, \
12+
STAGING_GITHUB_REPO_NAME
1013
from util.build_spec_loader import BuildSpecLoader
1114

1215

@@ -17,13 +20,22 @@ def __init__(self,
1720
scope: Construct,
1821
id: str,
1922
spec_file_path: str,
23+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2024
**kwargs) -> None:
21-
super().__init__(scope, id, **kwargs)
25+
super().__init__(scope, id, env=env, **kwargs)
26+
27+
# Define CodeBuild resource.
28+
github_repo_owner = GITHUB_REPO_OWNER
29+
github_repo_name = GITHUB_REPO_NAME
30+
31+
if env.account == PRE_PROD_ACCOUNT:
32+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
33+
github_repo_name = STAGING_GITHUB_REPO_NAME
2234

2335
# Define CodeBuild resource.
2436
git_hub_source = codebuild.Source.git_hub(
25-
owner=GITHUB_REPO_OWNER,
26-
repo=GITHUB_REPO_NAME,
37+
owner=github_repo_owner,
38+
repo=github_repo_name,
2739
webhook=True,
2840
webhook_filters=[
2941
codebuild.FilterGroup.in_event_of(codebuild.EventAction.PUSH)
@@ -34,7 +46,7 @@ def __init__(self,
3446
webhook_triggers_batch_build=True)
3547

3648
# Define a IAM role for this stack.
37-
metrics_policy = iam.PolicyDocument.from_json(code_build_publish_metrics_in_json())
49+
metrics_policy = iam.PolicyDocument.from_json(code_build_publish_metrics_in_json(env))
3850
inline_policies = {"metric_policy": metrics_policy}
3951
role = iam.Role(scope=self,
4052
id="{}-role".format(id),
@@ -52,7 +64,7 @@ def __init__(self,
5264
environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.LARGE,
5365
privileged=True,
5466
build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
55-
build_spec=BuildSpecLoader.load(spec_file_path))
67+
build_spec=BuildSpecLoader.load(spec_file_path, env))
5668
analytics.enable_batch_builds()
5769

58-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics, ec2_permissions=False)
70+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics, ec2_permissions=False, env=env)

‎tests/ci/cdk/cdk/aws_lc_android_ci_stack.py

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
3+
import typing
34

4-
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam
5+
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, Environment
56
from constructs import Construct
67

78
from cdk.components import PruneStaleGitHubBuilds
89
from util.iam_policies import code_build_batch_policy_in_json, device_farm_access_policy_in_json
9-
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME, GITHUB_PUSH_CI_BRANCH_TARGETS
10+
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME, GITHUB_PUSH_CI_BRANCH_TARGETS, PRE_PROD_ACCOUNT, \
11+
STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
1012
from util.build_spec_loader import BuildSpecLoader
1113

1214

@@ -20,13 +22,21 @@ def __init__(self,
2022
scope: Construct,
2123
id: str,
2224
spec_file_path: str,
25+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2326
**kwargs) -> None:
24-
super().__init__(scope, id, **kwargs)
27+
super().__init__(scope, id, env=env, **kwargs)
28+
29+
github_repo_owner = GITHUB_REPO_OWNER
30+
github_repo_name = GITHUB_REPO_NAME
31+
32+
if env.account == PRE_PROD_ACCOUNT:
33+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
34+
github_repo_name = STAGING_GITHUB_REPO_NAME
2535

2636
# Define CodeBuild resource.
2737
git_hub_source = codebuild.Source.git_hub(
28-
owner=GITHUB_REPO_OWNER,
29-
repo=GITHUB_REPO_NAME,
38+
owner=github_repo_owner,
39+
repo=github_repo_name,
3040
webhook=True,
3141
webhook_filters=[
3242
codebuild.FilterGroup.in_event_of(
@@ -40,10 +50,10 @@ def __init__(self,
4050

4151
# Define a IAM role for this stack.
4252
code_build_batch_policy = iam.PolicyDocument.from_json(
43-
code_build_batch_policy_in_json([id])
53+
code_build_batch_policy_in_json([id], env)
4454
)
4555
device_farm_policy = iam.PolicyDocument.from_json(
46-
device_farm_access_policy_in_json()
56+
device_farm_access_policy_in_json(env)
4757
)
4858
inline_policies = {"code_build_batch_policy": code_build_batch_policy, "device_farm_policy": device_farm_policy}
4959
role = iam.Role(scope=self,
@@ -62,7 +72,7 @@ def __init__(self,
6272
environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL,
6373
privileged=False,
6474
build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
65-
build_spec=BuildSpecLoader.load(spec_file_path))
75+
build_spec=BuildSpecLoader.load(spec_file_path, env))
6676
project.enable_batch_builds()
6777

68-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)
78+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False, env=env)

‎tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,19 @@
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
33

44
import subprocess
5+
import typing
6+
57
import boto3
68

79
from botocore.exceptions import ClientError
8-
from aws_cdk import CfnTag, Duration, Stack, Tags, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
10+
from aws_cdk import CfnTag, Duration, Stack, Tags, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, \
11+
aws_s3 as s3, aws_logs as logs, Environment
912
from constructs import Construct
1013

1114
from cdk.components import PruneStaleGitHubBuilds
12-
from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, LINUX_AARCH_ECR_REPO, \
13-
LINUX_X86_ECR_REPO
15+
from util.metadata import GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, \
16+
LINUX_AARCH_ECR_REPO, \
17+
LINUX_X86_ECR_REPO, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
1418
from util.iam_policies import code_build_batch_policy_in_json, ec2_policies_in_json, ssm_policies_in_json, s3_read_write_policy_in_json, ecr_power_user_policy_in_json
1519
from util.build_spec_loader import BuildSpecLoader
1620

@@ -23,13 +27,21 @@ def __init__(self,
2327
scope: Construct,
2428
id: str,
2529
spec_file_path: str,
30+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2631
**kwargs) -> None:
27-
super().__init__(scope, id, **kwargs)
32+
super().__init__(scope, id, env=env, **kwargs)
33+
34+
github_repo_owner = GITHUB_REPO_OWNER
35+
github_repo_name = GITHUB_REPO_NAME
36+
37+
if env.account == PRE_PROD_ACCOUNT:
38+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
39+
github_repo_name = STAGING_GITHUB_REPO_NAME
2840

2941
# Define CodeBuild resource.
3042
git_hub_source = codebuild.Source.git_hub(
31-
owner=GITHUB_REPO_OWNER,
32-
repo=GITHUB_REPO_NAME,
43+
owner=github_repo_owner,
44+
repo=github_repo_name,
3345
webhook=True,
3446
webhook_filters=[
3547
codebuild.FilterGroup.in_event_of(
@@ -43,7 +55,7 @@ def __init__(self,
4355

4456
# S3 bucket for testing internal fixes.
4557
s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json("aws-lc-codebuild"))
46-
ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json([LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO]))
58+
ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json([LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO], env))
4759
ec2_inline_policies = {"s3_read_write_policy": s3_read_write_policy, "ecr_power_user_policy": ecr_power_user_policy}
4860
ec2_role = iam.Role(scope=self, id="{}-ec2-role".format(id),
4961
role_name="{}-ec2-role".format(id),
@@ -62,16 +74,15 @@ def __init__(self,
6274
selected_subnets = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
6375

6476
# create security group with default rules
65-
security_group = ec2.SecurityGroup(self, id="{}-ec2-sg".format(id),
66-
allow_all_outbound=True,
67-
vpc=vpc,
68-
security_group_name='codebuild_ec2_sg')
69-
77+
# security_group = ec2.SecurityGroup(self, id="{}-ec2-sg".format(id),
78+
# allow_all_outbound=True,
79+
# vpc=vpc,
80+
# security_group_name='codebuild_ec2_sg')
7081

7182
# Define a IAM role for this stack.
72-
code_build_batch_policy = iam.PolicyDocument.from_json(code_build_batch_policy_in_json([id]))
73-
ec2_policy = iam.PolicyDocument.from_json(ec2_policies_in_json(ec2_role.role_name, security_group.security_group_id, selected_subnets.subnets[0].subnet_id, vpc.vpc_id))
74-
ssm_policy = iam.PolicyDocument.from_json(ssm_policies_in_json())
83+
code_build_batch_policy = iam.PolicyDocument.from_json(code_build_batch_policy_in_json([id], env))
84+
ec2_policy = iam.PolicyDocument.from_json(ec2_policies_in_json(ec2_role.role_name, vpc.vpc_default_security_group, selected_subnets.subnets[0].subnet_id, vpc.vpc_id, env))
85+
ssm_policy = iam.PolicyDocument.from_json(ssm_policies_in_json(env))
7586
codebuild_inline_policies = {"code_build_batch_policy": code_build_batch_policy,
7687
"ec2_policy": ec2_policy,
7788
"ssm_policy": ssm_policy}
@@ -94,10 +105,10 @@ def __init__(self,
94105
environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL,
95106
privileged=False,
96107
build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
97-
build_spec=BuildSpecLoader.load(spec_file_path),
108+
build_spec=BuildSpecLoader.load(spec_file_path, env),
98109
environment_variables= {
99110
"EC2_SECURITY_GROUP_ID": codebuild.BuildEnvironmentVariable(
100-
value=security_group.security_group_id
111+
value=vpc.vpc_default_security_group
101112
),
102113
"EC2_SUBNET_ID": codebuild.BuildEnvironmentVariable(
103114
value=selected_subnets.subnets[0].subnet_id
@@ -108,7 +119,7 @@ def __init__(self,
108119
})
109120
project.enable_batch_builds()
110121

111-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=True)
122+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=True, env=env)
112123

113124
# Define logs for SSM.
114125
log_group_name = "{}-cw-logs".format(id)

0 commit comments

Comments
 (0)