Merge pull request #1 from aws/feature/add-e2e-tests

Feature/add e2e tests

Author: wongni

.gitignore

@@ -20,6 +20,7 @@ pkg/mocks/*
 out
 .build
 .DS_Store
+_artifacts
 
 # CAPI used with Tilt.
 cluster-api

Dockerfile

@@ -1,33 +1,8 @@
-# Build the manager binary
-FROM golang:1.17 as builder
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-# TODO make this env var an optional input. Maybe use args, but need to consider Tilt usage.
-ENV GOPROXY=direct
-RUN go mod download
-
-# Copy the go source
-COPY main.go main.go
-COPY api/ api/
-COPY controllers/ controllers/
-COPY pkg/ pkg/
-
-# Build
-ARG ldflags
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
-    go build -a -ldflags "${ldflags} -extldflags '-static'" \
-    -o manager main.go
-
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/manager .
+COPY bin/manager-linux-amd64 ./manager
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -1,5 +1,6 @@
 # Image URL to use all building/pushing image targets
 IMG ?= public.ecr.aws/a4z9h2b1/cluster-api-provider-capc:latest
+IMG_LOCAL ?= localhost:5000/cluster-api-provider-cloudstack:latest
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -43,7 +44,8 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-MANIFEST_GEN_INPUTS=$(shell find ./api ./controllers -type f -name "*test*" -prune -o -name "*zz_generated*" -prune -o -print)
+ MANIFEST_GEN_INPUTS=$(shell find ./api ./controllers -type f -name "*test*" -prune -o -name "*zz_generated*" -prune -o -print)
+ 
 # Using a flag file here as config output is too complicated to be a target.
 # The following triggers manifest building if $(IMG) differs from that found in config/default/manager_image_patch.yaml.
 $(shell	grep -qs "$(IMG)" config/default/manager_image_patch_edited.yaml || rm -f config/.flag.mk)
@@ -78,14 +80,21 @@ build: binaries generate-deepcopy lint manifests release-manifests ## Build mana
 bin/manager: $(MANAGER_BIN_INPUTS)
 	go build -o bin/manager main.go
 
+.PHONY: build-for-docker
+build-for-docker: bin/manager-linux-amd64 ## Build manager binary for docker image building.
+bin/manager-linux-amd64: $(MANAGER_BIN_INPUTS)
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    	go build -a -ldflags "${ldflags} -extldflags '-static'" \
+    	-o bin/manager-linux-amd64 main.go
+
 .PHONY: run
 run: generate-deepcopy ## Run a controller from your host.
 	go run ./main.go
 
 # Using a flag file here as docker build doesn't produce a target file.
 DOCKER_BUILD_INPUTS=$(MANAGER_BIN_INPUTS) Dockerfile
 .PHONY: docker-build
-docker-build: .dockerflag.mk ## Build docker image containing the controller manager.
+docker-build: generate-deepcopy build-for-docker .dockerflag.mk ## Build docker image containing the controller manager.
 .dockerflag.mk: $(DOCKER_BUILD_INPUTS)
 	docker build -t ${IMG} .
 	@touch .dockerflag.mk
@@ -173,16 +182,36 @@ pkg/mocks/mock%.go: $(shell find ./pkg/cloud -type f -name "*test*" -prune -o -p
 ##@ Tilt
 
 .PHONY: tilt-up 
-tilt-up: cluster-api kind-cluster cluster-api/tilt-settings.json manifests cloud-config # Setup and run tilt for development.
+tilt-up: cluster-api kind-cluster cluster-api/tilt-settings.json manifests cloud-config ## Setup and run tilt for development.
 	export CLOUDSTACK_B64ENCODED_SECRET=$$(base64 -w0 -i cloud-config 2>/dev/null || base64 -b 0 -i cloud-config) && cd cluster-api && tilt up
 
 .PHONY: kind-cluster
-kind-cluster: cluster-api # Create a kind cluster with a local Docker repository.
+kind-cluster: cluster-api ## Create a kind cluster with a local Docker repository.
 	-./cluster-api/hack/kind-install-for-capd.sh
 
-cluster-api: # Clone cluster-api repository for tilt use.
+cluster-api: ## Clone cluster-api repository for tilt use.
 	git clone --branch v1.0.0 https://github.com/kubernetes-sigs/cluster-api.git
 
 cluster-api/tilt-settings.json: hack/tilt-settings.json cluster-api
 	cp ./hack/tilt-settings.json cluster-api
 
+##@ End-to-End Testing
+
+CLUSTER_TEMPLATES_INPUT_FILES=$(shell find test/e2e/data/infrastructure-cloudstack/*/cluster-template*/* test/e2e/data/infrastructure-cloudstack/*/bases/* -type f)
+CLUSTER_TEMPLATES_OUTPUT_FILES=$(shell find test/e2e/data/infrastructure-cloudstack -type d -name "cluster-template*" -exec echo {}.yaml \;)
+.PHONY: e2e-cluster-templates
+e2e-cluster-templates: $(CLUSTER_TEMPLATES_OUTPUT_FILES) ## Generate cluster template files for e2e testing.
+cluster-template%yaml: bin/kustomize $(CLUSTER_TEMPLATES_INPUT_FILES)
+	kustomize build --load-restrictor LoadRestrictionsNone $(basename $@) > $@
+
+e2e-essentials: bin/ginkgo e2e-cluster-templates kind-cluster ## Fulfill essential tasks for e2e testing.
+	IMG=$(IMG_LOCAL) make manifests docker-build docker-push
+
+JOB ?= .*
+run-e2e: e2e-essentials ## Run e2e testing. JOB is an optional REGEXP to select certainn test cases to run. e.g. JOB=PR-Blocking, JOB=Conformance
+	cd test/e2e && \
+	ginkgo -v -trace -tags=e2e -focus=$(JOB) -skip=Conformance -nodes=1 --noColor=false ./... -- \
+	    -e2e.artifacts-folder=${PROJECT_DIR}/_artifacts \
+	    -e2e.config=${PROJECT_DIR}/test/e2e/config/cloudstack.yaml \
+	    -e2e.skip-resource-cleanup=false -e2e.use-existing-cluster=true
+	kind delete clusters capi-test

config/default/credentials.yaml

@@ -5,4 +5,4 @@ metadata:
   namespace: system
 type: Opaque
 data:
-  cloud-config: ${CLOUDSTACK_B64ENCODED_SECRET}
+  cloud-config: "${CLOUDSTACK_B64ENCODED_SECRET}"

test/e2e/.gitignore

@@ -0,0 +1,2 @@
+kind-install-for-capd.sh
+data/infrastructure-cloudstack/v1beta1/*.yaml

test/e2e/README.md

@@ -0,0 +1,65 @@
+# Testing
+
+This document is to help developers understand how to test CAPC.
+
+## Code Origin
+
+The most of the code under test/e2e is from CAPD (Cluster API for Docker) e2e testing (https://github.com/kubernetes-sigs/cluster-api/tree/main/test/e2e)
+The ACS specific things are under test/e2e/config and test/e2e/data/infrastructure-cloudstack. 
+
+## e2e
+
+This section describes how to run end-to-end (e2e) testing with CAPC.
+
+### Requirements
+
+* Admin access to a Apache CloudStack (ACS) server
+* The testing must occur on a host that can access the ACS server
+* Docker ([download](https://www.docker.com/get-started))
+* Kind ([download](https://kind.sigs.k8s.io/docs/user/quick-start/#installing-with-a-package-manager))
+
+### Environment variables
+
+The first step to running the e2e tests is setting up the required environment variables:
+
+| Environment variable                        | Description                                                                      | Default Value               |
+|---------------------------------------------|----------------------------------------------------------------------------------|-----------------------------|
+| `CLOUDSTACK_ZONE_NAME`                      | The zone name                                                                    | `zone1`                     |
+| `CLOUDSTACK_NETWORK_NAME`                   | The network name. If not exisiting an isolated network with the name is created. | `Shared1`                   |
+| `CLUSTER_ENDPOINT_IP`                       | The cluster endpoint IP                                                          | `172.16.2.199`              |
+| `CLUSTER_ENDPOINT_PORT`                     | The cluster endpoint port                                                        | `6443`                      |
+| `CLUSTER_ENDPOINT_IP_2`                     | The cluster endpoint IP for a second cluster                                     | `172.16.2.199`              |
+| `CLUSTER_ENDPOINT_PORT_2`                   | The cluster endpoint port for a second cluster                                   | `6444`                      |
+| `CLOUDSTACK_CONTROL_PLANE_MACHINE_OFFERING` | The machine offering for the control plane VM instances                          | `Large Instance`            |
+| `CLOUDSTACK_WORKER_MACHINE_OFFERING`        | The machine offering for the worker node VM instances                            | `Medium Instance`           |
+| `CLOUDSTACK_TEMPLATE_NAME`                  | The machine template for both control plane and worke node VM instances          | `kube-v1.20.10/ubuntu-2004` |
+| `CLOUDSTACK_SSH_KEY_NAME`                   | The name of SSH key added to the VM instances                                    | `CAPCKeyPair6`              |
+
+You also have to export `CLOUDSTACK_B64ENCODED_SECRET` environment variable using this command `export CLOUDSTACK_B64ENCODED_SECRET=$(base64 -i cloud-config)` after creating `cloud-config` file with the following format.
+
+```
+[Global]
+api-key    = XXXXX
+secret-key = XXXXX
+api-url    = http://192.168.1.96:8080/client/api
+verify-ssl = true or false
+```
+
+The api-key and secret-key can be found or generated at Home > Accounts > admin > Users > admin of the ACS management UI. `verify-ssl` is an optional flag and its default value is true. CAPC skips verifying the host SSL certificates when the flag is set to false.
+
+### Running the e2e tests
+
+Run the following command to execute the CAPC e2e tests:
+
+```shell
+make run-e2e
+```
+This command runs all e2e test cases.
+
+You can specify JOB environment variable which value is a regular expression to select test cases to execute. 
+For example, 
+
+```shell
+JOB=PR-Blocking make run
+```
+This command runs the e2e tests that contains `PR-Blocking` in their spec names. 

test/e2e/affinity_group.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// AffinityGroupSpec implements a test that verifies that an app deployed to the workload cluster works.
+func AffinityGroupSpec(ctx context.Context, inputGetter func() CommonSpecInput) {
+	var (
+		specName         = "affinity-group"
+		input            CommonSpecInput
+		namespace        *corev1.Namespace
+		cancelWatches    context.CancelFunc
+		clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
+		affinityIds      []string
+	)
+
+	BeforeEach(func() {
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
+		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
+		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
+		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
+
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
+		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
+	})
+
+	It("Should have host affinity group when affinity is pro", func() {
+		executeTest(ctx, input, namespace, specName, clusterResources, "pro")
+	})
+
+	It("Should have host affinity group when affinity is anti", func() {
+		Skip("The ACS used by Prow doesn't have multiple hosts in the target zone")
+		executeTest(ctx, input, namespace, specName, clusterResources, "anti")
+	})
+
+	AfterEach(func() {
+		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
+		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
+
+		err := CheckAffinityGroupsDeleted(affinityIds)
+		if err != nil {
+			Fail(err.Error())
+		}
+	})
+}
+
+func executeTest(ctx context.Context, input CommonSpecInput, namespace *corev1.Namespace, specName string, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult, affinityType string) []string {
+	clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+		ClusterProxy:    input.BootstrapClusterProxy,
+		CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
+		ConfigCluster: clusterctl.ConfigClusterInput{
+			LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
+			ClusterctlConfigPath:     input.ClusterctlConfigPath,
+			KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+			InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+			Flavor:                   "affinity-group-" + affinityType,
+			Namespace:                namespace.Name,
+			ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+			KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
+			ControlPlaneMachineCount: pointer.Int64Ptr(3),
+			WorkerMachineCount:       pointer.Int64Ptr(3),
+		},
+		WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+		WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+		WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+	}, clusterResources)
+
+	affinityIds := CheckAffinityGroup(clusterResources.Cluster.Name, affinityType)
+
+	By("PASSED!")
+
+	return affinityIds
+}

test/e2e/affinity_group_test.go

@@ -0,0 +1,38 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = Describe("When testing affinity group", func() {
+	AffinityGroupSpec(context.TODO(), func() CommonSpecInput {
+		return CommonSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+		}
+	})
+
+})