Add the client part for k8s antithesis

Signed-off-by: Nont <[email protected]>

Author: Nont

tests/antithesis/README.md

@@ -29,7 +29,7 @@ docker images | grep etcd-client
 
 It should show something like:
 
-```
+```bash
 etcd-client        latest    <IMAGE_ID>    <DATE>
 ```
 
@@ -45,7 +45,7 @@ The command uses the etcd client and server images built from step 1.
 
 The client will continuously check the health of the etcd nodes and print logs similar to:
 
-```
+```bash
 [+] Running 4/4
  ✔ Container etcd0   Created                                                                                                                                                 0.0s 
  ✔ Container etcd2   Created                                                                                                                                                 0.0s 
@@ -151,6 +151,33 @@ If you use something other than `kind`, please make sure the images are accessib
 kubectl apply -f ./config/manifests
 ```
 
+#### 4. Run the driver(s)
+
+First, locate your client pod, which should be named with prefix client when you execute `kubectl get pods`.
+
+```bash
+> kubectl get pods
+NAME                      READY   STATUS    RESTARTS   AGE
+client-6b9885f69b-dcndw   1/1     Running   0          10m
+etcd-0                    1/1     Running   0          10m
+etcd-1                    1/1     Running   0          10m
+etcd-2                    1/1     Running   0          10m
+```
+
+With the pod name, you can execute the traffic and validation commands:
+
+```bash
+# Traffic generation
+kubectl exec -it client-6b9885f69b-dcndw -- /opt/antithesis/test/v1/robustness/single_driver_traffic
+
+# Collecting results and compiling the reports
+kubectl exec -it client-6b9885f69b-dcndw -- /opt/antithesis/test/v1/robustness/finally_validation
+```
+
+##### Making changes and retests
+
+If you make changes to the client or server, you will have to rebuild the images and load them into the cluster again. You can use the same commands as in step 2. Once that the images are loaded, you can delete the existing pods to force kubernetes to create new ones with the updated images:
+
 ### Tearing down the cluster
 
 ```bash

tests/antithesis/config/docker-compose-1-node.yml

@@ -9,7 +9,7 @@ services:
     group_add:
       - '${GROUP_ID:-root}'
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
       - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report
     command:
       - /bin/sh
@@ -43,7 +43,7 @@ services:
     ports:
       - 12379:2379
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}0:/var/etcd/data
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}0:/var/etcd/data
 
   client:
     image: 'etcd-client:${IMAGE_TAG:-latest}'
@@ -54,5 +54,5 @@ services:
       etcd0:
         condition: service_started
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
       - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report

tests/antithesis/config/docker-compose-3-node.yml

@@ -9,9 +9,9 @@ services:
     group_add:
       - '${GROUP_ID:-root}'
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}1:/var/etcddata1
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}2:/var/etcddata2
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}1:/var/etcddata1
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}2:/var/etcddata2
       - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report
     command:
       - /bin/sh
@@ -45,7 +45,7 @@ services:
     ports:
       - 12379:2379
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}0:/var/etcd/data
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}0:/var/etcd/data
 
   etcd1:
     image: 'etcd-server:${IMAGE_TAG:-latest}'
@@ -72,7 +72,7 @@ services:
     ports:
       - 22379:2379
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}1:/var/etcd/data
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}1:/var/etcd/data
 
   etcd2:
     image: 'etcd-server:${IMAGE_TAG:-latest}'
@@ -99,7 +99,7 @@ services:
     ports:
       - 32379:2379
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}2:/var/etcd/data
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}2:/var/etcd/data
 
   client:
     image: 'etcd-client:${IMAGE_TAG:-latest}'
@@ -114,7 +114,7 @@ services:
       etcd2:
         condition: service_started
     volumes:
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}1:/var/etcddata1
-      - ${ETCD_ROBUSTNESS_DATA_PATH:-/tmp/etcddata}2:/var/etcddata2
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}0:/var/etcddata0
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}1:/var/etcddata1
+      - ${ETCD_ROBUSTNESS_LOCAL_DATA_PATH:-/tmp/etcddata}2:/var/etcddata2
       - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report

tests/antithesis/config/manifests/default-etcd-3-replicas.yaml

@@ -101,3 +101,66 @@ spec:
         resources:
           requests:
             storage: 200Mi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: report
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 500Mi
+  storageClassName: standard
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: client
+  labels:
+    app: client
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: client
+  template:
+    metadata:
+      labels:
+        app: client
+    spec:
+      containers:
+        - name: etcd-client
+          image: etcd-client:latest
+          command: ["/opt/antithesis/entrypoint/entrypoint"]
+          imagePullPolicy: Never
+          env:
+            - name: ETCD_ROBUSTNESS_SERVER_ENDPOINTS
+              value: "etcd-0.etcd.default.svc.cluster.local:2379,etcd-1.etcd.default.svc.cluster.local:2379,etcd-2.etcd.default.svc.cluster.local:2379"
+            - name: ETCD_ROBUSTNESS_DATA_PATHS
+              value: "/var/etcddata-0,/var/etcddata-1,/var/etcddata-2"
+            - name: ETCD_ROBUSTNESS_REPORT_PATH
+              value: "/var/report/"
+          volumeMounts:
+            - name: etcd-0
+              mountPath: /var/etcddata-0
+            - name: etcd-1
+              mountPath: /var/etcddata-1
+            - name: etcd-2
+              mountPath: /var/etcddata-2
+            - name: report
+              mountPath: /var/report/
+      volumes:
+        - name: etcd-0
+          persistentVolumeClaim:
+            claimName: data-etcd-0
+        - name: etcd-1
+          persistentVolumeClaim:
+            claimName: data-etcd-1
+        - name: etcd-2
+          persistentVolumeClaim:
+            claimName: data-etcd-2
+        - name: report
+          persistentVolumeClaim:
+            claimName: report

tests/antithesis/test-template/entrypoint/main.go

@@ -36,12 +36,23 @@ var NodeCount = "3"
 func CheckHealth() bool {
 	cfg := common.MakeConfig(NodeCount)
 
+	var hosts []string
 	nodeOptions := []string{"etcd0", "etcd1", "etcd2"}[:cfg.NodeCount]
+	for _, n := range nodeOptions {
+		hosts = append(hosts, fmt.Sprintf("%s:2379", n))
+	}
+
+	// override hosts if environment variable is set
+	h, _, _ := common.PathsFromEnv()
+	if len(h) > 0 {
+		hosts = h
+	}
 
 	// iterate over each node and check health
-	for _, node := range nodeOptions {
+	for _, node := range hosts {
+		fmt.Printf("Client [entrypoint]: checking connection with %s\n", node)
 		cli, err := clientv3.New(clientv3.Config{
-			Endpoints:   []string{fmt.Sprintf("http://%s:2379", node)},
+			Endpoints:   []string{node},
 			DialTimeout: 5 * time.Second,
 		})
 		if err != nil {

tests/antithesis/test-template/robustness/common/path.go

@@ -19,6 +19,7 @@ package common
 import (
 	"fmt"
 	"os"
+	"strings"
 )
 
 const (
@@ -34,8 +35,12 @@ const (
 	localetcd2 = "127.0.0.1:32379"
 	// used by default when running the client locally
 	defaultetcdLocalDataPath = "/tmp/etcddata%d"
-	localetcdDataPathEnv     = "ETCD_ROBUSTNESS_DATA_PATH"
+	localetcdDataPathEnv     = "ETCD_ROBUSTNESS_LOCAL_DATA_PATH"
 	localReportPath          = "report"
+
+	serverEndpointEnv = "ETCD_ROBUSTNESS_SERVER_ENDPOINTS"
+	dataPathEnv       = "ETCD_ROBUSTNESS_DATA_PATHS"
+	reportPathEnv     = "ETCD_ROBUSTNESS_REPORT_PATH"
 )
 
 func DefaultPaths(cfg *Config) (hosts []string, reportPath string, dataPaths map[string]string) {
@@ -64,3 +69,20 @@ func etcdDataPaths(dir string, amount int) map[string]string {
 	}
 	return dataPaths
 }
+
+func PathsFromEnv() (hosts []string, reportPath string, dataPaths map[string]string) {
+	hosts = strings.Split(os.Getenv(serverEndpointEnv), ",")
+	d := strings.Split(os.Getenv(dataPathEnv), ",")
+
+	if len(hosts) != len(d) {
+		panic("The number of endpoints and data paths must be equal")
+	}
+
+	dataPaths = make(map[string]string)
+	for i := range hosts {
+		dataPaths[hosts[i]] = d[i]
+	}
+	reportPath = os.Getenv(reportPathEnv)
+
+	return hosts, reportPath, dataPaths
+}

tests/antithesis/test-template/robustness/finally/main.go

@@ -45,6 +45,16 @@ func main() {
 	cfg := common.MakeConfig(NodeCount)
 
 	_, reportPath, dirs := common.DefaultPaths(cfg)
+
+	// override paths if environment variables are set
+	_, rp, d := common.PathsFromEnv()
+	if rp != "" {
+		reportPath = rp
+	}
+	if len(d) > 0 {
+		dirs = d
+	}
+
 	if *local {
 		_, reportPath, dirs = common.LocalPaths(cfg)
 	}

tests/antithesis/test-template/robustness/traffic/main.go

@@ -65,6 +65,19 @@ func main() {
 	cfg := common.MakeConfig(NodeCount)
 
 	hosts, reportPath, etcdetcdDataPaths := common.DefaultPaths(cfg)
+
+	// override paths if environment variables are set
+	h, rp, d := common.PathsFromEnv()
+	if len(h) > 0 {
+		hosts = h
+	}
+	if rp != "" {
+		reportPath = rp
+	}
+	if len(d) > 0 {
+		etcdetcdDataPaths = d
+	}
+
 	if *local {
 		hosts, reportPath, etcdetcdDataPaths = common.LocalPaths(cfg)
 	}