add test coverage for bug OCPBUGS-63348

Signed-off-by: Huiran Wang <[email protected]>

Author: huiran0826

test/extended/networking/egressip.go

@@ -17,6 +17,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	"k8s.io/kubernetes/test/e2e/framework/skipper"
 	admissionapi "k8s.io/pod-security-admission/api"
 
@@ -564,6 +565,124 @@ var _ = g.Describe("[sig-network][Feature:EgressIP][apigroup:operator.openshift.
 				spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet)
 			}
 		})
+
+		g.It("Rebooting a node/Restarting CNCC pod should not change the EgressIPs capacity", func() {
+			g.By("Get one Egress node")
+			egressNodeName := egressIPNodesOrderedNames[0]
+			for _, node := range egressIPNodesOrderedNames[1:] {
+				// Only keep one egress node, remove egress labels from other nodes
+				_, _ = runOcWithRetry(oc.AsAdmin(), "label", "node", node, "k8s.ovn.org/egress-assignable-")
+			}
+
+			g.By("Get capacity of one Egress node before reboot")
+			egressNode, err := clientset.CoreV1().Nodes().Get(context.TODO(), egressNodeName, metav1.GetOptions{})
+			o.Expect(err).NotTo(o.HaveOccurred())
+			nodeEgressIPConfigs, err := getNodeEgressIPConfiguration(egressNode)
+			o.Expect(err).NotTo(o.HaveOccurred())
+			capacityBeforeReboot := nodeEgressIPConfigs[0].Capacity
+			framework.Logf("The capacity of node %s before reboot is %v", egressNodeName, capacityBeforeReboot)
+
+			g.By("Getting a map of source nodes and potential Egress IPs for these nodes")
+			var egressIPsPerNode int
+			if capacityBeforeReboot.IPv4 == 0 {
+				// On OpenStack and GCP, the capacity per node is the number of IPs per node, not the number of IPv4 addresses, like "capacity":{"ip":8}
+				egressIPsPerNode = capacityBeforeReboot.IP
+			} else {
+				egressIPsPerNode = capacityBeforeReboot.IPv4
+			}
+			if egressIPsPerNode > 20 {
+				egressIPsPerNode = 20
+			}
+
+			nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode)
+			framework.Logf("%v", nodeEgressIPMap)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			g.By(fmt.Sprintf("Creating %d EgressIPs objects for the Egress node", egressIPsPerNode))
+			defer func() {
+				g.By("Deleting all EgressIPs objects")
+				_, err := runOcWithRetry(oc.AsAdmin(), "delete", "egressIP", "--all")
+				o.Expect(err).NotTo(o.HaveOccurred())
+			}()
+			for i := 0; i < egressIPsPerNode; i++ {
+				egressIPSet := make(map[string]string)
+				egressIPSet[nodeEgressIPMap[egressNodeName][i]] = egressNodeName
+
+				g.By(fmt.Sprintf("Creating %dth EgressIP object ", i))
+				egressIPObjectName := egressIPNamespace + fmt.Sprintf("-%d", i)
+				egressIPYamlPath := tmpDirEgressIP + "/" + fmt.Sprintf("egressip-%d.yaml", i)
+				createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet)
+
+				g.By(fmt.Sprintf("Applying %dth EgressIP object ", i))
+				applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPObjectName, egressIPSet, egressUpdateTimeout)
+
+			}
+			g.By("Rebooting the node gracefully")
+			err = exutil.TriggerNodeRebootGraceful(clientset, egressNodeName)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			g.By("Waiting for the node to become NotReady")
+			ctx := context.TODO()
+			isNotReady := e2enode.WaitForNodeToBeNotReady(ctx, clientset, egressNodeName, 10*time.Minute)
+			o.Expect(isNotReady).To(o.BeTrue(), "Node should become NotReady after reboot trigger")
+
+			g.By("Waiting for the node to become Ready again after reboot")
+			isReady := e2enode.WaitForNodeToBeReady(ctx, clientset, egressNodeName, 15*time.Minute)
+			o.Expect(isReady).To(o.BeTrue(), "Node should become Ready again after reboot")
+
+			g.By("Waiting for all EgressIPs to be assigned")
+			err = waitAllEgressIPsAssigned(oc, 15*time.Minute)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			g.By("Get capacity of the Egress node after reboot")
+			nodeEgressIPConfigsAfterReboot, err := getNodeEgressIPConfiguration(egressNode)
+			o.Expect(err).NotTo(o.HaveOccurred())
+			capacityAfterReboot := nodeEgressIPConfigsAfterReboot[0].Capacity
+			framework.Logf("The capacity of node %s after reboot is %v", egressNodeName, capacityAfterReboot)
+
+			g.By("Checking CloudPrivateIPConfigs for CloudResponseError, should be 0, after node reboot")
+			errorCount, err := countCloudPrivateIPConfigsByReason(oc, "CloudResponseError")
+			o.Expect(err).NotTo(o.HaveOccurred())
+			framework.Logf("Found %d CloudPrivateIPConfigs with CloudResponseError", errorCount)
+			o.Expect(errorCount).To(o.Equal(0), "Expected no CloudPrivateIPConfigs with CloudResponseError, but found %d", errorCount)
+
+			g.By("Checking CloudPrivateIPConfigs for CloudResponseSuccess")
+			successCount, err := countCloudPrivateIPConfigsByReason(oc, "CloudResponseSuccess")
+			o.Expect(err).NotTo(o.HaveOccurred())
+			framework.Logf("Found %d CloudPrivateIPConfigs with CloudResponseSuccess", successCount)
+			o.Expect(successCount).To(o.Equal(egressIPsPerNode), "Expected %d CloudPrivateIPConfigs with CloudResponseSuccess, but found %d", egressIPsPerNode, successCount)
+
+			g.By("Comparing capacity before and after reboot")
+			o.Expect(capacityAfterReboot).To(o.Equal(capacityBeforeReboot),
+				"EgressIP capacity should remain the same after node reboot. Before: %v, After: %v",
+				capacityBeforeReboot, capacityAfterReboot)
+
+			g.By("Restarting the CNCC pod")
+			restartCNCCPod(oc, clientset)
+
+			g.By("Get capacity of the Egress node after CNCC restart")
+			nodeEgressIPConfigsAfterCNCCRestart, err := getNodeEgressIPConfiguration(egressNode)
+			o.Expect(err).NotTo(o.HaveOccurred())
+			capacityAfterCNCCRestart := nodeEgressIPConfigsAfterCNCCRestart[0].Capacity
+			framework.Logf("The capacity of node %s after CNCC restart is %v", egressNodeName, capacityAfterCNCCRestart)
+
+			g.By("Comparing capacity before and after CNCC restart")
+			o.Expect(capacityAfterCNCCRestart).To(o.Equal(capacityBeforeReboot),
+				"EgressIP capacity should remain the same after CNCC restart. Before: %v, After: %v",
+				capacityBeforeReboot, capacityAfterCNCCRestart)
+
+			g.By("Checking CloudPrivateIPConfigs for CloudResponseError,should be 0, after CNCC restart")
+			errorCount, err = countCloudPrivateIPConfigsByReason(oc, "CloudResponseError")
+			o.Expect(err).NotTo(o.HaveOccurred())
+			framework.Logf("Found %d CloudPrivateIPConfigs with CloudResponseError", errorCount)
+			o.Expect(errorCount).To(o.Equal(0), "Expected no CloudPrivateIPConfigs with CloudResponseError, but found %d", errorCount)
+
+			g.By("Checking CloudPrivateIPConfigs for CloudResponseSuccess")
+			successCount, err = countCloudPrivateIPConfigsByReason(oc, "CloudResponseSuccess")
+			o.Expect(err).NotTo(o.HaveOccurred())
+			framework.Logf("Found %d CloudPrivateIPConfigs with CloudResponseSuccess", successCount)
+			o.Expect(successCount).To(o.Equal(egressIPsPerNode), "Expected %d CloudPrivateIPConfigs with CloudResponseSuccess, but found %d", egressIPsPerNode, successCount)
+		})
 	}) // end testing to external targets
 })
 

test/extended/networking/egressip_helpers.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	o "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
@@ -1738,3 +1739,97 @@ func getEgressIP(oc *exutil.CLI, name string) (*EgressIP, error) {
 	}
 	return egressip, nil
 }
+
+// restartCNCCPod restarts the CNCC pod by deleting it and waiting for it to become Ready again.
+func restartCNCCPod(oc *exutil.CLI, clientset kubernetes.Interface) {
+	framework.Logf("Restarting CNCC pod by deleting it")
+	cnccPodLabel := exutil.ParseLabelsOrDie("app=cloud-network-config-controller")
+	cnccNamespace := "openshift-cloud-network-config-controller"
+	cnccPodNames, err := exutil.GetPodNamesByFilter(clientset.CoreV1().Pods(cnccNamespace), cnccPodLabel, exutil.CheckPodIsRunning)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(cnccPodNames).NotTo(o.BeEmpty())
+	_, err = runOcWithRetry(oc.AsAdmin(), "delete", "pod", cnccPodNames[0], "-n", cnccNamespace)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	framework.Logf("Waiting for the CNCC pod to become Ready again after restart")
+	_, err = exutil.WaitForPods(
+		clientset.CoreV1().Pods(cnccNamespace),
+		cnccPodLabel,
+		exutil.CheckPodIsRunning,
+		1,
+		2*time.Minute)
+	o.Expect(err).NotTo(o.HaveOccurred())
+}
+
+// countCloudPrivateIPConfigsByReason counts the number of CloudPrivateIPConfigs that have
+// a condition reason matching the specified reason (e.g., "CloudResponseError" or "CloudResponseSuccess").
+func countCloudPrivateIPConfigsByReason(oc *exutil.CLI, reason string) (int, error) {
+	output, err := runOcWithRetry(oc.AsAdmin(), "get", "cloudprivateipconfigs",
+		"-o", "custom-columns=NAME:.metadata.name,NODE:.spec.node,STATE:.status.conditions[].reason",
+		"--no-headers")
+	if err != nil {
+		return 0, fmt.Errorf("failed to get cloudprivateipconfigs: %v", err)
+	}
+
+	trimmedOutput := strings.TrimSpace(output)
+	if trimmedOutput == "" {
+		return 0, nil
+	}
+
+	lines := strings.Split(trimmedOutput, "\n")
+	count := 0
+	for _, line := range lines {
+		trimmedLine := strings.TrimSpace(line)
+		if trimmedLine != "" && strings.Contains(trimmedLine, reason) {
+			count++
+		}
+	}
+	return count, nil
+}
+
+// waitAllEgressIPsAssigned waits for all EgressIPs to have status.Items populated.
+func waitAllEgressIPsAssigned(oc *exutil.CLI, timeout time.Duration) error { // egressips
+	egressipList, err := listEgressIPs(oc)
+	if err != nil {
+		return err
+	}
+	for _, egressip := range egressipList.Items {
+		// Wait for status.Items to be populated
+		err = wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
+			eip, err := getEgressIP(oc, egressip.Name)
+			if err != nil {
+				return false, fmt.Errorf("failed to get EgressIP %s: %v", egressip.Name, err)
+			}
+			// Check that status is not empty
+			if len(eip.Status.Items) == 0 {
+				framework.Logf("EgressIP %s has empty status.Items, waiting...", egressip.Name)
+				return false, nil
+			}
+			// Check that each IP in spec has a corresponding entry in status.Items with node assigned
+			for _, ip := range egressip.Spec.EgressIPs {
+				foundInStatus := false
+				for _, statusItem := range eip.Status.Items {
+					if statusItem.EgressIP == ip {
+						if statusItem.Node == "" {
+							framework.Logf("EgressIP %s found in status but node is empty, waiting...", ip)
+							return false, nil
+						}
+						foundInStatus = true
+						break
+					}
+				}
+				if !foundInStatus {
+					framework.Logf("EgressIP %s not found in status.Items, waiting...", ip)
+					return false, nil
+				}
+			}
+			framework.Logf("EgressIP %s has all IPs assigned in status.Items", egressip.Name)
+			return true, nil
+		})
+		if err != nil {
+			return fmt.Errorf("EgressIP %s failed to appear in status.Items: %v", egressip.Name, err)
+		}
+	}
+
+	return nil
+}