Skip to content

Consolidate test util functions #13281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dereknola wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Consolidate test util functions #13281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f66ceed
Consolidate RunCommand between Docker and E2E tests
dereknola Nov 26, 2025
9fb77c8
Fix naming convention for docker test import
dereknola Nov 26, 2025
420227a
Move from ranchertest to mirrored-busybox
dereknola Dec 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions tests/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package tests
import (
"context"
"fmt"
"os"
"os/exec"
"strings"

corev1 "k8s.io/api/core/v1"
Expand All @@ -15,6 +17,21 @@ import (
// This file consolidates functions that are used across multiple testing frameworks.
// Most of it relates to interacting with the Kubernetes API and checking the status of resources.

func RunCommand(cmd string) (string, error) {
c := exec.Command("bash", "-c", cmd)
if kc, ok := os.LookupEnv("E2E_KUBECONFIG"); ok {
c.Env = append(os.Environ(), "KUBECONFIG="+kc)
}
if kc, ok := os.LookupEnv("DOCKER_KUBECONFIG"); ok {
c.Env = append(os.Environ(), "KUBECONFIG="+kc)
}
out, err := c.CombinedOutput()
if err != nil {
return string(out), fmt.Errorf("failed to run command: %s, %v", cmd, err)
}
return string(out), err
}

// CheckDefaultDeployments checks if the standard array of K3s deployments are ready, otherwise returns an error
func CheckDefaultDeployments(kubeconfigFile string) error {
return CheckDeployments(kubeconfigFile, "kube-system", "coredns", "local-path-provisioner", "metrics-server", "traefik")
Expand Down
15 changes: 7 additions & 8 deletions tests/docker/basics/basics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@ import (

"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/docker"
tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var k3sImage = flag.String("k3sImage", "", "The image used to provision containers")
var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
var config *tester.TestConfig
var config *docker.TestConfig

func Test_DockerBasic(t *testing.T) {
flag.Parse()
Expand All @@ -29,7 +28,7 @@ var _ = Describe("Basic Tests", Ordered, func() {
Context("Setup Cluster", func() {
It("should provision servers and agents", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config, err = docker.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
Expect(config.ProvisionServers(1)).To(Succeed())
Expect(config.ProvisionAgents(1)).To(Succeed())
Expand Down Expand Up @@ -57,9 +56,9 @@ var _ = Describe("Basic Tests", Ordered, func() {
Context("Verify Binaries and Images", func() {
It("has valid bundled binaries", func() {
for _, server := range config.Servers {
Expect(tester.VerifyValidVersion(server, "kubectl")).To(Succeed())
Expect(tester.VerifyValidVersion(server, "ctr")).To(Succeed())
Expect(tester.VerifyValidVersion(server, "crictl")).To(Succeed())
Expect(docker.VerifyValidVersion(server, "kubectl")).To(Succeed())
Expect(docker.VerifyValidVersion(server, "ctr")).To(Succeed())
Expect(docker.VerifyValidVersion(server, "crictl")).To(Succeed())
}
})
It("has valid airgap images", func() {
Expand Down Expand Up @@ -87,7 +86,7 @@ var _ = AfterSuite(func() {
})

// VerifyAirgapImages checks for changes in the airgap image list
func VerifyAirgapImages(config *tester.TestConfig) error {
func VerifyAirgapImages(config *docker.TestConfig) error {
// This file is generated during the build packaging step
const airgapImageList = "../../../scripts/airgap/image-list.txt"

Expand All @@ -97,7 +96,7 @@ func VerifyAirgapImages(config *tester.TestConfig) error {
// Collect all images from nodes
for _, node := range config.GetNodeNames() {
cmd := fmt.Sprintf("docker exec %s crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)'", node)
output, err := tester.RunCommand(cmd)
output, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed to execute crictl and jq: %v", err)

for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
Expand Down
5 changes: 2 additions & 3 deletions tests/docker/bootstraptoken/bootstraptoken_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@ import (

"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/docker"
tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
var config *tester.TestConfig
var config *docker.TestConfig

func Test_DockerBootstrapToken(t *testing.T) {
flag.Parse()
Expand All @@ -27,7 +26,7 @@ var _ = Describe("Boostrap Token Tests", Ordered, func() {
Context("Setup Cluster", func() {
It("should provision servers", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config, err = docker.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
Expect(config.ProvisionServers(1)).To(Succeed())
Eventually(func() error {
Expand Down
19 changes: 9 additions & 10 deletions tests/docker/cacerts/cacerts_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,13 @@ import (

"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/docker"
tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
var config *tester.TestConfig
var config *docker.TestConfig
var testID string

func Test_DockerCACerts(t *testing.T) {
Expand All @@ -36,7 +35,7 @@ var _ = Describe("CA Certs Tests", Ordered, func() {
// share it with the other containers that need the file.
It("should configure CA certs", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config, err = docker.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
Expect(os.MkdirAll(filepath.Join(config.TestDir, "pause"), 0755)).To(Succeed())

Expand All @@ -45,16 +44,16 @@ var _ = Describe("CA Certs Tests", Ordered, func() {
tlsMount := fmt.Sprintf("--mount type=volume,src=%s,dst=/var/lib/rancher/k3s/server/tls", pauseName)
cmd := fmt.Sprintf("docker run -d --name %s --hostname %s %s rancher/mirrored-pause:3.6",
pauseName, pauseName, tlsMount)
_, err = tester.RunCommand(cmd)
_, err = tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())

dataDir := filepath.Join(config.TestDir, "pause/k3s")
cmd = fmt.Sprintf("DATA_DIR=%s ../../../contrib/util/generate-custom-ca-certs.sh", dataDir)
_, err = tester.RunCommand(cmd)
_, err = tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())

cmd = fmt.Sprintf("docker cp %s %s:/var/lib/rancher", dataDir, pauseName)
_, err = tester.RunCommand(cmd)
_, err = tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())

// Set SERVER_ARGS to include the custom CA certs
Expand All @@ -76,7 +75,7 @@ var _ = Describe("CA Certs Tests", Ordered, func() {
// Example: Check if the custom CA certs are present in the server container
for _, server := range config.Servers {
cmd := fmt.Sprintf("docker exec %s ls /var/lib/rancher/k3s/server/tls", server.Name)
output, err := tester.RunCommand(cmd)
output, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed to list custom CA certs: %v", err)
Expect(output).To(ContainSubstring("ca.crt"))
}
Expand All @@ -98,13 +97,13 @@ var _ = AfterSuite(func() {
if config != nil && !failed {
config.Cleanup()
cmd := fmt.Sprintf("docker stop k3s-pause-%s", testID)
_, err := tester.RunCommand(cmd)
_, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
cmd = fmt.Sprintf("docker rm -v k3s-pause-%s", testID)
_, err = tester.RunCommand(cmd)
_, err = tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
cmd = fmt.Sprintf("docker volume rm k3s-pause-%s", testID)
_, err = tester.RunCommand(cmd)
_, err = tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
}

Expand Down
11 changes: 5 additions & 6 deletions tests/docker/conformance/conformance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (

"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/docker"
tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
Expand All @@ -22,7 +21,7 @@ var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision cont
var db = flag.String("db", "", "The database to use for the tests (sqlite, etcd, mysql, postgres)")
var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests")
var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
var config *tester.TestConfig
var config *docker.TestConfig

func Test_DockerConformance(t *testing.T) {
flag.Parse()
Expand All @@ -35,7 +34,7 @@ var _ = Describe("Conformance Tests", Ordered, func() {
Context("Setup Cluster", func() {
It("should provision servers and agents", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config, err = docker.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
config.DBType = *db
Expect(config.ProvisionServers(1)).To(Succeed())
Expand All @@ -58,7 +57,7 @@ var _ = Describe("Conformance Tests", Ordered, func() {
hydrophoneURL := fmt.Sprintf("https://github.com/kubernetes-sigs/hydrophone/releases/download/%s/hydrophone_Linux_%s.tar.gz",
hydrophoneVersion, hydrophoneArch)
cmd := fmt.Sprintf("curl -L %s | tar -xzf - -C %s", hydrophoneURL, config.TestDir)
_, err := tester.RunCommand(cmd)
_, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
Expect(os.Chmod(filepath.Join(config.TestDir, "hydrophone"), 0755)).To(Succeed())
})
Expand All @@ -84,7 +83,7 @@ var _ = Describe("Conformance Tests", Ordered, func() {
if hc.ProcessState != nil {
break
}
res, _ := tester.RunCommand(cmd)
res, _ := tests.RunCommand(cmd)
res = strings.TrimSpace(res)
fmt.Printf("Status Report %d: %s tests complete\n", i, res)
}
Expand All @@ -109,7 +108,7 @@ var _ = Describe("Conformance Tests", Ordered, func() {
break
}
time.Sleep(120 * time.Second)
res, _ := tester.RunCommand(cmd)
res, _ := tests.RunCommand(cmd)
res = strings.TrimSpace(res)
fmt.Printf("Status Report %d: %s tests complete\n", i, res)
}
Expand Down
10 changes: 5 additions & 5 deletions tests/docker/dualstack/dualstack_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ var _ = DescribeTableSubtree("DualStack Tests", Ordered, func(ipConfig string) {
Expect(err).NotTo(HaveOccurred())
Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
return docker.RunCommand(cmd)
return tests.RunCommand(cmd)
}, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod"))

// Checks both IPv4 and IPv6
Expand All @@ -100,7 +100,7 @@ var _ = DescribeTableSubtree("DualStack Tests", Ordered, func(ipConfig string) {
_, err := tc.DeployWorkload("dualstack_ingress.yaml")
Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
cmd := "kubectl get ingress ds-ingress --kubeconfig=" + tc.KubeconfigFile + " -o jsonpath=\"{.spec.rules[*].host}\""
hostName, err := docker.RunCommand(cmd)
hostName, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
for _, node := range append(tc.Servers, tc.Agents...) {
ips, err := tests.GetNodeIPs(node.Name, tc.KubeconfigFile)
Expand All @@ -112,7 +112,7 @@ var _ = DescribeTableSubtree("DualStack Tests", Ordered, func(ipConfig string) {
}
cmd := fmt.Sprintf("curl --header host:%s http://%s/name.html", hostName, ip)
Eventually(func() (string, error) {
return docker.RunCommand(cmd)
return tests.RunCommand(cmd)
}, "10s", "2s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd)
}
}
Expand All @@ -122,7 +122,7 @@ var _ = DescribeTableSubtree("DualStack Tests", Ordered, func(ipConfig string) {
_, err := tc.DeployWorkload("dualstack_nodeport.yaml")
Expect(err).NotTo(HaveOccurred())
cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + tc.KubeconfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := docker.RunCommand(cmd)
nodeport, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
for _, node := range append(tc.Servers, tc.Agents...) {
ips, err := tests.GetNodeIPs(node.Name, tc.KubeconfigFile)
Expand All @@ -134,7 +134,7 @@ var _ = DescribeTableSubtree("DualStack Tests", Ordered, func(ipConfig string) {
}
cmd = "curl -L --insecure http://" + ip + ":" + nodeport + "/name.html"
Eventually(func() (string, error) {
return docker.RunCommand(cmd)
return tests.RunCommand(cmd)
}, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
}
}
Expand Down
7 changes: 3 additions & 4 deletions tests/docker/etcd/etcd_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@ import (

"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/docker"
tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
var ci = flag.Bool("ci", false, "running on CI, forced cleanup")
var config *tester.TestConfig
var config *docker.TestConfig

func Test_DockerEtcd(t *testing.T) {
flag.Parse()
Expand All @@ -27,7 +26,7 @@ var _ = Describe("Etcd Tests", Ordered, func() {
Context("Test a 3 server cluster", func() {
It("should setup the cluster configuration", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config, err = docker.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
})
It("should provision servers", func() {
Expand All @@ -47,7 +46,7 @@ var _ = Describe("Etcd Tests", Ordered, func() {
Context("Test a Split Role cluster with 3 etcd, 2 control-plane, 1 agents", func() {
It("should setup the cluster configuration", func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
config, err = docker.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
Expect(os.Setenv("SERVER_0_ARGS", "--disable-apiserver --disable-controller-manager --disable-scheduler --cluster-init")).To(Succeed())
Expect(os.Setenv("SERVER_1_ARGS", "--disable-apiserver --disable-controller-manager --disable-scheduler")).To(Succeed())
Expand Down
16 changes: 8 additions & 8 deletions tests/docker/hardened/hardened_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ kubelet-arg:

for _, server := range config.Servers {
cmd := "docker cp ./cluster-level-pss.yaml " + server.Name + ":/tmp/cluster-level-pss.yaml"
Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
Expect(tests.RunCommand(cmd)).Error().NotTo(HaveOccurred())

cmd = "mkdir -p /var/lib/rancher/k3s/server/logs"
Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
Expand All @@ -81,8 +81,8 @@ kubelet-arg:
_, err := config.DeployWorkload("hardened-ingress.yaml")
Expect(err).NotTo(HaveOccurred())
Eventually(func() (string, error) {
cmd := "kubectl get daemonset -n default example -o jsonpath='{.status.numberReady}' --kubeconfig=" + config.KubeconfigFile
return docker.RunCommand(cmd)
cmd := "kubectl get daemonset -n default example -o jsonpath='{.status.numberReady}'"
return tests.RunCommand(cmd)
}, "60s", "5s").Should(Equal("2"))
_, err = config.DeployWorkload("hardened-netpool.yaml")
Expect(err).NotTo(HaveOccurred())
Expand All @@ -92,23 +92,23 @@ kubelet-arg:
Eventually(func(g Gomega) {
for _, server := range config.Servers {
cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, server.IP)
g.Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
g.Expect(tests.RunCommand(cmd)).Error().NotTo(HaveOccurred())
}
for _, agent := range config.Agents {
cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, agent.IP)
g.Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
g.Expect(tests.RunCommand(cmd)).Error().NotTo(HaveOccurred())
}
}, "30s", "10s").Should(Succeed())
}
})
It("confirms we can make a request through the nodeport service", func() {
for _, server := range config.Servers {
cmd := "kubectl get service/example -o 'jsonpath={.spec.ports[*].nodePort}' --kubeconfig=" + config.KubeconfigFile
ports, err := docker.RunCommand(cmd)
cmd := "kubectl get service/example -o 'jsonpath={.spec.ports[*].nodePort}'"
ports, err := tests.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
for _, port := range strings.Split(ports, " ") {
cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' http://%s:%s", server.IP, port)
Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
Expect(tests.RunCommand(cmd)).Error().NotTo(HaveOccurred())
}
}
})
Expand Down
Loading