kubernetes: add multi-node test and docs

Signed-off-by: Akihiro Suda <[email protected]>

Author: AkihiroSuda

hack/bats/extras/k8s.bats

@@ -26,11 +26,30 @@ get_num_nodes() {
 
 local_setup() {
     local nodes=$(get_num_nodes)
+    local join_command=""
     for ((i=0; i<nodes; i++)); do
         limactl delete --force "${NAME}-$i" || :
-        limactl start --tty=false --name "${NAME}-$i" "template:${TEMPLATE}" 3>&- 4>&-
-        # NOTE: No support for multi-node clusters yet.
+        local limactl_start_flags="--tty=false --name "${NAME}-$i""
+        # Multi-node setup requires user-v2 network for VM-to-VM communication
+        if [[ $nodes -gt 1 ]]; then
+            limactl_start_flags+=" --network lima:user-v2"
+        fi
+        limactl start ${limactl_start_flags} "template:${TEMPLATE}" 3>&- 4>&- &
     done
+    wait $(jobs -p)
+    # Multi-node setup
+    if [[ $nodes -gt 1 ]]; then
+        for ((i=0; i<nodes; i++)); do
+            if [[ $i -eq 0 ]]; then
+                # Get the join command from the first node
+                join_command=$(limactl shell "${NAME}-0" sudo kubeadm token create --print-join-command)
+            else
+                # Execute the join command on worker nodes
+                limactl shell "${NAME}-$i" sudo bash -euxc "kubeadm reset --force ; ip link delete cni0 ; ip link delete flannel.1 ; rm -rf /var/lib/cni /etc/cni"
+                limactl shell "${NAME}-$i" sudo ${join_command}
+            fi
+        done
+    fi
     for node in $(k get node -o name); do
 	    k wait --timeout=5m --for=condition=ready "${node}"
     done
@@ -81,4 +100,54 @@ k() {
     done
 }
 
-# TODO: add a test for multi-node
+# bats test_tags=nodes:3
+@test 'Multi-node' {
+    # Based on https://github.com/rootless-containers/usernetes/blob/gen2-v20250828.0/hack/test-smoke.sh
+    k apply -f - <<EOF
+apiVersion: v1
+kind: Service
+metadata:
+  name: dnstest
+  labels:
+    run: dnstest
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 80
+  selector:
+    run: dnstest
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: dnstest
+spec:
+  serviceName: dnstest
+  selector:
+    matchLabels:
+      run: dnstest
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        run: dnstest
+    spec:
+      containers:
+      - name: dnstest
+        image: ${TEST_CONTAINER_IMAGES[nginx]}
+        ports:
+        - containerPort: 80
+EOF
+    k rollout status --timeout=5m statefulset/dnstest || {
+        k describe pods -l run=dnstest
+        false
+    }
+    # --rm requires -i
+    k run -i --rm --image=${TEST_CONTAINER_IMAGES[nginx]} --restart=Never dnstest-shell -- sh -exc 'for f in $(seq 0 2); do wget -O- http://dnstest-${f}.dnstest.default.svc.cluster.local; done'
+    k delete service dnstest
+    k delete statefulset dnstest
+}

templates/k8s.yaml

@@ -18,7 +18,7 @@
 # (The join command printed here)
 #
 # $ limactl start --name k8s-1 --network lima:user-v2 template:k8s
-# $ limactl shell k8s-1 sudo kubeadm reset --force
+# $ limactl shell k8s-1 sudo bash -euxc "kubeadm reset --force ; ip link delete cni0 ; ip link delete flannel.1 ; rm -rf /var/lib/cni /etc/cni"
 # $ limactl shell k8s-1 sudo <JOIN_COMMAND_FROM_ABOVE>
 
 minimumLimaVersion: 2.0.0

website/content/en/docs/examples/containers/kubernetes/_index.md

@@ -56,10 +56,16 @@ See also <https://github.com/rootless-containers/usernetes>.
 
 A multi-node cluster can be created by creating multiple VMs connected via the [`lima:user-v2`](../../../config/network/user-v2.md) network.
 
+As of Lima v2.0, the built-in `k8s` template is designed to support multi-node mode.
+
 ```bash
-limactl create --name k8s-0 --network lima:user-v2
-limactl create --name k8s-1 --network lima:user-v2
+limactl start --name k8s-0 --network lima:user-v2 template:k8s
+limactl shell k8s-0 sudo kubeadm token create --print-join-command
+# (The join command printed here)
 ```
 
-The cluster has to be set up manually, as the built-in templates do not support multi-node mode yet.
-Support for multi-node template is tracked in <https://github.com/lima-vm/lima/issues/4100>.
+```bash
+limactl start --name k8s-1 --network lima:user-v2 template:k8s
+limactl shell k8s-1 sudo bash -euxc "kubeadm reset --force ; ip link delete cni0 ; ip link delete flannel.1 ; rm -rf /var/lib/cni /etc/cni"
+limactl shell k8s-1 sudo <JOIN_COMMAND_FROM_ABOVE>
+```