Support per-cluster Incus client configuration

api/v1alpha1/incuscluster_types.go

@@ -40,6 +40,25 @@ type IncusClusterSpec struct {
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	// +optional
 	ControlPlaneEndpoint APIEndpoint `json:"controlPlaneEndpoint"`
+
+	// Incus is the configuration for the Incus API.
+	Incus Incus `json:"incus"`
+}
+
+type Incus struct {
+	// Endpoint is the API endpoint to reach the Incus API.
+	Endpoint string `json:"endpoint"`
+
+	// secretRef is a reference to the secret containing the credentials to access the Incus API.
+	SecretRef SecretReference `json:"secretRef"`
+
+	// Project is the project to use when interacting with the Incus API.
+	Project string `json:"project"`
+}
+
+type SecretReference struct {
+	// Name is the name of the secret.
+	Name string `json:"name"`
 }
 
 // APIEndpoint represents a reachable Kubernetes API endpoint.

cmd/main.go

@@ -20,15 +20,13 @@ import (
 	"context"
 	"crypto/tls"
 	"flag"
-	"net/http"
 	"os"
 	"os/signal"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
-	incusclient "github.com/lxc/incus/client"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -42,7 +40,6 @@ import (
 	infrav1alpha1 "github.com/miscord-dev/cluster-api-provider-incus/api/v1alpha1"
 	"github.com/miscord-dev/cluster-api-provider-incus/internal/controller"
 	"github.com/miscord-dev/cluster-api-provider-incus/pkg/incus"
-	"github.com/miscord-dev/cluster-api-provider-incus/pkg/transport"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	// +kubebuilder:scaffold:imports
 )
@@ -60,45 +57,6 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
-func incusConnection() func() (url string, args incusclient.ConnectionArgs) {
-	var url string
-	var args incusclient.ConnectionArgs
-	var oidcTokenFile string
-	flag.StringVar(&url, "incus-url", "http://localhost:8080", "The URL of the Incus server")
-	flag.BoolVar(&args.InsecureSkipVerify, "incus-insecure-skip-verify", false, "Skip SSL certificate verification")
-	flag.StringVar(&args.TLSCA, "incus-tls-ca", "", "The path to the CA certificate")
-	flag.StringVar(&args.TLSClientCert, "incus-tls-client-cert", "", "The path to the client certificate")
-	flag.StringVar(&args.TLSClientKey, "incus-tls-client-key", "", "The path to the client key")
-	flag.StringVar(&oidcTokenFile, "incus-oidc-token-file", "", "The path to the OIDC token file (Supports hot-reloading)")
-
-	return func() (string, incusclient.ConnectionArgs) {
-		if args.TLSCA != "" {
-			args.TLSCA = string(loadFile(args.TLSCA))
-		}
-		if args.TLSClientCert != "" {
-			args.TLSClientCert = string(loadFile(args.TLSClientCert))
-		}
-		if args.TLSClientKey != "" {
-			args.TLSClientKey = string(loadFile(args.TLSClientKey))
-		}
-		if oidcTokenFile != "" {
-			args.TransportWrapper = func(t *http.Transport) incusclient.HTTPTransporter {
-				return transport.NewTransport(t, oidcTokenFile, "Bearer")
-			}
-		}
-
-		return url, args
-	}
-}
-
-func loadFile(file string) []byte {
-	b, err := os.ReadFile(file)
-	if err != nil {
-		setupLog.Error(err, "failed to load file", "file", file)
-	}
-
-	return b
-}
 
 func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
@@ -121,7 +79,6 @@ func main() {
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
-	initIncus := incusConnection()
 
 	opts := zap.Options{
 		Development: true,
@@ -131,14 +88,6 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	incusURL, connectionArgs := initIncus()
-	incusServer, err := incusclient.ConnectIncus(incusURL, &connectionArgs)
-	if err != nil {
-		setupLog.Error(err, "failed to connect to Incus")
-		os.Exit(1)
-	}
-	incusClient := incus.NewClient(incusServer)
-
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -216,9 +165,9 @@ func main() {
 		os.Exit(1)
 	}
 	if err = (&controller.IncusMachineReconciler{
-		Client:      mgr.GetClient(),
-		Scheme:      mgr.GetScheme(),
-		IncusClient: incusClient,
+		Client:             mgr.GetClient(),
+		Scheme:             mgr.GetScheme(),
+		IncusClientFactory: incus.NewClientFactory(mgr.GetClient()),
 	}).SetupWithManager(ctx, mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "IncusMachine")
 		os.Exit(1)

components/metadata.yaml

@@ -0,0 +1,6 @@
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+kind: Metadata
+releaseSeries:
+- major: 0
+  minor: 0
+  contract: v1beta1

config/crd/bases/infrastructure.cluster.x-k8s.io_incusclusters.yaml

@@ -55,6 +55,33 @@ spec:
                 - host
                 - port
                 type: object
+              incus:
+                description: Incus is the configuration for the Incus API.
+                properties:
+                  endpoint:
+                    description: Endpoint is the API endpoint to reach the Incus API.
+                    type: string
+                  project:
+                    description: Project is the project to use when interacting with
+                      the Incus API.
+                    type: string
+                  secretRef:
+                    description: secretRef is a reference to the secret containing
+                      the credentials to access the Incus API.
+                    properties:
+                      name:
+                        description: Name is the name of the secret.
+                        type: string
+                    required:
+                    - name
+                    type: object
+                required:
+                - endpoint
+                - project
+                - secretRef
+                type: object
+            required:
+            - incus
             type: object
           status:
             description: IncusClusterStatus defines the observed state of IncusCluster.

config/crd/bases/infrastructure.cluster.x-k8s.io_incusclustertemplates.yaml

@@ -86,6 +86,34 @@ spec:
                         - host
                         - port
                         type: object
+                      incus:
+                        description: Incus is the configuration for the Incus API.
+                        properties:
+                          endpoint:
+                            description: Endpoint is the API endpoint to reach the
+                              Incus API.
+                            type: string
+                          project:
+                            description: Project is the project to use when interacting
+                              with the Incus API.
+                            type: string
+                          secretRef:
+                            description: secretRef is a reference to the secret containing
+                              the credentials to access the Incus API.
+                            properties:
+                              name:
+                                description: Name is the name of the secret.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                        required:
+                        - endpoint
+                        - project
+                        - secretRef
+                        type: object
+                    required:
+                    - incus
                     type: object
                 required:
                 - spec

config/dev/kustomization.yaml

@@ -0,0 +1,12 @@
+resources:
+- ../default
+
+patches:
+- path: patch_deployment.yaml
+
+images:
+- digest: sha256:7da441466a91e54c3b0a8994482251f7dd1fa83f8ec079bf786f9b4b7709a68d
+  name: controller
+  newName: ghcr.io/miscord-dev/cluster-api-provider-incus
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

config/dev/patch_deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-api-provider-incus-controller-manager
+  namespace: capincus-system
+spec:
+  template:
+    spec:
+      securityContext:
+        runAsNonRoot: false
+      initContainers:
+        - name: init-incus
+          image: ghcr.io/miscord-dev/dexsidecar:pr-2
+          imagePullPolicy: Always
+          restartPolicy: Always
+          env:
+            - name: dex_access_token_file
+              value: /var/run/secrets/miscord.win/dex/token
+            - name: dex_endpoint
+              value: "https://dex.tsuzu.dev/token"
+            - name: dex_basic_auth
+              value: "incus:"
+            - name: dex_connector_id
+              value: k3s
+            - name: dex_grant_type
+              value: urn:ietf:params:oauth:grant-type:token-exchange
+            - name: dex_scope
+              value: "openid federated_id"
+            - name: dex_requested_token_type
+              value: urn:ietf:params:oauth:token-type:access_token
+            - name: dex_file_subject_token
+              value: /var/run/secrets/kubernetes.io/dex/token
+            - name: dex_subject_token_type
+              value: urn:ietf:params:oauth:token-type:id_token
+          volumeMounts:
+            - name: incus-api-key
+              mountPath: /var/run/secrets/miscord.win/dex
+            - name: dex
+              mountPath: /var/run/secrets/kubernetes.io/dex
+      containers:
+        - name: manager
+          args:
+            - --leader-elect
+            - --health-probe-bind-address=:8081
+            - --incus-url=https://incus.tsuzu.dev:8443
+            - --incus-oidc-token-file=/var/run/secrets/miscord.win/dex/token
+          volumeMounts:
+            - name: incus-api-key
+              mountPath: /var/run/secrets/miscord.win/dex
+      volumes:
+        - name: incus-api-key
+          emptyDir: {}
+        - name: dex
+          projected:
+            defaultMode: 420
+            sources:
+            - serviceAccountToken:
+                audience: dex
+                expirationSeconds: 7200
+                path: token