Add --node-selector-label flag to VKC controller

When creating VirtualKubernetesClusters especially, it will sometimes
be useful to limit the nodes used by the vCluster, with a node
selector. The setting `fromHost.nodes.selector.labels` both limits
which nodes appear from inside the vcluster, and which nodes pods can
be scheduled on.

I've represented this as a flag `--node-selector-label` -- label
because the _value_ is going to be particular to the vcluster. It has
a counterpart in the Helm chart values.yaml.

Author: squaremo

charts/kubernetes/templates/virtualcluster-controller/deployment.yaml

@@ -26,6 +26,9 @@ spec:
         {{- with $domain := .Values.virtualClusterController.virtualKubernetesClusterDomain }}
         - --virtual-kubernetes-cluster-domain={{ $domain }}
         {{- end }}
+        {{- with $label := .Values.virtualClusterController.nodeSelectorLabel }}{{ if $label}}
+        - --node-selector-label={{ $label }}
+        {{- end }}{{- end }}
         ports:
         - name: prometheus
           containerPort: 8080

charts/kubernetes/values.yaml

@@ -30,6 +30,8 @@ virtualClusterController:
   image: ~
   # Sets the DDNS domain virtual clusters will be part of
   virtualKubernetesClusterDomain: ~
+  # Tells the controller to use a nodeSelector when creating vClusters
+  nodeSelectorLabel: ""
 
 # Monitor specific configuration.
 monitor:

pkg/provisioners/helmapplications/virtualcluster/provisioner.go

@@ -48,14 +48,19 @@ func init() {
 	metrics.Registry.MustRegister(durationMetric)
 }
 
+type ProvisionerOptions struct {
+	Domain            string
+	NodeSelectorLabel string
+}
+
 type Provisioner struct {
-	domain string
+	Options ProvisionerOptions
 }
 
 // New returns a new initialized provisioner object.
-func New(getApplication application.GetterFunc, domain string) *application.Provisioner {
+func New(getApplication application.GetterFunc, options ProvisionerOptions) *application.Provisioner {
 	p := &Provisioner{
-		domain: domain,
+		Options: options,
 	}
 
 	return application.New(getApplication).WithGenerator(p)
@@ -84,7 +89,8 @@ func (p *Provisioner) Values(ctx context.Context, version unikornv1core.Semantic
 	// and the cost is "what you use", we'll need to worry about billing, so it may
 	// be prudent to add organization, project and cluster labels to pods.
 	// We use SNI to demutiplex at the ingress to the correct vcluster instance.
-	hostname := p.ReleaseName(ctx) + "." + p.domain
+	releaseName := p.ReleaseName(ctx)
+	hostname := releaseName + "." + p.Options.Domain
 
 	// Allow users to actually hit the cluster.
 	ingress := map[string]any{
@@ -132,12 +138,21 @@ func (p *Provisioner) Values(ctx context.Context, version unikornv1core.Semantic
 		"statefulSet":  statefulSet,
 	}
 
+	syncNodes := map[string]any{
+		"enabled":          true,
+		"clearImageStatus": true,
+	}
+	if nodeSelectorLabel := p.Options.NodeSelectorLabel; nodeSelectorLabel != "" {
+		syncNodes["selector"] = map[string]any{
+			"labels": map[string]string{
+				nodeSelectorLabel: releaseName,
+			},
+		}
+	}
+
 	sync := map[string]any{
 		"fromHost": map[string]any{
-			"nodes": map[string]any{
-				"enabled":          true,
-				"clearImageStatus": true,
-			},
+			"nodes": syncNodes,
 			"runtimeClasses": map[string]any{
 				"enabled": true,
 			},

pkg/provisioners/managers/virtualcluster/provisioner.go

@@ -117,9 +117,8 @@ type Options struct {
 	// we need to talk to identity to get a token, and then to region
 	// to ensure cloud identities and networks are provisioned, as well
 	// as deprovisioning them.
-	clientOptions coreclient.HTTPClientOptions
-	// domain vclusters should appear in.
-	domain string
+	clientOptions      coreclient.HTTPClientOptions
+	provisionerOptions virtualcluster.ProvisionerOptions
 }
 
 func (o *Options) AddFlags(f *pflag.FlagSet) {
@@ -135,7 +134,8 @@ func (o *Options) AddFlags(f *pflag.FlagSet) {
 	o.regionOptions.AddFlags(f)
 	o.clientOptions.AddFlags(f)
 
-	f.StringVar(&o.domain, "virtual-kubernetes-cluster-domain", "virtual-kubernetes.example.com", "DNS domain for vclusters to be hosts of.")
+	f.StringVar(&o.provisionerOptions.Domain, "virtual-kubernetes-cluster-domain", "virtual-kubernetes.example.com", "DNS domain for vclusters to be hosts of.")
+	f.StringVar(&o.provisionerOptions.NodeSelectorLabel, "node-selector-label", "", "Label to use for vCluster node selectors (with the value of the vcluster name).")
 }
 
 // Provisioner encapsulates control plane provisioning.
@@ -245,7 +245,7 @@ func (p *Provisioner) getProvisioner(kubeconfig []byte) provisioners.Provisioner
 	// from the workload pool.  This information and the scheduling
 	// stuff needs passing into the provisioner.
 	provisioner := remoteCluster.ProvisionOn(
-		virtualcluster.New(apps.vCluster, p.options.domain).InNamespace(p.cluster.Name),
+		virtualcluster.New(apps.vCluster, p.options.provisionerOptions).InNamespace(p.cluster.Name),
 		// NOTE: If you are using a unikorn-provisioned physical cluster as a region
 		// then you'll end up with two remotes for the same thing, and the
 		// secrets will alias (aka split brain), so override the secret name