diff --git a/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml b/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml index 957eebc..36bcdc4 100644 --- a/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml +++ b/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml @@ -26,6 +26,9 @@ spec: {{- with $domain := .Values.virtualClusterController.virtualKubernetesClusterDomain }} - --virtual-kubernetes-cluster-domain={{ $domain }} {{- end }} + {{- with $label := .Values.virtualClusterController.nodeSelectorLabel }}{{ if $label}} + - --node-selector-label={{ $label }} + {{- end }}{{- end }} ports: - name: prometheus containerPort: 8080 diff --git a/charts/kubernetes/values.yaml b/charts/kubernetes/values.yaml index 2701696..5677e0d 100644 --- a/charts/kubernetes/values.yaml +++ b/charts/kubernetes/values.yaml @@ -30,6 +30,8 @@ virtualClusterController: image: ~ # Sets the DDNS domain virtual clusters will be part of virtualKubernetesClusterDomain: ~ + # Tells the controller to use a nodeSelector when creating vClusters + nodeSelectorLabel: "" # Monitor specific configuration. monitor: diff --git a/pkg/provisioners/helmapplications/virtualcluster/provisioner.go b/pkg/provisioners/helmapplications/virtualcluster/provisioner.go index bba232a..601d3f6 100644 --- a/pkg/provisioners/helmapplications/virtualcluster/provisioner.go +++ b/pkg/provisioners/helmapplications/virtualcluster/provisioner.go @@ -48,14 +48,19 @@ func init() { metrics.Registry.MustRegister(durationMetric) } +type ProvisionerOptions struct { + Domain string + NodeSelectorLabel string +} + type Provisioner struct { - domain string + Options ProvisionerOptions } // New returns a new initialized provisioner object. -func New(getApplication application.GetterFunc, domain string) *application.Provisioner { +func New(getApplication application.GetterFunc, options ProvisionerOptions) *application.Provisioner { p := &Provisioner{ - domain: domain, + Options: options, } return application.New(getApplication).WithGenerator(p) @@ -84,7 +89,8 @@ func (p *Provisioner) Values(ctx context.Context, version unikornv1core.Semantic // and the cost is "what you use", we'll need to worry about billing, so it may // be prudent to add organization, project and cluster labels to pods. // We use SNI to demutiplex at the ingress to the correct vcluster instance. - hostname := p.ReleaseName(ctx) + "." + p.domain + releaseName := p.ReleaseName(ctx) + hostname := releaseName + "." + p.Options.Domain // Allow users to actually hit the cluster. ingress := map[string]any{ @@ -132,12 +138,21 @@ func (p *Provisioner) Values(ctx context.Context, version unikornv1core.Semantic "statefulSet": statefulSet, } + syncNodes := map[string]any{ + "enabled": true, + "clearImageStatus": true, + } + if nodeSelectorLabel := p.Options.NodeSelectorLabel; nodeSelectorLabel != "" { + syncNodes["selector"] = map[string]any{ + "labels": map[string]string{ + nodeSelectorLabel: releaseName, + }, + } + } + sync := map[string]any{ "fromHost": map[string]any{ - "nodes": map[string]any{ - "enabled": true, - "clearImageStatus": true, - }, + "nodes": syncNodes, "runtimeClasses": map[string]any{ "enabled": true, }, diff --git a/pkg/provisioners/managers/virtualcluster/provisioner.go b/pkg/provisioners/managers/virtualcluster/provisioner.go index ded43c2..affeac9 100644 --- a/pkg/provisioners/managers/virtualcluster/provisioner.go +++ b/pkg/provisioners/managers/virtualcluster/provisioner.go @@ -117,9 +117,8 @@ type Options struct { // we need to talk to identity to get a token, and then to region // to ensure cloud identities and networks are provisioned, as well // as deprovisioning them. - clientOptions coreclient.HTTPClientOptions - // domain vclusters should appear in. - domain string + clientOptions coreclient.HTTPClientOptions + provisionerOptions virtualcluster.ProvisionerOptions } func (o *Options) AddFlags(f *pflag.FlagSet) { @@ -135,7 +134,8 @@ func (o *Options) AddFlags(f *pflag.FlagSet) { o.regionOptions.AddFlags(f) o.clientOptions.AddFlags(f) - f.StringVar(&o.domain, "virtual-kubernetes-cluster-domain", "virtual-kubernetes.example.com", "DNS domain for vclusters to be hosts of.") + f.StringVar(&o.provisionerOptions.Domain, "virtual-kubernetes-cluster-domain", "virtual-kubernetes.example.com", "DNS domain for vclusters to be hosts of.") + f.StringVar(&o.provisionerOptions.NodeSelectorLabel, "node-selector-label", "", "Label to use for vCluster node selectors (with the value of the vcluster name).") } // Provisioner encapsulates control plane provisioning. @@ -245,7 +245,7 @@ func (p *Provisioner) getProvisioner(kubeconfig []byte) provisioners.Provisioner // from the workload pool. This information and the scheduling // stuff needs passing into the provisioner. provisioner := remoteCluster.ProvisionOn( - virtualcluster.New(apps.vCluster, p.options.domain).InNamespace(p.cluster.Name), + virtualcluster.New(apps.vCluster, p.options.provisionerOptions).InNamespace(p.cluster.Name), // NOTE: If you are using a unikorn-provisioned physical cluster as a region // then you'll end up with two remotes for the same thing, and the // secrets will alias (aka split brain), so override the secret name