diff --git a/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml b/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml index 957eebc2..b53b11b2 100644 --- a/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml +++ b/charts/kubernetes/templates/virtualcluster-controller/deployment.yaml @@ -23,9 +23,15 @@ spec: {{- include "unikorn.region.flags" . | nindent 8 }} {{- include "unikorn.otlp.flags" . | nindent 8 }} {{- include "unikorn.mtls.flags" . | nindent 8 }} - {{- with $domain := .Values.virtualClusterController.virtualKubernetesClusterDomain }} + {{ with $domain := .Values.virtualClusterController.virtualKubernetesClusterDomain -}} - --virtual-kubernetes-cluster-domain={{ $domain }} {{- end }} + {{ with $label := .Values.virtualClusterController.nodeSelectorLabel }} + - --node-selector-label={{ $label }} + {{- end }} + {{ if .Values.virtualClusterController.nodeSelectorLabelIsPrefix -}} + - --node-selector-label-is-prefix + {{- end }} ports: - name: prometheus containerPort: 8080 diff --git a/charts/kubernetes/values.yaml b/charts/kubernetes/values.yaml index 2701696e..b73c0ecd 100644 --- a/charts/kubernetes/values.yaml +++ b/charts/kubernetes/values.yaml @@ -30,6 +30,9 @@ virtualClusterController: image: ~ # Sets the DDNS domain virtual clusters will be part of virtualKubernetesClusterDomain: ~ + # Tells the controller to use a nodeSelector when creating vClusters + nodeSelectorLabel: "" + nodeSelectorLabelIsPrefix: false # Monitor specific configuration. monitor: diff --git a/pkg/provisioners/helmapplications/virtualcluster/provisioner.go b/pkg/provisioners/helmapplications/virtualcluster/provisioner.go index bba232a6..323aabfc 100644 --- a/pkg/provisioners/helmapplications/virtualcluster/provisioner.go +++ b/pkg/provisioners/helmapplications/virtualcluster/provisioner.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/pflag" unikornv1core "github.com/unikorn-cloud/core/pkg/apis/unikorn/v1alpha1" "github.com/unikorn-cloud/core/pkg/provisioners/application" @@ -48,14 +49,45 @@ func init() { metrics.Registry.MustRegister(durationMetric) } +type ProvisionerOptions struct { + Domain string + NodeSelectorLabel string + // if true, then instead of making a label `foo: ` for the selector, + // make `foo/: ""` (assuming the NodeSelectorLabel is `foo/`) + NodeSelectorLabelIsPrefix bool +} + +func (opts *ProvisionerOptions) AddFlags(f *pflag.FlagSet) { + f.StringVar(&opts.Domain, "virtual-kubernetes-cluster-domain", "virtual-kubernetes.example.com", "DNS domain for vclusters to be hosts of.") + f.StringVar(&opts.NodeSelectorLabel, "node-selector-label", "", "Label to use for vCluster node selectors (will be given the value of the vcluster name, in the selector).") + f.BoolVar(&opts.NodeSelectorLabelIsPrefix, "node-selector-label-is-prefix", false, `If set, the node selector label will be the vcluster name appended to --node-selector-label after a '/', and the value an empty string`) +} + +// NodeSelector creates a `MatchLabels`-style map for supplying to the vcluster chart, based +// on the options given. This is used to restrict the nodes that will be available to the vcluster. +// `vclusterName` is any value that identifies the vcluster in question. +func (opts *ProvisionerOptions) NodeSelector(vclusterName string) map[string]string { + var selector map[string]string + if nodeSelectorLabel := opts.NodeSelectorLabel; nodeSelectorLabel != "" { + selector = map[string]string{} + if opts.NodeSelectorLabelIsPrefix { + selector[nodeSelectorLabel+"/"+vclusterName] = "" + } else { + selector[nodeSelectorLabel] = vclusterName + } + } + + return selector +} + type Provisioner struct { - domain string + Options ProvisionerOptions } // New returns a new initialized provisioner object. -func New(getApplication application.GetterFunc, domain string) *application.Provisioner { +func New(getApplication application.GetterFunc, options ProvisionerOptions) *application.Provisioner { p := &Provisioner{ - domain: domain, + Options: options, } return application.New(getApplication).WithGenerator(p) @@ -84,7 +116,8 @@ func (p *Provisioner) Values(ctx context.Context, version unikornv1core.Semantic // and the cost is "what you use", we'll need to worry about billing, so it may // be prudent to add organization, project and cluster labels to pods. // We use SNI to demutiplex at the ingress to the correct vcluster instance. - hostname := p.ReleaseName(ctx) + "." + p.domain + releaseName := p.ReleaseName(ctx) + hostname := releaseName + "." + p.Options.Domain // Allow users to actually hit the cluster. ingress := map[string]any{ @@ -132,12 +165,20 @@ func (p *Provisioner) Values(ctx context.Context, version unikornv1core.Semantic "statefulSet": statefulSet, } + syncNodes := map[string]any{ + "enabled": true, + "clearImageStatus": true, + } + + // Supply a node selector to the vcluster if the options say to use one. The release name is + // used as the vcluster name. + if selector := p.Options.NodeSelector(releaseName); selector != nil { + syncNodes["selector"] = selector + } + sync := map[string]any{ "fromHost": map[string]any{ - "nodes": map[string]any{ - "enabled": true, - "clearImageStatus": true, - }, + "nodes": syncNodes, "runtimeClasses": map[string]any{ "enabled": true, }, diff --git a/pkg/provisioners/managers/virtualcluster/provisioner.go b/pkg/provisioners/managers/virtualcluster/provisioner.go index ded43c2d..663940a2 100644 --- a/pkg/provisioners/managers/virtualcluster/provisioner.go +++ b/pkg/provisioners/managers/virtualcluster/provisioner.go @@ -117,9 +117,8 @@ type Options struct { // we need to talk to identity to get a token, and then to region // to ensure cloud identities and networks are provisioned, as well // as deprovisioning them. - clientOptions coreclient.HTTPClientOptions - // domain vclusters should appear in. - domain string + clientOptions coreclient.HTTPClientOptions + provisionerOptions virtualcluster.ProvisionerOptions } func (o *Options) AddFlags(f *pflag.FlagSet) { @@ -134,8 +133,7 @@ func (o *Options) AddFlags(f *pflag.FlagSet) { o.identityOptions.AddFlags(f) o.regionOptions.AddFlags(f) o.clientOptions.AddFlags(f) - - f.StringVar(&o.domain, "virtual-kubernetes-cluster-domain", "virtual-kubernetes.example.com", "DNS domain for vclusters to be hosts of.") + o.provisionerOptions.AddFlags(f) } // Provisioner encapsulates control plane provisioning. @@ -245,7 +243,7 @@ func (p *Provisioner) getProvisioner(kubeconfig []byte) provisioners.Provisioner // from the workload pool. This information and the scheduling // stuff needs passing into the provisioner. provisioner := remoteCluster.ProvisionOn( - virtualcluster.New(apps.vCluster, p.options.domain).InNamespace(p.cluster.Name), + virtualcluster.New(apps.vCluster, p.options.provisionerOptions).InNamespace(p.cluster.Name), // NOTE: If you are using a unikorn-provisioned physical cluster as a region // then you'll end up with two remotes for the same thing, and the // secrets will alias (aka split brain), so override the secret name