diff --git a/ray-operator/apis/config/v1alpha1/configuration_types.go b/ray-operator/apis/config/v1alpha1/configuration_types.go index 975521242b9..7ed0ce1baae 100644 --- a/ray-operator/apis/config/v1alpha1/configuration_types.go +++ b/ray-operator/apis/config/v1alpha1/configuration_types.go @@ -2,6 +2,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -73,6 +74,17 @@ type Configuration struct { // EnableMetrics indicates whether KubeRay operator should emit control plane metrics. EnableMetrics bool `json:"enableMetrics,omitempty"` + + // Host used for Ray Dashboard ingresses. The host will be the same for all `RayClusters` and they + // will be differentiated by their paths. + IngressHost string `json:"ingressHost,omitempty"` + + // TLS configuration for the Ray Dashboard ingresses. Applies to all `RayClusters`. + IngressTLS []networkingv1.IngressTLS `json:"ingressTLS,omitempty"` + + // Default annotations for the Ray Dashboard ingresses. Annotations on the `RayCluster` will override + // these on a case-by-case basis. + IngressAnnotations map[string]string `json:"ingressAnnotations,omitempty"` } func (config Configuration) GetDashboardClient(mgr manager.Manager) func() utils.RayDashboardClientInterface { diff --git a/ray-operator/apis/config/v1alpha1/zz_generated.deepcopy.go b/ray-operator/apis/config/v1alpha1/zz_generated.deepcopy.go index b237fee3554..5586b2c5af0 100644 --- a/ray-operator/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/ray-operator/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -32,6 +33,20 @@ func (in *Configuration) DeepCopyInto(out *Configuration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.IngressTLS != nil { + in, out := &in.IngressTLS, &out.IngressTLS + *out = make([]networkingv1.IngressTLS, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.IngressAnnotations != nil { + in, out := &in.IngressAnnotations, &out.IngressAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. diff --git a/ray-operator/controllers/ray/common/ingress.go b/ray-operator/controllers/ray/common/ingress.go index ef71c3f2701..32e47ea0867 100644 --- a/ray-operator/controllers/ray/common/ingress.go +++ b/ray-operator/controllers/ray/common/ingress.go @@ -15,7 +15,7 @@ const IngressClassAnnotationKey = "kubernetes.io/ingress.class" // BuildIngressForHeadService Builds the ingress for head service dashboard. // This is used to expose dashboard for external traffic. -func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) (*networkingv1.Ingress, error) { +func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster, host string, ingressTLS []networkingv1.IngressTLS, defaultAnnotations map[string]string) (*networkingv1.Ingress, error) { log := ctrl.LoggerFrom(ctx) labels := map[string]string{ @@ -32,10 +32,16 @@ func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) ( excludeSet := map[string]struct{}{ IngressClassAnnotationKey: {}, } - annotation := map[string]string{} + annotations := map[string]string{} + for key, value := range defaultAnnotations { + if _, ok := excludeSet[key]; !ok { + annotations[key] = value + } + } + // cluster.Annotations takes precedence, so we add these after defaultAnnotations. for key, value := range cluster.Annotations { if _, ok := excludeSet[key]; !ok { - annotation[key] = value + annotations[key] = value } } @@ -70,11 +76,13 @@ func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) ( Name: utils.GenerateIngressName(cluster.Name), Namespace: cluster.Namespace, Labels: labels, - Annotations: annotation, + Annotations: annotations, }, Spec: networkingv1.IngressSpec{ + TLS: ingressTLS, Rules: []networkingv1.IngressRule{ { + Host: host, IngressRuleValue: networkingv1.IngressRuleValue{ HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: paths, @@ -85,8 +93,13 @@ func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) ( }, } - // Get ingress class name from rayCluster annotations. this is a required field to use ingress. - if ingressClassName, ok := cluster.Annotations[IngressClassAnnotationKey]; !ok { + // First try to get ingress class name from rayCluster annotations. + ingressClassName, ok := cluster.Annotations[IngressClassAnnotationKey] + if !ok { + ingressClassName, ok = defaultAnnotations[IngressClassAnnotationKey] + } + + if !ok { log.Info("Ingress class annotation is not set for the cluster.", "clusterNamespace", cluster.Namespace, "clusterName", cluster.Name) } else { // TODO: in AWS EKS, set up IngressClassName will cause an error due to conflict with annotation. diff --git a/ray-operator/controllers/ray/common/ingress_test.go b/ray-operator/controllers/ray/common/ingress_test.go index c4fc70b8c0f..3890d192c9e 100644 --- a/ray-operator/controllers/ray/common/ingress_test.go +++ b/ray-operator/controllers/ray/common/ingress_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -19,6 +20,8 @@ var instanceWithIngressEnabled = &rayv1.RayCluster{ Namespace: "default", Annotations: map[string]string{ IngressClassAnnotationKey: "nginx", + "annotation0": "value", + "annotation1": "value", }, }, Spec: rayv1.RayClusterSpec{ @@ -74,15 +77,18 @@ var instanceWithIngressEnabledWithoutIngressClass = &rayv1.RayCluster{ // only throw warning message and rely on Kubernetes to assign default ingress class func TestBuildIngressForHeadServiceWithoutIngressClass(t *testing.T) { - ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabledWithoutIngressClass) + ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabledWithoutIngressClass, "", []networkingv1.IngressTLS{}, map[string]string{}) assert.NotNil(t, ingress) require.NoError(t, err) } func TestBuildIngressForHeadService(t *testing.T) { - ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabled) + ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabled, "", []networkingv1.IngressTLS{}, map[string]string{}) require.NoError(t, err) + // annotations count + assert.Len(t, ingress.Annotations, 2) + // check ingress.class annotation assert.Equal(t, instanceWithIngressEnabled.Name, ingress.Labels[utils.RayClusterLabelKey]) @@ -107,4 +113,45 @@ func TestBuildIngressForHeadService(t *testing.T) { for _, path := range paths { assert.Equal(t, headSvcName, path.Backend.Service.Name) } + + // check host + assert.Equal(t, ingress.Spec.Rules[0].Host, "") + + // tls count + assert.Len(t, ingress.Spec.TLS, 0) +} + +func TestBuildIngressForHeadServiceWithControllerConfigs(t *testing.T) { + host := "ray.example.com" + tls := []networkingv1.IngressTLS{ + { + Hosts: []string{host}, + SecretName: "ray-tls-secret", + }, + } + ingressClass := "different-ingress-class" + annotations := map[string]string{"annotation0": "value2", "annotation1": "value2", IngressClassAnnotationKey: ingressClass} + ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabledWithoutIngressClass, host, tls, annotations) + require.NoError(t, err) + + assert.Equal(t, *ingress.Spec.IngressClassName, ingressClass) + assert.Equal(t, ingress.Annotations, map[string]string{ + "annotation0": "value2", "annotation1": "value2", + }) + assert.Equal(t, ingress.Spec.Rules[0].Host, host) + assert.Equal(t, ingress.Spec.TLS, tls) +} + +func TestBuildIngressForHeadServiceClusterSpecificAnnotationsTakePrecedence(t *testing.T) { + annotations := map[string]string{"annotation0": "value2", "annotation2": "value2", IngressClassAnnotationKey: "different-ingress-class"} + ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabled, "", []networkingv1.IngressTLS{}, annotations) + require.NoError(t, err) + + delete(annotations, IngressClassAnnotationKey) + assert.Equal(t, ingress.Annotations, map[string]string{ + "annotation0": "value", // Overridden by cluster annotation + "annotation1": "value", + "annotation2": "value2", + }) + assert.Equal(t, instanceWithIngressEnabled.Annotations[IngressClassAnnotationKey], *ingress.Spec.IngressClassName) } diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go index 92f10afe86d..1f8f73dbc79 100644 --- a/ray-operator/controllers/ray/raycluster_controller.go +++ b/ray-operator/controllers/ray/raycluster_controller.go @@ -96,6 +96,9 @@ type RayClusterReconcilerOptions struct { HeadSidecarContainers []corev1.Container WorkerSidecarContainers []corev1.Container IsOpenShift bool + IngressHost string + IngressTLS []networkingv1.IngressTLS + IngressAnnotations map[string]string } // Reconcile reads that state of the cluster for a RayCluster object and makes changes based on it @@ -478,7 +481,7 @@ func (r *RayClusterReconciler) reconcileIngressKubernetes(ctx context.Context, i } if len(headIngresses.Items) == 0 { - ingress, err := common.BuildIngressForHeadService(ctx, *instance) + ingress, err := common.BuildIngressForHeadService(ctx, *instance, r.options.IngressHost, r.options.IngressTLS, r.options.IngressAnnotations) if err != nil { return err } diff --git a/ray-operator/main.go b/ray-operator/main.go index 1f1aa717c0b..02b49b80f4d 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -251,6 +251,9 @@ func main() { WorkerSidecarContainers: config.WorkerSidecarContainers, IsOpenShift: utils.GetClusterType(), RayClusterMetricsManager: rayClusterMetricsManager, + IngressHost: config.IngressHost, + IngressTLS: config.IngressTLS, + IngressAnnotations: config.IngressAnnotations, } exitOnError(ray.NewReconciler(ctx, mgr, rayClusterOptions, config).SetupWithManager(mgr, config.ReconcileConcurrency), "unable to create controller", "controller", "RayCluster") diff --git a/ray-operator/main_test.go b/ray-operator/main_test.go index 211a87e925c..1f6facdaa96 100644 --- a/ray-operator/main_test.go +++ b/ray-operator/main_test.go @@ -6,6 +6,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -97,6 +98,42 @@ workerSidecarContainers: }, expectErr: false, }, + { + name: "config with ingress options", + configData: `apiVersion: config.ray.io/v1alpha1 +kind: Configuration +ingressHost: ray.example.com +ingressTLS: +- hosts: + - ray.example.com + secretName: ray-tls-secret +ingressAnnotations: + annotation0: value0 + annotation1: value1 +`, + expectedConfig: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + Kind: "Configuration", + APIVersion: "config.ray.io/v1alpha1", + }, + MetricsAddr: ":8080", + ProbeAddr: ":8082", + EnableLeaderElection: ptr.To(true), + ReconcileConcurrency: 1, + IngressHost: "ray.example.com", + IngressTLS: []networkingv1.IngressTLS{ + { + Hosts: []string{"ray.example.com"}, + SecretName: "ray-tls-secret", + }, + }, + IngressAnnotations: map[string]string{ + "annotation0": "value0", + "annotation1": "value1", + }, + }, + expectErr: false, + }, { name: "unknown filed ignored", configData: `apiVersion: config.ray.io/v1alpha1