diff --git a/controller/hybridgateway/builder/kongtarget.go b/controller/hybridgateway/builder/kongtarget.go index c20f959f6..a04b1bd47 100644 --- a/controller/hybridgateway/builder/kongtarget.go +++ b/controller/hybridgateway/builder/kongtarget.go @@ -52,25 +52,21 @@ func (b *KongTargetBuilder) WithLabels(route *gwtypes.HTTPRoute) *KongTargetBuil return b } -// WithBackendRef sets the target specification based on the given HTTPRoute and backend reference. -func (b *KongTargetBuilder) WithBackendRef(httpRoute *gwtypes.HTTPRoute, bRef *gwtypes.HTTPBackendRef) *KongTargetBuilder { - // Build the dns name of the service for the backendRef. - // TODO(alacuku): We need to handle the cluster domain properly for the cluster where we are running. - var namespace string - if bRef.Namespace == nil || *bRef.Namespace == "" { - namespace = httpRoute.Namespace - } else { - namespace = string(*bRef.Namespace) - } - - host := string(bRef.Name) + "." + namespace + ".svc.cluster.local" - port := strconv.Itoa(int(*bRef.Port)) - target := net.JoinHostPort(host, port) +// WithTarget sets the target (host:port) for the KongTarget. +func (b *KongTargetBuilder) WithTarget(host string, port gwtypes.PortNumber) *KongTargetBuilder { + target := net.JoinHostPort(host, strconv.Itoa(int(port))) b.target.Spec.Target = target - // Weight is optional, default to 100 if not specified - if bRef.Weight != nil { - b.target.Spec.Weight = int(*bRef.Weight) + return b +} + +// WithWeight sets the weight for the KongTarget. If weight is nil, it defaults to 100. +func (b *KongTargetBuilder) WithWeight(weight *int32) *KongTargetBuilder { + b.target.Spec.Weight = 100 // Weight is optional, default to 100 if not specified + if weight != nil { + newWeight := new(int) + *newWeight = int(*weight) + b.target.Spec.Weight = *newWeight } return b } diff --git a/controller/hybridgateway/builder/kongtarget_test.go b/controller/hybridgateway/builder/kongtarget_test.go index 46d0f2e0b..bba1f0391 100644 --- a/controller/hybridgateway/builder/kongtarget_test.go +++ b/controller/hybridgateway/builder/kongtarget_test.go @@ -3,6 +3,7 @@ package builder import ( "testing" + "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -54,129 +55,38 @@ func TestKongTargetBuilder_WithLabels(t *testing.T) { assert.NotEmpty(t, target.Labels) } -func TestKongTargetBuilder_WithBackendRef(t *testing.T) { - port := gatewayv1.PortNumber(8080) - weight := int32(100) +func TestNewKongTargetBuilder_WithTarget(t *testing.T) { + builder := NewKongTarget().WithTarget("myhost", 8080) + + target, err := builder.Build() + require.NoError(t, err) + assert.Equal(t, "myhost:8080", target.Spec.Target) +} +func TestKongTargetBuilder_WithWeight(t *testing.T) { tests := []struct { name string - httpRoute *gwtypes.HTTPRoute - backendRef *gwtypes.HTTPBackendRef - expectedTarget string + weight *int32 expectedWeight int }{ { - name: "backend ref with same namespace", - httpRoute: &gwtypes.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-route", - Namespace: "test-namespace", - }, - }, - backendRef: &gwtypes.HTTPBackendRef{ - BackendRef: gatewayv1.BackendRef{ - BackendObjectReference: gatewayv1.BackendObjectReference{ - Name: "test-service", - Port: &port, - }, - Weight: &weight, - }, - }, - expectedTarget: "test-service.test-namespace.svc.cluster.local:8080", + name: "backend ref with weight 100", + weight: lo.ToPtr(int32(100)), expectedWeight: 100, }, { - name: "backend ref with different namespace", - httpRoute: &gwtypes.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-route", - Namespace: "test-namespace", - }, - }, - backendRef: &gwtypes.HTTPBackendRef{ - BackendRef: gatewayv1.BackendRef{ - BackendObjectReference: gatewayv1.BackendObjectReference{ - Name: "test-service", - Namespace: &[]gatewayv1.Namespace{"other-namespace"}[0], - Port: &port, - }, - Weight: &weight, - }, - }, - expectedTarget: "test-service.other-namespace.svc.cluster.local:8080", + name: "backend ref without weight", + weight: nil, expectedWeight: 100, }, - { - name: "backend ref with empty namespace", - httpRoute: &gwtypes.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-route", - Namespace: "test-namespace", - }, - }, - backendRef: &gwtypes.HTTPBackendRef{ - BackendRef: gatewayv1.BackendRef{ - BackendObjectReference: gatewayv1.BackendObjectReference{ - Name: "test-service", - Namespace: &[]gatewayv1.Namespace{""}[0], - Port: &port, - }, - Weight: &weight, - }, - }, - expectedTarget: "test-service.test-namespace.svc.cluster.local:8080", - expectedWeight: 100, - }, - { - name: "backend ref with nil namespace", - httpRoute: &gwtypes.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-route", - Namespace: "test-namespace", - }, - }, - backendRef: &gwtypes.HTTPBackendRef{ - BackendRef: gatewayv1.BackendRef{ - BackendObjectReference: gatewayv1.BackendObjectReference{ - Name: "test-service", - Namespace: nil, - Port: &port, - }, - Weight: &weight, - }, - }, - expectedTarget: "test-service.test-namespace.svc.cluster.local:8080", - expectedWeight: 100, - }, - { - name: "backend ref without weight", - httpRoute: &gwtypes.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-route", - Namespace: "test-namespace", - }, - }, - backendRef: &gwtypes.HTTPBackendRef{ - BackendRef: gatewayv1.BackendRef{ - BackendObjectReference: gatewayv1.BackendObjectReference{ - Name: "test-service", - Port: &port, - }, - Weight: nil, - }, - }, - expectedTarget: "test-service.test-namespace.svc.cluster.local:8080", - expectedWeight: 0, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - builder := NewKongTarget().WithBackendRef(tt.httpRoute, tt.backendRef) + builder := NewKongTarget().WithWeight(tt.weight) target, err := builder.Build() require.NoError(t, err) - assert.Equal(t, tt.expectedTarget, target.Spec.Target) assert.Equal(t, tt.expectedWeight, target.Spec.Weight) }) } @@ -328,7 +238,8 @@ func TestKongTargetBuilder_Chaining(t *testing.T) { target := NewKongTarget(). WithName("test-target"). WithNamespace("test-namespace"). - WithBackendRef(httpRoute, backendRef). + WithTarget(string(backendRef.Name)+"."+httpRoute.Namespace+".svc.cluster.local", *backendRef.Port). + WithWeight(backendRef.Weight). WithUpstreamRef("test-upstream"). WithOwner(httpRoute). WithLabels(httpRoute). diff --git a/controller/hybridgateway/controller.go b/controller/hybridgateway/controller.go index e4585d97f..6ef416c16 100644 --- a/controller/hybridgateway/controller.go +++ b/controller/hybridgateway/controller.go @@ -4,15 +4,22 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/kong/kong-operator/controller/hybridgateway/converter" "github.com/kong/kong-operator/controller/hybridgateway/route" "github.com/kong/kong-operator/controller/hybridgateway/watch" "github.com/kong/kong-operator/controller/pkg/log" + "github.com/kong/kong-operator/internal/utils/index" ) //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongroutes,verbs=get;list;watch;create;update;patch;delete @@ -63,6 +70,19 @@ func (r *HybridGatewayReconciler[t, tPtr]) SetupWithManager(ctx context.Context, builder = builder.Owns(owned) } + // Watch for services to trigger reconciliation of HTTPRoutes that reference them. + // Watch for services to trigger reconciliation of HTTPRoutes that reference them. + builder.Watches( + &corev1.Service{}, + handler.EnqueueRequestsFromMapFunc(r.findHTTPRoutesForService), + ) + + // Watch for endpoint slices to trigger reconciliation of HTTPRoutes that reference them. + builder.Watches( + &discoveryv1.EndpointSlice{}, + handler.EnqueueRequestsFromMapFunc(r.findHTTPRoutesForEndpointSlice), + ) + return builder.Complete(r) } @@ -105,3 +125,66 @@ func (r *HybridGatewayReconciler[t, tPtr]) Reconcile(ctx context.Context, req ct return ctrl.Result{}, nil } + +func (r *HybridGatewayReconciler[t, tPtr]) findHTTPRoutesForService(ctx context.Context, obj client.Object) []reconcile.Request { + logger := ctrllog.FromContext(ctx).WithName("HybridGatewayServiceWatcher") + service, ok := obj.(*corev1.Service) + if !ok { + logger.Error(fmt.Errorf("unexpected type %T, expected %T", obj, &corev1.Service{}), "failed to cast object to service") + return nil + } + return r.httpRoutesForService(ctx, service) +} + +func (r *HybridGatewayReconciler[t, tPtr]) findHTTPRoutesForEndpointSlice(ctx context.Context, obj client.Object) []reconcile.Request { + logger := ctrllog.FromContext(ctx).WithName("HybridGatewayEndpointSliceWatcher") + endpointSlice, ok := obj.(*discoveryv1.EndpointSlice) + if !ok { + logger.Error(fmt.Errorf("unexpected type %T, expected %T", obj, &discoveryv1.EndpointSlice{}), "failed to cast object to endpointslice") + return nil + } + + serviceName, ok := endpointSlice.Labels[discoveryv1.LabelServiceName] + if !ok { + logger.Info("endpointslice has no service name label", "namespace", endpointSlice.Namespace, "name", endpointSlice.Name) + return nil + } + + service := &corev1.Service{} + if err := r.Get(ctx, types.NamespacedName{Namespace: endpointSlice.Namespace, Name: serviceName}, service); err != nil { + logger.Error(err, "failed to get service for endpointslice", "servicename", serviceName, "endpointslicenamespace", endpointSlice.Namespace) + return nil + } + + return r.httpRoutesForService(ctx, service) +} + +func (r *HybridGatewayReconciler[t, tPtr]) httpRoutesForService(ctx context.Context, service *corev1.Service) []reconcile.Request { + logger := ctrllog.FromContext(ctx).WithName("HybridGatewayWatcher") + var httpRoutes gatewayv1.HTTPRouteList + if err := r.List(ctx, &httpRoutes, + client.MatchingFields{ + index.BackendServicesOnHTTPRouteIndex: service.Namespace + "/" + service.Name, + }, + ); err != nil { + logger.Error(err, "failed to list httproutes") + return nil + } + + requests := make(map[reconcile.Request]struct{}) + for _, httpRoute := range httpRoutes.Items { + requests[reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: httpRoute.Namespace, + Name: httpRoute.Name, + }, + }] = struct{}{} + } + + reqs := make([]reconcile.Request, 0, len(requests)) + for req := range requests { + reqs = append(reqs, req) + } + + return reqs +} diff --git a/controller/hybridgateway/converter/http_route.go b/controller/hybridgateway/converter/http_route.go index dba4e8ee9..3c0211172 100644 --- a/controller/hybridgateway/converter/http_route.go +++ b/controller/hybridgateway/converter/http_route.go @@ -6,8 +6,12 @@ import ( "strings" "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -23,6 +27,14 @@ import ( var _ APIConverter[gwtypes.HTTPRoute] = &httpRouteConverter{} +// BackendRefTarget represents a target derived from a BackendRef in an HTTPRoute. +type BackendRefTarget struct { + Name string + Host string + Port gwtypes.PortNumber + Weight *int32 +} + // httpRouteConverter is a concrete implementation of the APIConverter interface for HTTPRoute. type httpRouteConverter struct { client.Client @@ -188,22 +200,28 @@ func (c *httpRouteConverter) translate(ctx context.Context) error { // Build the target resources. for _, bRef := range val.BackendRefs { - targetName := bRef.String() - - target, err := builder.NewKongTarget(). - WithName(targetName). - WithNamespace(c.route.Namespace). - WithLabels(c.route). - WithAnnotations(c.route, c.ir.GetParentRefByName(bRef.Name)). - WithUpstreamRef(name). - WithBackendRef(c.route, &bRef.BackendRef). - WithOwner(c.route).Build() + targets, err := c.getTargets(ctx, bRef) if err != nil { - // TODO: decide how to handle build errors in converter - // For now, skip this resource + // If we can't get targets for a backend ref, skip it. continue } - c.outputStore = append(c.outputStore, &target) + for _, target := range targets { + kongTarget, err := builder.NewKongTarget(). + WithName(target.Name). + WithNamespace(c.route.Namespace). + WithLabels(c.route). + WithAnnotations(c.route, c.ir.GetParentRefByName(bRef.Name)). + WithUpstreamRef(name). + WithTarget(target.Host, target.Port). + WithWeight(target.Weight). + WithOwner(c.route).Build() + if err != nil { + // TODO: decide how to handle build errors in converter + // For now, skip this resource + continue + } + c.outputStore = append(c.outputStore, &kongTarget) + } } // Build the kong route resource. @@ -381,3 +399,102 @@ func hostnameIntersection(listenerHostname, routeHostname string) string { return "" // No intersection } + +// getTargets get targets to the intermediate representation based on the BackendRefs in the HTTPRoute. +func (c *httpRouteConverter) getTargets(ctx context.Context, bRef intermediate.BackendRef) ([]BackendRefTarget, error) { + targets := []BackendRefTarget{} + + // retrieve the service with name and namespace from the BackendRef + svcNamespace := namespaceFromBackendRef(bRef.BackendRef, c.route.Namespace) + svc := &corev1.Service{} + err := c.Get(ctx, client.ObjectKey{Name: string(bRef.BackendRef.Name), Namespace: svcNamespace}, svc) + if err != nil { + // If the service is not found, return an empty target list (it might be created later). + return nil, err + } + // find the port in the service that matches the port in the BackendRef + svcPort, svcPortFound := lo.Find(svc.Spec.Ports, func(p corev1.ServicePort) bool { + return p.Port == int32(*bRef.BackendRef.Port) + }) + if !svcPortFound { + // If the port is not found, return an empty target list (it might be created later). + return nil, fmt.Errorf("port %v not found in service %s/%s", *bRef.BackendRef.Port, svcNamespace, svc.Name) + } + + // TODO: we have to add a way to configure if we want to use EndpointSlices or Service FQDN + // For now, we will always use EndpointSlices if available + // If you want to use Service FQDN, uncomment the following block + if false { + // Use Service FQDN as target + target := BackendRefTarget{ + Name: bRef.String(), + Host: TargetHostAsServiceFQDN(bRef.BackendRef, c.route.Namespace), + Port: *bRef.BackendRef.Port, + Weight: bRef.BackendRef.Weight, + } + targets = append(targets, target) + + return targets, nil + } + + // Use EndpointSlices as targets + // List EndpointSlices for the service + // Reference: https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/ + // Note: EndpointSlices are namespaced resources + // Reference: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#endpointslice-v1-discovery-k8s-io + endpointSlices := &discoveryv1.EndpointSliceList{} + req, err := labels.NewRequirement(discoveryv1.LabelServiceName, selection.Equals, []string{svc.Name}) + if err != nil { + return nil, err + } + labelSelector := labels.NewSelector().Add(*req) + err = c.List(ctx, endpointSlices, &client.ListOptions{Namespace: svcNamespace, LabelSelector: labelSelector}) + + if err == nil { + for _, endpointSlice := range endpointSlices.Items { + leng := len(endpointSlice.Endpoints) + + for _, p := range endpointSlice.Ports { + if p.Port == nil || *p.Port < 0 || *p.Protocol != svcPort.Protocol || *p.Name != svcPort.Name { + continue + } + upstreamPort := *p.Port + + for _, endpoint := range endpointSlice.Endpoints { + if endpoint.Conditions.Ready != nil && !*endpoint.Conditions.Ready { + // Skip not ready endpoints + continue + } + + for _, addr := range endpoint.Addresses { + weight := *bRef.BackendRef.Weight / int32(leng) + target := BackendRefTarget{ + Name: fmt.Sprintf("%s-%s", bRef.String(), strings.ReplaceAll(addr, ".", "-")), + Host: addr, + Port: gwtypes.PortNumber(upstreamPort), + Weight: &weight, + } + targets = append(targets, target) + } + } + } + } + } + return targets, nil +} + +// TargetHostAsServiceFQDN constructs the fully qualified domain name (FQDN) for a backend service. +// It combines the backend reference name, namespace, and standard Kubernetes service domain suffix. +func TargetHostAsServiceFQDN(bRef gwtypes.HTTPBackendRef, defaultNamespace string) string { + namespace := namespaceFromBackendRef(bRef, defaultNamespace) + return string(bRef.Name) + "." + namespace + ".svc.cluster.local" +} + +// namespaceFromBackendRef extracts the namespace from a BackendRef. +// If the BackendRef does not specify a namespace, it defaults to the provided defaultNamespace. +func namespaceFromBackendRef(bRef gwtypes.HTTPBackendRef, defaultNamespace string) string { + if bRef.Namespace != nil { + return string(*bRef.Namespace) + } + return defaultNamespace +}