diff --git a/apis/gateway/v1beta1/targetgroupconfig_types.go b/apis/gateway/v1beta1/targetgroupconfig_types.go index 680f96a696..15c1bae8d1 100644 --- a/apis/gateway/v1beta1/targetgroupconfig_types.go +++ b/apis/gateway/v1beta1/targetgroupconfig_types.go @@ -20,7 +20,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Reference defines how to look up the Target Group configuration for a service. +// Reference defines how to look up the Target Group configuration for a kubernetes object. type Reference struct { // Group is the group of the referent. For example, "gateway.networking.k8s.io". // When unspecified or empty string, core API group is inferred. @@ -108,7 +108,7 @@ const ( TargetTypeIP TargetType = "ip" ) -// +kubebuilder:validation:Enum=http;https;tcp +// +kubebuilder:validation:Enum=HTTP;HTTPS;TCP type TargetGroupHealthCheckProtocol string const ( diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 106f8bd7d5..93af18d9c6 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -821,9 +821,9 @@ spec: with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, in seconds, @@ -1014,9 +1014,9 @@ spec: and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, diff --git a/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml index c7b9a6cd7a..8327eaf0e7 100644 --- a/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -82,9 +82,9 @@ spec: with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, in seconds, @@ -275,9 +275,9 @@ spec: and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, diff --git a/controllers/gateway/eventhandlers/target_group_configuration_events.go b/controllers/gateway/eventhandlers/target_group_configuration_events.go index 5867b090bc..c1a2dbc246 100644 --- a/controllers/gateway/eventhandlers/target_group_configuration_events.go +++ b/controllers/gateway/eventhandlers/target_group_configuration_events.go @@ -5,6 +5,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" @@ -13,16 +14,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // NewEnqueueRequestsForTargetGroupConfigurationEvent creates handler for TargetGroupConfiguration resources -func NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan chan<- event.TypedGenericEvent[*corev1.Service], +func NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan chan<- event.TypedGenericEvent[*corev1.Service], tcpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TCPRoute], k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*elbv2gw.TargetGroupConfiguration, reconcile.Request] { return &enqueueRequestsForTargetGroupConfigurationEvent{ - svcEventChan: svcEventChan, - k8sClient: k8sClient, - eventRecorder: eventRecorder, - logger: logger, + svcEventChan: svcEventChan, + tcpRouteEventChan: tcpRouteEventChan, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, } } @@ -30,49 +33,103 @@ var _ handler.TypedEventHandler[*elbv2gw.TargetGroupConfiguration, reconcile.Req // enqueueRequestsForTargetGroupConfigurationEvent handles TargetGroupConfiguration events type enqueueRequestsForTargetGroupConfigurationEvent struct { - svcEventChan chan<- event.TypedGenericEvent[*corev1.Service] - k8sClient client.Client - eventRecorder record.EventRecorder - logger logr.Logger + svcEventChan chan<- event.TypedGenericEvent[*corev1.Service] + tcpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TCPRoute] + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Create(ctx context.Context, e event.TypedCreateEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfigNew := e.Object h.logger.V(1).Info("enqueue targetgroupconfiguration create event", "targetgroupconfiguration", tgconfigNew.Name) - h.enqueueImpactedService(ctx, tgconfigNew, queue) + h.enqueueImpactedObject(ctx, tgconfigNew, queue) } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfigNew := e.ObjectNew h.logger.V(1).Info("enqueue targetgroupconfiguration update event", "targetgroupconfiguration", tgconfigNew.Name) - h.enqueueImpactedService(ctx, tgconfigNew, queue) + h.enqueueImpactedObject(ctx, tgconfigNew, queue) } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfig := e.Object h.logger.V(1).Info("enqueue targetgroupconfiguration delete event", "targetgroupconfiguration", tgconfig.Name) - h.enqueueImpactedService(ctx, tgconfig, queue) + h.enqueueImpactedObject(ctx, tgconfig, queue) } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfig := e.Object h.logger.V(1).Info("enqueue targetgroupconfiguration generic event", "targetgroupconfiguration", tgconfig.Name) - h.enqueueImpactedService(ctx, tgconfig, queue) + h.enqueueImpactedObject(ctx, tgconfig, queue) } -func (h *enqueueRequestsForTargetGroupConfigurationEvent) enqueueImpactedService(ctx context.Context, tgconfig *elbv2gw.TargetGroupConfiguration, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { - svcName := types.NamespacedName{Namespace: tgconfig.Namespace, Name: tgconfig.Spec.TargetReference.Name} - svc := &corev1.Service{} - if err := h.k8sClient.Get(ctx, svcName, svc); err != nil { - h.logger.V(1).Info("ignoring targetgroupconfiguration event for unknown service", +func (h *enqueueRequestsForTargetGroupConfigurationEvent) enqueueImpactedObject(ctx context.Context, tgconfig *elbv2gw.TargetGroupConfiguration, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + objName := types.NamespacedName{Namespace: tgconfig.Namespace, Name: tgconfig.Spec.TargetReference.Name} + + if tgconfig.Spec.TargetReference.Kind == nil || *tgconfig.Spec.TargetReference.Kind == "Service" { + svc := &corev1.Service{} + if err := h.k8sClient.Get(ctx, objName, svc); err != nil { + h.logger.V(1).Info("ignoring targetgroupconfiguration event for unknown service", + "targetgroupconfiguration", k8s.NamespacedName(tgconfig), + "service", k8s.NamespacedName(svc)) + return + } + h.logger.V(1).Info("enqueue service for targetgroupconfiguration event", "targetgroupconfiguration", k8s.NamespacedName(tgconfig), "service", k8s.NamespacedName(svc)) - return + h.svcEventChan <- event.TypedGenericEvent[*corev1.Service]{ + Object: svc, + } + } + + // TODO - We should probably use an indexer here, we have a task to do this. + if tgconfig.Spec.TargetReference.Kind != nil && *tgconfig.Spec.TargetReference.Kind == "Gateway" && h.tcpRouteEventChan != nil { + tcpRouteList := &gwalpha2.TCPRouteList{} + + if err := h.k8sClient.List(ctx, tcpRouteList); err != nil { + h.logger.V(1).Info("failed to list tcp routes for target group configuration event", "targetgroupconfiguration", k8s.NamespacedName(tgconfig)) + return + } + + impactedRoutes := getImpactedTCPRoutes(tcpRouteList, tgconfig) + for i := range impactedRoutes { + h.tcpRouteEventChan <- event.TypedGenericEvent[*gwalpha2.TCPRoute]{ + Object: impactedRoutes[i], + } + } + } - h.logger.V(1).Info("enqueue service for targetgroupconfiguration event", - "targetgroupconfiguration", k8s.NamespacedName(tgconfig), - "service", k8s.NamespacedName(svc)) - h.svcEventChan <- event.TypedGenericEvent[*corev1.Service]{ - Object: svc, +} + +func getImpactedTCPRoutes(list *gwalpha2.TCPRouteList, tgconfig *elbv2gw.TargetGroupConfiguration) []*gwalpha2.TCPRoute { + seen := sets.Set[types.NamespacedName]{} + res := make([]*gwalpha2.TCPRoute, 0) + + for i, route := range list.Items { + nsn := k8s.NamespacedName(&route) + for _, rule := range route.Spec.Rules { + for _, beRef := range rule.BackendRefs { + if beRef.Kind != nil && *beRef.Kind == "Gateway" { + if string(beRef.Name) == tgconfig.Spec.TargetReference.Name { + + // The route backend ns + var routeNs string + if beRef.Namespace == nil { + routeNs = route.Namespace + } else { + routeNs = string(*beRef.Namespace) + } + + if routeNs == tgconfig.Namespace && !seen.Has(nsn) { + res = append(res, &list.Items[i]) + seen.Insert(nsn) + } + + } + } + } + } } + return res } diff --git a/controllers/gateway/eventhandlers/target_group_configuration_events_test.go b/controllers/gateway/eventhandlers/target_group_configuration_events_test.go new file mode 100644 index 0000000000..25c925c527 --- /dev/null +++ b/controllers/gateway/eventhandlers/target_group_configuration_events_test.go @@ -0,0 +1,232 @@ +package eventhandlers + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "testing" +) + +func TestGetImpactedTCPRoutes(t *testing.T) { + tests := []struct { + name string + list *gwalpha2.TCPRouteList + tgconfig *elbv2gw.TargetGroupConfiguration + want []types.NamespacedName + }{ + { + name: "no routes", + list: &gwalpha2.TCPRouteList{}, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{}, + }, + { + name: "matching gateway backend", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "test-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{ + {Name: "route1", Namespace: "test-ns"}, + }, + }, + { + name: "non-matching gateway name", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "test-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "other-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{}, + }, + { + name: "different namespace", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "other-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{}, + }, + { + name: "cross-namespace with explicit namespace", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "route-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + Namespace: (*gwalpha2.Namespace)(awssdk.String("test-ns")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{ + {Name: "route1", Namespace: "route-ns"}, + }, + }, + { + name: "duplicate routes filtered", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "test-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{ + {Name: "route1", Namespace: "test-ns"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getImpactedTCPRoutes(tt.list, tt.tgconfig) + res := make([]types.NamespacedName, 0) + + for i := range got { + res = append(res, k8s.NamespacedName(got[i])) + } + + assert.Equal(t, tt.want, res) + }) + } +} diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 2e03ef2f57..d2aa222265 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -81,7 +81,7 @@ func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerT modelBuilder := gatewaymodel.NewModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), lbType, trackingProvider, elbv2TaggingManager, controllerConfig, cloud.EC2(), cloud.ELBV2(), cloud.ACM(), k8sClient, controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, supportedAddons, logger) stackMarshaller := deploy.NewDefaultStackMarshaller() - stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName, true, targetGroupCollector) + stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName, true, targetGroupCollector, lbType == elbv2model.LoadBalancerTypeNetwork) cfgResolver := newGatewayConfigResolver() @@ -351,7 +351,7 @@ func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gatewa } func (r *gatewayReconciler) deployModel(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, secrets []types.NamespacedName) error { - if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName, nil); err != nil { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName); err != nil { var requeueNeededAfter *ctrlerrors.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { return err @@ -531,7 +531,7 @@ func (r *gatewayReconciler) setupALBGatewayControllerWatches(ctrl controller.Con grpcRouteEventChan := make(chan event.TypedGenericEvent[*gwv1.GRPCRoute]) svcEventChan := make(chan event.TypedGenericEvent[*corev1.Service]) secretEventsChan := make(chan event.TypedGenericEvent[*corev1.Secret]) - tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, r.k8sClient, r.eventRecorder, + tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, nil, r.k8sClient, r.eventRecorder, loggerPrefix.WithName("TargetGroupConfiguration")) listenerRuleConfigEventHandler := eventhandlers.NewEnqueueRequestsForListenerRuleConfigurationEvent(httpRouteEventChan, grpcRouteEventChan, r.k8sClient, loggerPrefix.WithName("ListenerRuleConfiguration")) grpcRouteEventHandler := eventhandlers.NewEnqueueRequestsForGRPCRouteEvent(r.k8sClient, r.eventRecorder, @@ -591,7 +591,7 @@ func (r *gatewayReconciler) setupNLBGatewayControllerWatches(ctrl controller.Con udpRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.UDPRoute]) tlsRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.TLSRoute]) svcEventChan := make(chan event.TypedGenericEvent[*corev1.Service]) - tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, r.k8sClient, r.eventRecorder, + tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, tcpRouteEventChan, r.k8sClient, r.eventRecorder, loggerPrefix.WithName("TargetGroupConfiguration")) tcpRouteEventHandler := eventhandlers.NewEnqueueRequestsForTCPRouteEvent(r.k8sClient, r.eventRecorder, loggerPrefix.WithName("TCPRoute")) diff --git a/controllers/gateway/utils.go b/controllers/gateway/utils.go index 73ddab7ff7..78a396c94b 100644 --- a/controllers/gateway/utils.go +++ b/controllers/gateway/utils.go @@ -9,8 +9,6 @@ import ( "time" "unicode/utf8" - "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" - "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -184,7 +182,7 @@ func getServicesFromRoutes(listenerRouteMap map[int32][]routeutils.RouteDescript for _, rr := range route.GetAttachedRules() { for _, be := range rr.GetBackends() { if be.ServiceBackend != nil { - res.Insert(k8s.NamespacedName(be.ServiceBackend.Service)) + res.Insert(be.ServiceBackend.GetBackendNamespacedName()) } } } diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 47ec0cb648..9e776bb900 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -70,7 +70,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), targetGroupNameToArnMapper, logger, metricsCollector) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, - controllerConfig, ingressTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector) + controllerConfig, ingressTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector, true) classLoader := ingress.NewDefaultClassLoader(k8sClient, true) classAnnotationMatcher := ingress.NewDefaultClassAnnotationMatcher(controllerConfig.IngressConfig.IngressClass) manageIngressesWithoutIngressClass := controllerConfig.IngressConfig.IngressClass == "" @@ -206,10 +206,9 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr var secrets []types.NamespacedName var backendSGRequired bool var err error - var frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState var frontendNlb *elbv2model.LoadBalancer buildModelFn := func() { - stack, lb, secrets, backendSGRequired, frontendNlbTargetGroupDesiredState, frontendNlb, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector) + stack, lb, secrets, backendSGRequired, frontendNlb, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector) } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "build_model", buildModelFn) if err != nil { @@ -224,7 +223,7 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr r.logger.Info("successfully built model", "model", stackJSON) deployModelFn := func() { - err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress", frontendNlbTargetGroupDesiredState) + err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress") } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "deploy_model", deployModelFn) if err != nil { diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index d7ab802d06..2595c7d935 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -55,7 +55,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector, controllerConfig.FeatureGates.Enabled(config.EnableTCPUDPListenerType)) stackMarshaller := deploy.NewDefaultStackMarshaller() - stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector) + stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector, false) return &serviceReconciler{ k8sClient: k8sClient, eventRecorder: eventRecorder, @@ -154,7 +154,7 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) } func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error { - if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service", nil); err != nil { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service"); err != nil { var requeueNeededAfter *ctrlerrors.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { return err diff --git a/helm/aws-load-balancer-controller/crds/gateway-crds.yaml b/helm/aws-load-balancer-controller/crds/gateway-crds.yaml index 106f8bd7d5..93af18d9c6 100644 --- a/helm/aws-load-balancer-controller/crds/gateway-crds.yaml +++ b/helm/aws-load-balancer-controller/crds/gateway-crds.yaml @@ -821,9 +821,9 @@ spec: with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, in seconds, @@ -1014,9 +1014,9 @@ spec: and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, diff --git a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go index 7fd011e616..dda03c75fd 100644 --- a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go +++ b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go @@ -21,7 +21,7 @@ type TargetGroupsResult struct { Err error } -func NewFrontendNlbTargetSynthesizer(k8sClient client.Client, trackingProvider tracking.Provider, taggingManager TaggingManager, frontendNlbTargetsManager FrontendNlbTargetsManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState, findSDKTargetGroups func() TargetGroupsResult) *frontendNlbTargetSynthesizer { +func NewFrontendNlbTargetSynthesizer(k8sClient client.Client, trackingProvider tracking.Provider, taggingManager TaggingManager, frontendNlbTargetsManager FrontendNlbTargetsManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack, frontendNlbTargetGroupDesiredState *elbv2model.FrontendNlbTargetGroupDesiredState, findSDKTargetGroups func() TargetGroupsResult) *frontendNlbTargetSynthesizer { return &frontendNlbTargetSynthesizer{ k8sClient: k8sClient, trackingProvider: trackingProvider, @@ -43,12 +43,13 @@ type frontendNlbTargetSynthesizer struct { featureGates config.FeatureGates logger logr.Logger stack core.Stack - frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState + frontendNlbTargetGroupDesiredState *elbv2model.FrontendNlbTargetGroupDesiredState findSDKTargetGroups func() TargetGroupsResult } // Synthesize processes AWS target groups and deregisters ALB targets based on the desired state. func (s *frontendNlbTargetSynthesizer) Synthesize(ctx context.Context) error { + var resTGs []*elbv2model.TargetGroup s.stack.ListResources(&resTGs) res := s.findSDKTargetGroups() @@ -109,6 +110,10 @@ func (s *frontendNlbTargetSynthesizer) deregisterCurrentTarget(ctx context.Conte } func (s *frontendNlbTargetSynthesizer) PostSynthesize(ctx context.Context) error { + if s.frontendNlbTargetGroupDesiredState == nil { + return nil + } + var resTGs []*elbv2model.TargetGroup s.stack.ListResources(&resTGs) diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index 05da167307..12fa5ea967 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" awsmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sync" "github.com/go-logr/logr" @@ -22,16 +23,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - ingressController = "ingress" -) - // Using elbv2.TargetGroupsResult instead of defining our own // StackDeployer will deploy a resource stack into AWS and K8S. type StackDeployer interface { // Deploy a resource stack. - Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) error + Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string) error } // NewDefaultStackDeployer constructs new defaultStackDeployer. @@ -39,7 +36,7 @@ func NewDefaultStackDeployer(cloud services.Cloud, k8sClient client.Client, networkingManager networking.NetworkingManager, networkingSGManager networking.SecurityGroupManager, networkingSGReconciler networking.SecurityGroupReconciler, elbv2TaggingManager elbv2.TaggingManager, config config.ControllerConfig, tagPrefix string, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, controllerName string, enhancedDefaultingPolicyEnabled bool, - targetGroupCollector awsmetrics.TargetGroupCollector) *defaultStackDeployer { + targetGroupCollector awsmetrics.TargetGroupCollector, enableFrontendNLB bool) *defaultStackDeployer { trackingProvider := tracking.NewDefaultProvider(tagPrefix, config.ClusterName) ec2TaggingManager := ec2.NewDefaultTaggingManager(cloud.EC2(), networkingSGManager, cloud.VpcID(), logger) @@ -67,6 +64,7 @@ func NewDefaultStackDeployer(cloud services.Cloud, k8sClient client.Client, logger: logger, metricsCollector: metricsCollector, controllerName: controllerName, + enableFrontendNLB: enableFrontendNLB, } } @@ -95,6 +93,7 @@ type defaultStackDeployer struct { vpcID string metricsCollector lbcmetrics.MetricCollector controllerName string + enableFrontendNLB bool logger logr.Logger } @@ -105,7 +104,7 @@ type ResourceSynthesizer interface { } // Deploy a resource stack. -func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) error { +func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string) error { synthesizers := []ResourceSynthesizer{ ec2.NewSecurityGroupSynthesizer(d.cloud.EC2(), d.trackingProvider, d.ec2TaggingManager, d.ec2SGManager, d.vpcID, d.logger, stack), } @@ -121,9 +120,16 @@ func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, met return elbv2.TargetGroupsResult{TargetGroups: tgs, Err: err} }) - if controllerName == ingressController { + if d.enableFrontendNLB { + var desiredFENLBState []*elbv2model.FrontendNlbTargetGroupDesiredState + stack.ListResources(&desiredFENLBState) + var frontendNLBState *elbv2model.FrontendNlbTargetGroupDesiredState + if len(desiredFENLBState) == 1 { + frontendNLBState = desiredFENLBState[0] + } + synthesizers = append(synthesizers, elbv2.NewFrontendNlbTargetSynthesizer( - d.k8sClient, d.trackingProvider, d.elbv2TaggingManager, d.elbv2FrontendNlbTargetsManager, d.logger, d.featureGates, stack, frontendNlbTargetGroupDesiredState, findSDKTargetGroups)) + d.k8sClient, d.trackingProvider, d.elbv2TaggingManager, d.elbv2FrontendNlbTargetsManager, d.logger, d.featureGates, stack, frontendNLBState, findSDKTargetGroups)) } synthesizers = append(synthesizers, diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index 2143e1cb2b..d909d3849a 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -200,6 +200,8 @@ func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway psa.AddToStack(stack, lb.LoadBalancerARN()) } + _ = elbv2model.NewFrontendNlbTargetGroupDesiredState(stack, tgBuilder.getLocalFrontendNlbData()) + return stack, lb, newAddonConfig, securityGroups.backendSecurityGroupAllocated, secrets, nil } diff --git a/pkg/gateway/model/mock_tg_builder.go b/pkg/gateway/model/mock_tg_builder.go index 08ec381a39..21b080bb49 100644 --- a/pkg/gateway/model/mock_tg_builder.go +++ b/pkg/gateway/model/mock_tg_builder.go @@ -2,7 +2,6 @@ package model import ( "k8s.io/apimachinery/pkg/util/intstr" - elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" @@ -11,12 +10,17 @@ import ( ) type mockTargetGroupBuilder struct { - tgs []*elbv2model.TargetGroup - buildErr error + tgs []*elbv2model.TargetGroup + localFrontendNlbData map[string]*elbv2model.FrontendNlbTargetGroupState + buildErr error +} + +func (m *mockTargetGroupBuilder) getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState { + return m.localFrontendNlbData } func (m *mockTargetGroupBuilder) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { var tg *elbv2model.TargetGroup if len(m.tgs) > 0 { diff --git a/pkg/gateway/model/model_build_listener.go b/pkg/gateway/model/model_build_listener.go index 7ebe37e7ee..e1ebafabd1 100644 --- a/pkg/gateway/model/model_build_listener.go +++ b/pkg/gateway/model/model_build_listener.go @@ -71,7 +71,7 @@ func (l listenerBuilderImpl) buildListeners(ctx context.Context, stack core.Stac // build rules only for L7 gateways if l.loadBalancerType == elbv2model.LoadBalancerTypeApplication { - secretKeys, err := l.buildListenerRules(ctx, stack, ls, lb.Spec.IPAddressType, gw, port, lbCfg, routes) + secretKeys, err := l.buildListenerRules(ctx, stack, ls, lb.Spec.IPAddressType, gw, port, routes) if err != nil { return nil, err } @@ -177,7 +177,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core return nil, nil } - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, lbCfg, lb.Spec.IPAddressType, routeDescriptor, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, lb.Spec.IPAddressType, routeDescriptor, backend) if tgErr != nil { return &elbv2model.ListenerSpec{}, tgErr } @@ -185,7 +185,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core return listenerSpec, nil } -func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core.Stack, ls *elbv2model.Listener, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, port int32, lbCfg elbv2gw.LoadBalancerConfiguration, routes map[int32][]routeutils.RouteDescriptor) ([]types.NamespacedName, error) { +func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core.Stack, ls *elbv2model.Listener, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, port int32, routes map[int32][]routeutils.RouteDescriptor) ([]types.NamespacedName, error) { // sort all rules based on precedence rulesWithPrecedenceOrder := routeutils.SortAllRulesByPrecedence(routes[port], port) secrets := make([]types.NamespacedName, 0) @@ -222,7 +222,7 @@ func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core. } targetGroupTuples := make([]elbv2model.TargetGroupTuple, 0, len(rule.GetBackends())) for _, backend := range rule.GetBackends() { - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, lbCfg, ipAddressType, route, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, ipAddressType, route, backend) if tgErr != nil { return nil, tgErr } diff --git a/pkg/gateway/model/model_build_listener_test.go b/pkg/gateway/model/model_build_listener_test.go index 61f81eb000..0eb414e0c1 100644 --- a/pkg/gateway/model/model_build_listener_test.go +++ b/pkg/gateway/model/model_build_listener_test.go @@ -7,7 +7,6 @@ import ( elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "reflect" @@ -1318,11 +1317,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, }, @@ -1391,11 +1387,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, }, @@ -1451,11 +1444,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, ListenerRuleConfig: &elbv2gw.ListenerRuleConfiguration{ @@ -1526,11 +1516,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, ListenerRuleConfig: &elbv2gw.ListenerRuleConfiguration{ @@ -1718,7 +1705,7 @@ func Test_BuildListenerRules(t *testing.T) { Spec: elbv2model.ListenerSpec{ Protocol: tc.listenerProtocol, }, - }, tc.ipAddressType, &gwv1.Gateway{}, tc.port, elbv2gw.LoadBalancerConfiguration{}, tc.routes) + }, tc.ipAddressType, &gwv1.Gateway{}, tc.port, tc.routes) assert.NoError(t, err) var resLRs []*elbv2model.ListenerRule diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index a96cfc6387..ce826db721 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -20,7 +20,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" elbv2modelk8s "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2/k8s" - "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_utils" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "strconv" @@ -33,7 +32,8 @@ type buildTargetGroupOutput struct { type targetGroupBuilder interface { buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) + getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState } type targetGroupBuilderImpl struct { @@ -49,6 +49,8 @@ type targetGroupBuilderImpl struct { tgbNetworkBuilder targetGroupBindingNetworkBuilder targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper + localFrontendNlbData map[string]*elbv2model.FrontendNlbTargetGroupState + defaultTargetType elbv2model.TargetType defaultHealthCheckMatcherHTTPCode string @@ -71,6 +73,10 @@ type targetGroupBuilderImpl struct { defaultHealthCheckUnhealthyThresholdForInstanceModeLocal int32 } +func (builder *targetGroupBuilderImpl) getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState { + return builder.localFrontendNlbData +} + func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper, loadBalancerType elbv2model.LoadBalancerType, tgbNetworkBuilder targetGroupBindingNetworkBuilder, tgPropertiesConstructor gateway.TargetGroupConfigConstructor, defaultTargetType string, targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper) targetGroupBuilder { return &targetGroupBuilderImpl{ loadBalancerType: loadBalancerType, @@ -80,6 +86,7 @@ func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper tgPropertiesConstructor: tgPropertiesConstructor, targetGroupNameToArnMapper: targetGroupNameToArnMapper, tgByResID: make(map[string]*elbv2model.TargetGroup), + localFrontendNlbData: make(map[string]*elbv2model.FrontendNlbTargetGroupState), tagHelper: tagHelper, defaultTargetType: elbv2model.TargetType(defaultTargetType), defaultHealthCheckMatcherHTTPCode: "200-399", @@ -101,10 +108,18 @@ func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper } func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { if backend.ServiceBackend != nil { - tg, err := builder.buildTargetGroupFromService(stack, gw, lbConfig, lbIPType, routeDescriptor, *backend.ServiceBackend) + tg, err := builder.buildTargetGroupFromService(stack, gw, lbIPType, routeDescriptor, *backend.ServiceBackend) + if err != nil { + return nil, err + } + return tg.TargetGroupARN(), nil + } + + if backend.GatewayBackend != nil { + tg, err := builder.buildTargetGroupFromGateway(stack, gw, listenerPort, lbIPType, routeDescriptor, *backend.GatewayBackend) if err != nil { return nil, err } @@ -120,14 +135,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, } func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { - targetGroupProps := backendConfig.ELBV2TargetGroupProps - tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backendConfig.Service), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.ServicePort.TargetPort) + gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { + targetGroupProps := backendConfig.GetTargetGroupProps() + tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), backendConfig.GetBackendNamespacedName(), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.GetIdentifierPort()) if tg, exists := builder.tgByResID[tgResID]; exists { return tg, nil } - tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbConfig, lbIPType, backendConfig, targetGroupProps) + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, &backendConfig, targetGroupProps) if err != nil { return nil, err } @@ -149,6 +164,33 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.St return tg, nil } +func (builder *targetGroupBuilderImpl) buildTargetGroupFromGateway(stack core.Stack, + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.GatewayBackendConfig) (*elbv2model.TargetGroup, error) { + targetGroupProps := backendConfig.GetTargetGroupProps() + tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), backendConfig.GetBackendNamespacedName(), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.GetIdentifierPort()) + if tg, exists := builder.tgByResID[tgResID]; exists { + return tg, nil + } + + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, &backendConfig, targetGroupProps) + if err != nil { + return nil, err + } + + tg := elbv2model.NewTargetGroup(stack, tgResID, tgSpec) + builder.tgByResID[tgResID] = tg + + builder.localFrontendNlbData[tgSpec.Name] = &elbv2model.FrontendNlbTargetGroupState{ + Name: tgSpec.Name, + ARN: tg.TargetGroupARN(), + Port: listenerPort, + TargetARN: core.LiteralStringToken(backendConfig.GetALBARN()), + TargetPort: *tg.Spec.Port, + } + + return tg, nil +} + func (builder *targetGroupBuilderImpl) buildTargetGroupFromStaticName(cfg routeutils.LiteralTargetGroupConfig) (core.StringToken, error) { tgArn, err := builder.targetGroupNameToArnMapper.GetArnByName(context.Background(), cfg.Name) @@ -162,9 +204,9 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupFromStaticName(cfg routeu func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gateway, tgProps *elbv2gw.TargetGroupProps, tgSpec elbv2model.TargetGroupSpec, nodeSelector *metav1.LabelSelector, backendConfig routeutils.ServiceBackendConfig) (elbv2modelk8s.TargetGroupBindingResourceSpec, error) { targetType := elbv2api.TargetType(tgSpec.TargetType) - targetPort := backendConfig.ServicePort.TargetPort + targetPort := backendConfig.GetServicePort().TargetPort if targetType == elbv2api.TargetTypeInstance { - targetPort = intstr.FromInt32(backendConfig.ServicePort.NodePort) + targetPort = intstr.FromInt32(backendConfig.GetServicePort().NodePort) } tgbNetworking, err := builder.tgbNetworkBuilder.buildTargetGroupBindingNetworking(tgSpec, targetPort) if err != nil { @@ -193,7 +235,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate return elbv2modelk8s.TargetGroupBindingResourceSpec{ Template: elbv2modelk8s.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ - Namespace: backendConfig.Service.Namespace, + Namespace: backendConfig.GetBackendNamespacedName().Namespace, Name: tgSpec.Name, Annotations: annotations, Labels: labels, @@ -202,8 +244,8 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate TargetGroupARN: nil, // This should get filled in later! TargetType: &targetType, ServiceRef: elbv2api.ServiceReference{ - Name: backendConfig.Service.Name, - Port: intstr.FromInt32(backendConfig.ServicePort.Port), + Name: backendConfig.GetBackendNamespacedName().Name, + Port: intstr.FromInt32(backendConfig.GetServicePort().Port), }, Networking: tgbNetworking, NodeSelector: nodeSelector, @@ -216,8 +258,8 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate }, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, backendConfig routeutils.ServiceBackendConfig, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { - targetType := builder.buildTargetGroupTargetType(targetGroupProps) +func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbIPType elbv2model.IPAddressType, backendConfig routeutils.TargetGroupConfigurator, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { + targetType := backendConfig.GetTargetType(builder.defaultTargetType) tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route) if err != nil { return elbv2model.TargetGroupSpec{}, err @@ -229,7 +271,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro return elbv2model.TargetGroupSpec{}, err } tgAttributesMap := builder.buildTargetGroupAttributes(targetGroupProps) - ipAddressType, err := builder.buildTargetGroupIPAddressType(backendConfig.Service, lbIPType) + ipAddressType, err := builder.buildTargetGroupIPAddressType(backendConfig, lbIPType) if err != nil { return elbv2model.TargetGroupSpec{}, err } @@ -238,8 +280,8 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro if err != nil { return elbv2model.TargetGroupSpec{}, err } - tgPort := builder.buildTargetGroupPort(targetType, *backendConfig.ServicePort) - name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), route.GetRouteKind(), k8s.NamespacedName(backendConfig.Service), tgPort, targetType, tgProtocol, tgProtocolVersion) + tgPort := backendConfig.GetTargetGroupPort(targetType) + name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), route.GetRouteKind(), backendConfig.GetBackendNamespacedName(), tgPort, targetType, tgProtocol, tgProtocolVersion) if tgPort == 0 { if targetType == elbv2model.TargetTypeIP { @@ -293,44 +335,12 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *el return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid) } -func (builder *targetGroupBuilderImpl) buildTargetGroupTargetType(targetGroupProps *elbv2gw.TargetGroupProps) elbv2model.TargetType { - if targetGroupProps == nil || targetGroupProps.TargetType == nil { - return builder.defaultTargetType +func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(backendConfig routeutils.TargetGroupConfigurator, loadBalancerIPAddressType elbv2model.IPAddressType) (elbv2model.TargetGroupIPAddressType, error) { + addressType := backendConfig.GetIPAddressType() + if addressType == elbv2model.TargetGroupIPAddressTypeIPv6 && !isIPv6Supported(loadBalancerIPAddressType) { + return "", errors.New("unsupported IPv6 configuration, lb not dual-stack") } - - return elbv2model.TargetType(*targetGroupProps.TargetType) -} - -func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(svc *corev1.Service, loadBalancerIPAddressType elbv2model.IPAddressType) (elbv2model.TargetGroupIPAddressType, error) { - var ipv6Configured bool - for _, ipFamily := range svc.Spec.IPFamilies { - if ipFamily == corev1.IPv6Protocol { - ipv6Configured = true - break - } - } - if ipv6Configured { - if !isIPv6Supported(loadBalancerIPAddressType) { - return "", errors.New("unsupported IPv6 configuration, lb not dual-stack") - } - return elbv2model.TargetGroupIPAddressTypeIPv6, nil - } - return elbv2model.TargetGroupIPAddressTypeIPv4, nil -} - -// buildTargetGroupPort constructs the TargetGroup's port. -// Note: TargetGroup's port is not in the data path as we always register targets with port specified. -// so these settings don't really matter to our controller, -// and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing. -func (builder *targetGroupBuilderImpl) buildTargetGroupPort(targetType elbv2model.TargetType, svcPort corev1.ServicePort) int32 { - if targetType == elbv2model.TargetTypeInstance { - return svcPort.NodePort - } - if svcPort.TargetPort.Type == intstr.Int { - return int32(svcPort.TargetPort.IntValue()) - } - // If all else fails, just return 1 as alluded to above, this setting doesn't really matter. - return 1 + return addressType, nil } func (builder *targetGroupBuilderImpl) buildTargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { @@ -418,19 +428,19 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupProtocolVersion(targetGro return &http1 } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion, targetType elbv2model.TargetType, backendConfig routeutils.ServiceBackendConfig) (elbv2model.TargetGroupHealthCheckConfig, error) { +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion, targetType elbv2model.TargetType, backendConfig routeutils.TargetGroupConfigurator) (elbv2model.TargetGroupHealthCheckConfig, error) { // add ServiceExternalTrafficPolicyLocal support var isServiceExternalTrafficPolicyTypeLocal = false if targetType == elbv2model.TargetTypeInstance && - backendConfig.Service.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeLocal && + backendConfig.GetExternalTrafficPolicy() == corev1.ServiceExternalTrafficPolicyTypeLocal && builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { isServiceExternalTrafficPolicyTypeLocal = true } - healthCheckPort, err := builder.buildTargetGroupHealthCheckPort(targetGroupProps, targetType, backendConfig.Service, isServiceExternalTrafficPolicyTypeLocal) + healthCheckPort, err := backendConfig.GetHealthCheckPort(targetType, isServiceExternalTrafficPolicyTypeLocal) if err != nil { return elbv2model.TargetGroupHealthCheckConfig{}, err } - healthCheckProtocol := builder.buildTargetGroupHealthCheckProtocol(targetGroupProps, tgProtocol, isServiceExternalTrafficPolicyTypeLocal) // + healthCheckProtocol := builder.buildTargetGroupHealthCheckProtocol(targetGroupProps, targetType, tgProtocol, isServiceExternalTrafficPolicyTypeLocal) // healthCheckPath := builder.buildTargetGroupHealthCheckPath(targetGroupProps, tgProtocolVersion, healthCheckProtocol, isServiceExternalTrafficPolicyTypeLocal) // healthCheckMatcher := builder.buildTargetGroupHealthCheckMatcher(targetGroupProps, tgProtocolVersion, healthCheckProtocol) // @@ -452,44 +462,17 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetG return hcConfig, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPort(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, svc *corev1.Service, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) { - - portConfigNotExist := targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckPort == nil - - if portConfigNotExist && isServiceExternalTrafficPolicyTypeLocal { - return intstr.FromInt32(svc.Spec.HealthCheckNodePort), nil - } - - if portConfigNotExist || *targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { - return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil - } - - healthCheckPort := intstr.Parse(*targetGroupProps.HealthCheckConfig.HealthCheckPort) - if healthCheckPort.Type == intstr.Int { - return healthCheckPort, nil - } - hcSvcPort, err := k8s.LookupServicePort(svc, healthCheckPort) - if err != nil { - return intstr.FromString(""), err - } - - if targetType == elbv2model.TargetTypeInstance { - return intstr.FromInt(int(hcSvcPort.NodePort)), nil - } - - if hcSvcPort.TargetPort.Type == intstr.Int { - return hcSvcPort.TargetPort, nil - } - return intstr.IntOrString{}, errors.New("cannot use named healthCheckPort for IP TargetType when service's targetPort is a named port") -} - -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, isServiceExternalTrafficPolicyTypeLocal bool) elbv2model.Protocol { +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, isServiceExternalTrafficPolicyTypeLocal bool) elbv2model.Protocol { if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckProtocol == nil { if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { if isServiceExternalTrafficPolicyTypeLocal { return builder.defaultHealthCheckProtocolForInstanceModeLocal } + // ALB targets only support HTTP / HTTPS health checks. + if targetType == elbv2model.TargetTypeALB { + return elbv2model.ProtocolHTTP + } return elbv2model.ProtocolTCP } return tgProtocol diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index 387e13346f..64880fd0fd 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -13,6 +13,8 @@ import ( elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" elbv2modelk8s "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2/k8s" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" @@ -31,7 +33,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { defaultTargetType string gateway *gwv1.Gateway route *routeutils.MockRoute - backend routeutils.ServiceBackendConfig + backend *routeutils.ServiceBackendConfig tagErr error expectErr bool expectedTgSpec elbv2model.TargetGroupSpec @@ -53,14 +55,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -68,8 +71,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, NodePort: 8080, - }, - }, + }), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-8d8111f6ac", TargetType: elbv2model.TargetTypeInstance, @@ -108,14 +110,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -123,8 +126,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, NodePort: 8080, - }, - }, + }), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-224f4b6ea6", TargetType: elbv2model.TargetTypeInstance, @@ -168,14 +170,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -183,8 +186,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, NodePort: 8080, - }, - }, + }), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-3bce8b0f70", TargetType: elbv2model.TargetTypeIP, @@ -223,14 +225,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -239,7 +242,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, @@ -283,14 +286,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -298,7 +302,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, }, - }, + ), expectErr: true, }, } @@ -313,7 +317,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { builder := newTargetGroupBuilder("my-cluster", "vpc-xxx", tagger, tc.lbType, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), tc.defaultTargetType, nil) - out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(tc.gateway, tc.route, elbv2gw.LoadBalancerConfiguration{}, elbv2model.IPAddressTypeIPV4, tc.backend, nil) + out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(tc.gateway, tc.route, elbv2model.IPAddressTypeIPV4, tc.backend, nil) if tc.expectErr { assert.Error(t, err) return @@ -337,7 +341,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { defaultTargetType string gateway *gwv1.Gateway route *routeutils.MockRoute - backend routeutils.ServiceBackendConfig + backend *routeutils.ServiceBackendConfig tagErr error expectErr bool expectedTgSpec elbv2model.TargetGroupSpec @@ -359,14 +363,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -375,7 +380,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-d02da2803b", TargetType: elbv2model.TargetTypeInstance, @@ -433,14 +438,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -449,7 +455,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-224f4b6ea6", TargetType: elbv2model.TargetTypeInstance, @@ -512,14 +518,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -528,7 +535,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-3bce8b0f70", TargetType: elbv2model.TargetTypeIP, @@ -586,14 +593,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -602,7 +610,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, @@ -675,14 +683,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -691,7 +700,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, @@ -754,7 +763,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { builder := newTargetGroupBuilder("my-cluster", "vpc-xxx", tagger, tc.lbType, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), tc.defaultTargetType, nil) - out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupBindingSpec(tc.gateway, nil, tc.expectedTgSpec, nil, tc.backend) + out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupBindingSpec(tc.gateway, nil, tc.expectedTgSpec, nil, *tc.backend) assert.Equal(t, tc.expectedTgBindingSpec, out) assert.NoError(t, err) @@ -816,24 +825,6 @@ func Test_buildTargetGroupName(t *testing.T) { } } -func Test_buildTargetGroupTargetType(t *testing.T) { - builder := targetGroupBuilderImpl{ - defaultTargetType: elbv2model.TargetTypeIP, - } - - res := builder.buildTargetGroupTargetType(nil) - assert.Equal(t, elbv2model.TargetTypeIP, res) - - res = builder.buildTargetGroupTargetType(&elbv2gw.TargetGroupProps{}) - assert.Equal(t, elbv2model.TargetTypeIP, res) - - inst := elbv2gw.TargetTypeInstance - res = builder.buildTargetGroupTargetType(&elbv2gw.TargetGroupProps{ - TargetType: &inst, - }) - assert.Equal(t, elbv2model.TargetTypeInstance, res) -} - func Test_buildTargetGroupIPAddressType(t *testing.T) { testCases := []struct { name string @@ -901,7 +892,7 @@ func Test_buildTargetGroupIPAddressType(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { builder := targetGroupBuilderImpl{} - res, err := builder.buildTargetGroupIPAddressType(tc.svc, tc.loadBalancerIPAddressType) + res, err := builder.buildTargetGroupIPAddressType(routeutils.NewServiceBackendConfig(tc.svc, nil, nil), tc.loadBalancerIPAddressType) if tc.expectErr { assert.Error(t, err) return @@ -913,57 +904,6 @@ func Test_buildTargetGroupIPAddressType(t *testing.T) { } } -func Test_buildTargetGroupPort(t *testing.T) { - testCases := []struct { - name string - targetType elbv2model.TargetType - svcPort corev1.ServicePort - expected int32 - }{ - { - name: "instance", - svcPort: corev1.ServicePort{ - NodePort: 8080, - }, - targetType: elbv2model.TargetTypeInstance, - expected: 8080, - }, - { - name: "instance - no node port", - svcPort: corev1.ServicePort{}, - targetType: elbv2model.TargetTypeInstance, - expected: 0, - }, - { - name: "ip", - svcPort: corev1.ServicePort{ - NodePort: 8080, - TargetPort: intstr.FromInt32(80), - }, - targetType: elbv2model.TargetTypeIP, - expected: 80, - }, - { - name: "ip - str port", - svcPort: corev1.ServicePort{ - NodePort: 8080, - TargetPort: intstr.FromString("foo"), - }, - targetType: elbv2model.TargetTypeIP, - expected: 1, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - builder := targetGroupBuilderImpl{} - res := builder.buildTargetGroupPort(tc.targetType, tc.svcPort) - assert.Equal(t, tc.expected, res) - - }) - } -} - func Test_buildTargetGroupProtocol(t *testing.T) { testCases := []struct { name string @@ -1244,192 +1184,10 @@ func Test_buildTargetGroupProtocolVersion(t *testing.T) { } } -func Test_buildTargetGroupHealthCheckPort(t *testing.T) { - testCases := []struct { - name string - isServiceExternalTrafficPolicyTypeLocal bool - targetGroupProps *elbv2gw.TargetGroupProps - targetType elbv2model.TargetType - svc *corev1.Service - expected intstr.IntOrString - expectErr bool - }{ - { - name: "nil props", - isServiceExternalTrafficPolicyTypeLocal: false, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "nil hc props", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{}, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "nil hc port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{}, - }, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "explicit is use traffic port hc port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), - }, - }, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "explicit port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("80"), - }, - }, - expected: intstr.FromInt32(80), - }, - { - name: "resolve str port", - isServiceExternalTrafficPolicyTypeLocal: false, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromInt32(80), - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expected: intstr.FromInt32(80), - }, - { - name: "resolve str port - instance", - isServiceExternalTrafficPolicyTypeLocal: false, - targetType: elbv2model.TargetTypeInstance, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromInt32(80), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expected: intstr.FromInt32(1000), - }, - { - name: "resolve str port - resolves to other str port (error)", - isServiceExternalTrafficPolicyTypeLocal: false, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromString("bar"), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expectErr: true, - }, - { - name: "resolve str port - resolves to other str port but instance mode", - isServiceExternalTrafficPolicyTypeLocal: false, - targetType: elbv2model.TargetTypeInstance, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromString("bar"), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expected: intstr.FromInt32(1000), - }, - { - name: "resolve str port - cant find configured port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetType: elbv2model.TargetTypeInstance, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "baz", - TargetPort: intstr.FromString("bar"), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expectErr: true, - }, - { - name: "with ExternalTrafficPolicyTypeLocal and HealthCheckNodePort specified", - isServiceExternalTrafficPolicyTypeLocal: true, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - HealthCheckNodePort: 32000, - ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, - }, - }, - expected: intstr.FromInt32(32000), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - builder := targetGroupBuilderImpl{} - res, err := builder.buildTargetGroupHealthCheckPort(tc.targetGroupProps, tc.targetType, tc.svc, tc.isServiceExternalTrafficPolicyTypeLocal) - if tc.expectErr { - assert.Error(t, err, res) - return - } - assert.NoError(t, err) - assert.Equal(t, tc.expected, res) - }) - } -} - func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { testCases := []struct { name string + targetType elbv2model.TargetType lbType elbv2model.LoadBalancerType targetGroupProps *elbv2gw.TargetGroupProps tgProtocol elbv2model.Protocol @@ -1437,19 +1195,22 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { }{ { name: "nlb - default", + targetType: elbv2model.TargetTypeInstance, lbType: elbv2model.LoadBalancerTypeNetwork, tgProtocol: elbv2model.ProtocolUDP, expected: elbv2model.ProtocolTCP, }, { name: "alb - default", + targetType: elbv2model.TargetTypeInstance, lbType: elbv2model.LoadBalancerTypeApplication, tgProtocol: elbv2model.ProtocolHTTP, expected: elbv2model.ProtocolHTTP, }, { - name: "specified http", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "specified http", + targetType: elbv2model.TargetTypeInstance, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolHTTP))), @@ -1459,8 +1220,9 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTP, }, { - name: "specified https", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "specified https", + targetType: elbv2model.TargetTypeInstance, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolHTTPS))), @@ -1470,8 +1232,9 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTPS, }, { - name: "specified tcp", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "specified tcp", + targetType: elbv2model.TargetTypeInstance, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolTCP))), @@ -1480,6 +1243,13 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { tgProtocol: elbv2model.ProtocolTCP, expected: elbv2model.ProtocolTCP, }, + { + name: "nlb with alb target default", + targetType: elbv2model.TargetTypeALB, + lbType: elbv2model.LoadBalancerTypeNetwork, + tgProtocol: elbv2model.ProtocolTCP, + expected: elbv2model.ProtocolHTTP, + }, } for _, tc := range testCases { @@ -1488,7 +1258,7 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { loadBalancerType: tc.lbType, } - res := builder.buildTargetGroupHealthCheckProtocol(tc.targetGroupProps, tc.tgProtocol, false) + res := builder.buildTargetGroupHealthCheckProtocol(tc.targetGroupProps, tc.targetType, tc.tgProtocol, false) assert.Equal(t, tc.expected, res) }) } @@ -1862,14 +1632,15 @@ func Test_buildTargetGroupTags(t *testing.T) { Namespace: "test-namespace", } - backend := routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend := routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-namespace", Name: "test-service", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -1877,7 +1648,7 @@ func Test_buildTargetGroupTags(t *testing.T) { Type: intstr.Int, }, }, - } + ) // Create target group props with user tags if specified var tgProps *elbv2gw.TargetGroupProps @@ -1887,7 +1658,7 @@ func Test_buildTargetGroupTags(t *testing.T) { } } - tgSpec, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(gateway, route, elbv2gw.LoadBalancerConfiguration{}, elbv2model.IPAddressTypeIPV4, backend, tgProps) + tgSpec, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(gateway, route, elbv2model.IPAddressTypeIPV4, backend, tgProps) if tc.expectErr { assert.Error(t, err) @@ -1900,6 +1671,123 @@ func Test_buildTargetGroupTags(t *testing.T) { } } +func Test_buildTargetGroupFromGateway(t *testing.T) { + testCases := []struct { + name string + gateway *gwv1.Gateway + listenerPort int32 + lbIPType elbv2model.IPAddressType + route *routeutils.MockRoute + backendConfig *routeutils.GatewayBackendConfig + existingTG bool + expectedTGName string + expectedFrontendData bool + }{ + { + name: "new target group creation", + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-gw", + }, + }, + listenerPort: 80, + lbIPType: elbv2model.IPAddressTypeIPV4, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "test-route", + Namespace: "test-ns", + }, + backendConfig: routeutils.NewGatewayBackendConfig(&gwv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "backend-gw", Namespace: "backend-ns"}}, nil, "arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/test-alb/1234567890123456", 8080), + expectedTGName: "k8s-testns-testrout", + expectedFrontendData: true, + }, + { + name: "existing target group reuse", + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-gw", + }, + }, + listenerPort: 80, + lbIPType: elbv2model.IPAddressTypeIPV4, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "test-route", + Namespace: "test-ns", + }, + backendConfig: routeutils.NewGatewayBackendConfig(&gwv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "backend-gw", Namespace: "backend-ns"}}, nil, "arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/test-alb/1234567890123456", 8080), + existingTG: true, + expectedFrontendData: false, + }, + { + name: "with target group props", + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-gw", + }, + }, + listenerPort: 443, + lbIPType: elbv2model.IPAddressTypeIPV4, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "test-route", + Namespace: "test-ns", + }, + backendConfig: routeutils.NewGatewayBackendConfig(&gwv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "backend-gw", Namespace: "backend-ns"}}, &elbv2gw.TargetGroupProps{ + TargetGroupName: awssdk.String("custom-tg-name"), + }, "arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/test-alb/1234567890123456", 8443), + expectedTGName: "custom-tg-name", + expectedFrontendData: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tagger := &mockTagHelper{ + tags: make(map[string]string), + } + + builder := newTargetGroupBuilder("test-cluster", "vpc-xxx", tagger, elbv2model.LoadBalancerTypeApplication, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), string(elbv2model.TargetTypeALB), nil) + impl := builder.(*targetGroupBuilderImpl) + + stack := core.NewDefaultStack(core.StackID{Namespace: "test", Name: "test"}) + + // Pre-populate existing target group if needed + if tc.existingTG { + tgResID := impl.buildTargetGroupResourceID(k8s.NamespacedName(tc.gateway), tc.backendConfig.GetBackendNamespacedName(), tc.route.GetRouteNamespacedName(), tc.route.GetRouteKind(), tc.backendConfig.GetIdentifierPort()) + existingTG := elbv2model.NewTargetGroup(stack, tgResID, elbv2model.TargetGroupSpec{ + Name: "existing-tg", + }) + impl.tgByResID[tgResID] = existingTG + } + + result, err := impl.buildTargetGroupFromGateway(stack, tc.gateway, tc.listenerPort, tc.lbIPType, tc.route, *tc.backendConfig) + + assert.NoError(t, err) + assert.NotNil(t, result) + + if tc.existingTG { + assert.Equal(t, "existing-tg", result.Spec.Name) + } else { + assert.Contains(t, result.Spec.Name, tc.expectedTGName) + assert.Equal(t, elbv2model.TargetTypeALB, result.Spec.TargetType) + } + + // Check frontend NLB data + if tc.expectedFrontendData { + frontendData, exists := impl.localFrontendNlbData[result.Spec.Name] + assert.True(t, exists) + assert.Equal(t, result.Spec.Name, frontendData.Name) + assert.Equal(t, tc.listenerPort, frontendData.Port) + assert.Equal(t, *result.Spec.Port, frontendData.TargetPort) + } + }) + } +} + func protocolPtr(protocol elbv2gw.Protocol) *elbv2gw.Protocol { return &protocol } diff --git a/pkg/gateway/routeutils/backend.go b/pkg/gateway/routeutils/backend.go index 6923db3dec..1d11f07244 100644 --- a/pkg/gateway/routeutils/backend.go +++ b/pkg/gateway/routeutils/backend.go @@ -3,15 +3,14 @@ package routeutils import ( "context" "fmt" - "strings" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" - "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -19,6 +18,7 @@ import ( const ( serviceKind = "Service" + gatewayKind = "Gateway" referenceGrantNotExists = "No explicit ReferenceGrant exists to allow the reference." maxWeight = 999 ) @@ -27,21 +27,31 @@ var ( tgConfigConstructor = gateway.NewTargetGroupConfigConstructor() ) -type ServiceBackendConfig struct { - Service *corev1.Service - ELBV2TargetGroupProps *elbv2gw.TargetGroupProps - ServicePort *corev1.ServicePort -} - -type LiteralTargetGroupConfig struct { - // GW API limits names to 253 characters, while a TG ARN might be 256, so just using the name. - Name string +// TargetGroupConfigurator defines methods used to construct an ELB target group from a Kubernetes based backend. +type TargetGroupConfigurator interface { + // GetTargetType returns the Target Type to associate with this target group. + GetTargetType(defaultTargetType elbv2model.TargetType) elbv2model.TargetType + // GetTargetGroupProps returns the target group properties associated with this backend + GetTargetGroupProps() *elbv2gw.TargetGroupProps + // GetBackendNamespacedName returns the namespaced name associated with the underlying backend. + GetBackendNamespacedName() types.NamespacedName + // GetIdentifierPort returns the port used when constructing the resource ID for the resource stack. + GetIdentifierPort() intstr.IntOrString + // GetExternalTrafficPolicy returns the external traffic policy for this backend service, if not applicable returns "ServiceExternalTrafficPolicyCluster". + GetExternalTrafficPolicy() corev1.ServiceExternalTrafficPolicyType + // GetIPAddressType returns the Target Group IP address type + GetIPAddressType() elbv2model.TargetGroupIPAddressType + // GetTargetGroupPort returns the port to attach to the Target Group + GetTargetGroupPort(targetType elbv2model.TargetType) int32 + // GetHealthCheckPort returns the port to send health check traffic + GetHealthCheckPort(targetType elbv2model.TargetType, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) } // Backend an abstraction on the Gateway Backend, meant to hide the underlying backend type from consumers (unless they really want to see it :)) type Backend struct { ServiceBackend *ServiceBackendConfig LiteralTargetGroup *LiteralTargetGroupConfig + GatewayBackend *GatewayBackendConfig Weight int } @@ -117,20 +127,23 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, backendRe var serviceBackend *ServiceBackendConfig var literalTargetGroup *LiteralTargetGroupConfig + var gatewayBackend *GatewayBackendConfig var warn error var fatal error // We only support references of type service. - if backendRef.Kind == nil || *backendRef.Kind == "Service" { + if backendRef.Kind == nil || *backendRef.Kind == serviceKind { serviceBackend, warn, fatal = serviceLoader(ctx, k8sClient, routeIdentifier, routeKind, backendRef) - } else if string(*backendRef.Kind) == TargetGroupNameBackend { + } else if string(*backendRef.Kind) == targetGroupNameBackend { literalTargetGroup, warn, fatal = literalTargetGroupLoader(backendRef) + } else if string(*backendRef.Kind) == gatewayKind { + gatewayBackend, warn, fatal = gatewayLoader(ctx, k8sClient, routeIdentifier, routeKind, backendRef) } if warn != nil || fatal != nil { return nil, warn, fatal } - if serviceBackend == nil && literalTargetGroup == nil { + if serviceBackend == nil && literalTargetGroup == nil && gatewayBackend == nil { initialErrorMessage := "Unknown backend reference kind" wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonInvalidKind, &wrappedGatewayErrorMessage, nil), nil @@ -157,126 +170,12 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, backendRe } return &Backend{ ServiceBackend: serviceBackend, + GatewayBackend: gatewayBackend, LiteralTargetGroup: literalTargetGroup, Weight: weight, }, nil, nil } -func serviceLoader(ctx context.Context, k8sClient client.Client, routeIdentifier types.NamespacedName, routeKind RouteKind, backendRef gwv1.BackendRef) (*ServiceBackendConfig, error, error) { - if backendRef.Port == nil { - initialErrorMessage := "Port is required" - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedValue, &wrappedGatewayErrorMessage, nil), nil - } - - var svcNamespace string - if backendRef.Namespace == nil { - svcNamespace = routeIdentifier.Namespace - } else { - svcNamespace = string(*backendRef.Namespace) - } - - svcIdentifier := types.NamespacedName{ - Namespace: svcNamespace, - Name: string(backendRef.Name), - } - - // Check for reference grant when performing cross namespace gateway -> route attachment - if svcNamespace != routeIdentifier.Namespace { - allowed, err := referenceGrantCheck(ctx, k8sClient, svcIdentifier, routeIdentifier, routeKind) - if err != nil { - // Currently, this API only fails for a k8s related error message, hence no status update + make the error fatal. - return nil, nil, errors.Wrapf(err, "Unable to perform reference grant check") - } - - // We should not give any hints about the existence of this resource, therefore, we return nil. - // That way, users can't infer if the route is missing because of a misconfigured service reference - // or the sentence grant is not allowing the connection. - if !allowed { - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(referenceGrantNotExists, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", referenceGrantNotExists), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonRefNotPermitted, &wrappedGatewayErrorMessage, nil), nil - } - } - - svc := &corev1.Service{} - err := k8sClient.Get(ctx, svcIdentifier, svc) - if err != nil { - - convertToNotFoundError := client.IgnoreNotFound(err) - - if convertToNotFoundError == nil { - // Svc not found, post an updated status. - initialErrorMessage := fmt.Sprintf("Service (%s:%s) not found)", svcIdentifier.Namespace, svcIdentifier.Name) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil - } - // Otherwise, general error. No need for status update. - return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) - } - - // TODO -- This should be updated, to handle UDP and TCP on the same service port. - // Currently, it will just arbitrarily take one. - - var servicePort *corev1.ServicePort - - for _, svcPort := range svc.Spec.Ports { - if svcPort.Port == int32(*backendRef.Port) { - servicePort = &svcPort - break - } - } - - if servicePort == nil { - initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil - } - - tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc)) - - if err != nil { - // As of right now, this error can only be thrown because of a k8s api error hence no status update. - return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) - } - - var tgProps *elbv2gw.TargetGroupProps - - if tgConfig != nil { - tgProps = tgConfigConstructor.ConstructTargetGroupConfigForRoute(tgConfig, routeIdentifier.Name, routeIdentifier.Namespace, string(routeKind)) - } - - // validate if protocol version is compatible with appProtocol - if tgProps != nil && servicePort.AppProtocol != nil { - appProtocol := strings.ToLower(*servicePort.AppProtocol) - if tgProps.ProtocolVersion != nil { - isCompatible := true - switch *tgProps.ProtocolVersion { - case elbv2gw.ProtocolVersionGRPC: - if appProtocol == "http" { - isCompatible = false - } - case elbv2gw.ProtocolVersionHTTP1, elbv2gw.ProtocolVersionHTTP2: - if appProtocol == "grpc" { - isCompatible = false - } - } - if !isCompatible { - initialErrorMessage := fmt.Sprintf("Service port appProtocol %s is not compatible with target group protocolVersion %s", *servicePort.AppProtocol, *tgProps.ProtocolVersion) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - - // This potentially could be fatal, but let's make the reconcile cycle as resilient as possible. - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedProtocol, &wrappedGatewayErrorMessage, nil), nil - } - } - } - - return &ServiceBackendConfig{ - Service: svc, - ServicePort: servicePort, - ELBV2TargetGroupProps: tgProps, - }, nil, nil -} - func literalTargetGroupLoader(backendRef gwv1.BackendRef) (*LiteralTargetGroupConfig, error, error) { return &LiteralTargetGroupConfig{ Name: string(backendRef.Name), @@ -285,22 +184,30 @@ func literalTargetGroupLoader(backendRef gwv1.BackendRef) (*LiteralTargetGroupCo // LookUpTargetGroupConfiguration given a service, lookup the target group configuration associated with the service. // recall that target group configuration always lives within the same namespace as the service. -func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, serviceMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) { +func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, objectKind string, objectMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) { tgConfigList := &elbv2gw.TargetGroupConfigurationList{} // TODO - Add index - if err := k8sClient.List(ctx, tgConfigList, client.InNamespace(serviceMetadata.Namespace)); err != nil { + if err := k8sClient.List(ctx, tgConfigList, client.InNamespace(objectMetadata.Namespace)); err != nil { return nil, err } for _, tgConfig := range tgConfigList.Items { - if tgConfig.Spec.TargetReference.Kind != nil && *tgConfig.Spec.TargetReference.Kind != serviceKind { + + var isEligible bool + // Special case, nil kind == Service. + if tgConfig.Spec.TargetReference.Kind == nil && objectKind == serviceKind { + isEligible = true + } else if tgConfig.Spec.TargetReference.Kind != nil && objectKind == *tgConfig.Spec.TargetReference.Kind { + isEligible = true + } + + if !isEligible { continue } - // TODO - Add a webhook to validate that only one target group config references this service. // TODO - Add an index for this - if tgConfig.Spec.TargetReference.Name == serviceMetadata.Name { + if tgConfig.Spec.TargetReference.Name == objectMetadata.Name { return &tgConfig, nil } } @@ -309,9 +216,9 @@ func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client // Implements the reference grant API // https://gateway-api.sigs.k8s.io/api-types/referencegrant/ -func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind RouteKind) (bool, error) { +func referenceGrantCheck(ctx context.Context, k8sClient client.Client, objKind string, objIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind RouteKind) (bool, error) { referenceGrantList := &gwbeta1.ReferenceGrantList{} - if err := k8sClient.List(ctx, referenceGrantList, client.InNamespace(svcIdentifier.Namespace)); err != nil { + if err := k8sClient.List(ctx, referenceGrantList, client.InNamespace(objIdentifier.Namespace)); err != nil { return false, err } @@ -328,13 +235,13 @@ func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdenti if routeAllowed { for _, to := range grant.Spec.To { - // As this is a backend reference, we only care about the "Service" Kind. - if to.Kind != serviceKind { + // Make sure the kind is correct for our query. + if string(to.Kind) != objKind { continue } // If name is specified, we need to ensure that svc name matches the "to" name. - if to.Name != nil && string(*to.Name) != svcIdentifier.Name { + if to.Name != nil && string(*to.Name) != objIdentifier.Name { continue } diff --git a/pkg/gateway/routeutils/backend_gateway.go b/pkg/gateway/routeutils/backend_gateway.go new file mode 100644 index 0000000000..4fcc3be7ce --- /dev/null +++ b/pkg/gateway/routeutils/backend_gateway.go @@ -0,0 +1,184 @@ +package routeutils + +import ( + "context" + "fmt" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" +) + +var _ TargetGroupConfigurator = &GatewayBackendConfig{} + +type GatewayBackendConfig struct { + gateway *gwv1.Gateway + targetGroupProps *elbv2gw.TargetGroupProps + arn string + port int32 +} + +func NewGatewayBackendConfig(gateway *gwv1.Gateway, targetGroupProps *elbv2gw.TargetGroupProps, arn string, port int32) *GatewayBackendConfig { + return &GatewayBackendConfig{ + gateway: gateway, + targetGroupProps: targetGroupProps, + arn: arn, + port: port, + } +} + +func (g *GatewayBackendConfig) GetALBARN() string { + return g.arn +} + +func (g *GatewayBackendConfig) GetTargetType(_ elbv2model.TargetType) elbv2model.TargetType { + return elbv2model.TargetTypeALB +} + +func (g *GatewayBackendConfig) GetTargetGroupProps() *elbv2gw.TargetGroupProps { + return g.targetGroupProps +} + +func (g *GatewayBackendConfig) GetBackendNamespacedName() types.NamespacedName { + return k8s.NamespacedName(g.gateway) +} + +func (g *GatewayBackendConfig) GetIdentifierPort() intstr.IntOrString { + return intstr.FromInt32(g.port) +} + +// GetExternalTrafficPolicy doesn't really apply to this backend type, so we return the most permissive type. +func (g *GatewayBackendConfig) GetExternalTrafficPolicy() corev1.ServiceExternalTrafficPolicyType { + return corev1.ServiceExternalTrafficPolicyTypeCluster +} + +// GetIPAddressType Gateway based backends always communicate over IPv4. +func (g *GatewayBackendConfig) GetIPAddressType() elbv2model.TargetGroupIPAddressType { + return elbv2model.TargetGroupIPAddressTypeIPv4 +} + +// GetTargetGroupPort Gateway based backends always forward traffic to the Gateway listener. +func (g *GatewayBackendConfig) GetTargetGroupPort(_ elbv2model.TargetType) int32 { + return g.port +} + +func (g *GatewayBackendConfig) GetHealthCheckPort(_ elbv2model.TargetType, _ bool) (intstr.IntOrString, error) { + portConfigNotExist := g.targetGroupProps == nil || g.targetGroupProps.HealthCheckConfig == nil || g.targetGroupProps.HealthCheckConfig.HealthCheckPort == nil + + if portConfigNotExist || *g.targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil + } + + return intstr.FromInt32(g.port), nil +} + +func gatewayLoader(ctx context.Context, k8sClient client.Client, routeIdentifier types.NamespacedName, routeKind RouteKind, backendRef gwv1.BackendRef) (*GatewayBackendConfig, error, error) { + if backendRef.Port == nil { + initialErrorMessage := "Port is required" + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedValue, &wrappedGatewayErrorMessage, nil), nil + } + + var gwNamespace string + if backendRef.Namespace == nil { + gwNamespace = routeIdentifier.Namespace + } else { + gwNamespace = string(*backendRef.Namespace) + } + + gwIdentifier := types.NamespacedName{ + Namespace: gwNamespace, + Name: string(backendRef.Name), + } + + // Check for reference grant when performing cross namespace gateway -> route attachment + if gwIdentifier.Namespace != routeIdentifier.Namespace { + allowed, err := referenceGrantCheck(ctx, k8sClient, gatewayKind, gwIdentifier, routeIdentifier, routeKind) + if err != nil { + // Currently, this API only fails for a k8s related error message, hence no status update + make the error fatal. + return nil, nil, errors.Wrapf(err, "Unable to perform reference grant check") + } + + // We should not give any hints about the existence of this resource, therefore, we return nil. + // That way, users can't infer if the route is missing because of a misconfigured gateway reference + // or the sentence grant is not allowing the connection. + if !allowed { + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(referenceGrantNotExists, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", referenceGrantNotExists), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonRefNotPermitted, &wrappedGatewayErrorMessage, nil), nil + } + } + + gw := &gwv1.Gateway{} + err := k8sClient.Get(ctx, gwIdentifier, gw) + if err != nil { + + convertToNotFoundError := client.IgnoreNotFound(err) + + if convertToNotFoundError == nil { + // Svc not found, post an updated status. + initialErrorMessage := fmt.Sprintf("Gateway (%s:%s) not found)", gwIdentifier.Namespace, gwIdentifier.Name) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + // Otherwise, general error. No need for status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch gw object %+v", gw)) + } + + tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, gatewayKind, k8s.NamespacedName(gw)) + + if err != nil { + // As of right now, this error can only be thrown because of a k8s api error hence no status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) + } + + var tgProps *elbv2gw.TargetGroupProps + + if tgConfig != nil { + tgProps = tgConfigConstructor.ConstructTargetGroupConfigForRoute(tgConfig, routeIdentifier.Name, routeIdentifier.Namespace, string(routeKind)) + } + + var arn string + + // Find the ALB ARN within the Gateway Programmed Condition, the controller will always embed the ARN there. + for _, cond := range gw.Status.Conditions { + if cond.Type == string(gwv1.GatewayConditionProgrammed) { + if cond.Status == metav1.ConditionTrue { + arn = cond.Message + break + } + } + } + + if arn == "" { + // If the ARN is not available, then the backend is not yet usable. + initialErrorMessage := fmt.Sprintf("Gateway (%s:%s) is not usable yet, LB ARN is not provisioned)", gwIdentifier.Namespace, gwIdentifier.Name) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + // This needs to be a fatal error, otherwise we will not run another reconcile cycle to pick up the ARN. + return nil, nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil) + } + + err = validateGatewayARN(arn) + if err != nil { + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(err.Error(), routeKind, routeIdentifier) + // This can be a warning, as we know that retrying reconcile will do nothing to fix this situation. + return nil, wrapError(err, gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + + return NewGatewayBackendConfig(gw, tgProps, arn, int32(*backendRef.Port)), nil, nil +} + +func validateGatewayARN(arn string) error { + parts := strings.Split(arn, "/") + if len(parts) < 2 || parts[1] != "app" { + return errors.Errorf("invalid gateway ARN: %s, the resource type must be application load balancer", arn) + } + return nil +} diff --git a/pkg/gateway/routeutils/backend_gateway_test.go b/pkg/gateway/routeutils/backend_gateway_test.go new file mode 100644 index 0000000000..1c32eccd75 --- /dev/null +++ b/pkg/gateway/routeutils/backend_gateway_test.go @@ -0,0 +1,250 @@ +package routeutils + +import ( + "testing" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestGatewayBackendConfig_GetTargetType(t *testing.T) { + config := &GatewayBackendConfig{} + + tests := []struct { + name string + defaultType elbv2model.TargetType + expectedType elbv2model.TargetType + }{ + { + name: "returns ALB regardless of default", + defaultType: elbv2model.TargetTypeInstance, + expectedType: elbv2model.TargetTypeALB, + }, + { + name: "returns ALB with IP default", + defaultType: elbv2model.TargetTypeIP, + expectedType: elbv2model.TargetTypeALB, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := config.GetTargetType(tt.defaultType) + assert.Equal(t, tt.expectedType, result) + }) + } +} + +func TestGatewayBackendConfig_GetTargetGroupProps(t *testing.T) { + props := &elbv2gw.TargetGroupProps{} + config := &GatewayBackendConfig{targetGroupProps: props} + + assert.Equal(t, props, config.GetTargetGroupProps()) +} + +func TestGatewayBackendConfig_GetBackendNamespacedName(t *testing.T) { + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", + Namespace: "test-namespace", + }, + } + config := &GatewayBackendConfig{gateway: gateway} + + expected := types.NamespacedName{ + Name: "test-gateway", + Namespace: "test-namespace", + } + + assert.Equal(t, expected, config.GetBackendNamespacedName()) +} + +func TestGatewayBackendConfig_GetIdentifierPort(t *testing.T) { + config := &GatewayBackendConfig{port: 8080} + + expected := intstr.FromInt32(8080) + assert.Equal(t, expected, config.GetIdentifierPort()) +} + +func TestGatewayBackendConfig_GetExternalTrafficPolicy(t *testing.T) { + config := &GatewayBackendConfig{} + + result := config.GetExternalTrafficPolicy() + assert.Equal(t, corev1.ServiceExternalTrafficPolicyTypeCluster, result) +} + +func TestGatewayBackendConfig_GetIPAddressType(t *testing.T) { + config := &GatewayBackendConfig{} + + result := config.GetIPAddressType() + assert.Equal(t, elbv2model.TargetGroupIPAddressTypeIPv4, result) +} + +func TestGatewayBackendConfig_GetTargetGroupPort(t *testing.T) { + config := &GatewayBackendConfig{port: 9090} + + tests := []struct { + name string + targetType elbv2model.TargetType + expected int32 + }{ + { + name: "ALB target type", + targetType: elbv2model.TargetTypeALB, + expected: 9090, + }, + { + name: "Instance target type", + targetType: elbv2model.TargetTypeInstance, + expected: 9090, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := config.GetTargetGroupPort(tt.targetType) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGatewayBackendConfig_GetHealthCheckPort(t *testing.T) { + tests := []struct { + name string + targetGroupProps *elbv2gw.TargetGroupProps + port int32 + targetType elbv2model.TargetType + useNodePort bool + expectedPort intstr.IntOrString + expectedError bool + }{ + { + name: "no target group props", + targetGroupProps: nil, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "no health check config", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: nil, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "no health check port config", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: nil, + }, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "health check port set to traffic-port", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), + }, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "health check port set to custom value", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("9090"), + }, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromInt32(8080), + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := &GatewayBackendConfig{ + targetGroupProps: tt.targetGroupProps, + port: tt.port, + } + + result, err := config.GetHealthCheckPort(tt.targetType, tt.useNodePort) + + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedPort, result) + } + }) + } +} + +func TestValidateGatewayARN(t *testing.T) { + tests := []struct { + name string + arn string + wantErr bool + }{ + { + name: "valid ALB ARN", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer/app/k8s-echoserv-testgwal-3c92fc24ed/9604d5627427405c", + wantErr: false, + }, + { + name: "invalid NLB ARN", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer/net/my-nlb/1234567890123456", + wantErr: true, + }, + { + name: "invalid format - no slashes", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer", + wantErr: true, + }, + { + name: "invalid format - only one part", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer/", + wantErr: true, + }, + { + name: "empty string", + arn: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateGatewayARN(tt.arn) + if (err != nil) != tt.wantErr { + t.Errorf("validateGatewayARN() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/gateway/routeutils/backend_literal.go b/pkg/gateway/routeutils/backend_literal.go new file mode 100644 index 0000000000..3693bb6cfb --- /dev/null +++ b/pkg/gateway/routeutils/backend_literal.go @@ -0,0 +1,6 @@ +package routeutils + +type LiteralTargetGroupConfig struct { + // GW API limits names to 253 characters, while a TG ARN might be 256, so just using the name. + Name string +} diff --git a/pkg/gateway/routeutils/backend_service.go b/pkg/gateway/routeutils/backend_service.go new file mode 100644 index 0000000000..295473bf7b --- /dev/null +++ b/pkg/gateway/routeutils/backend_service.go @@ -0,0 +1,232 @@ +package routeutils + +import ( + "context" + "fmt" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" +) + +type ServiceBackendConfig struct { + service *corev1.Service + targetGroupProps *elbv2gw.TargetGroupProps + servicePort *corev1.ServicePort +} + +var _ TargetGroupConfigurator = &ServiceBackendConfig{} + +func NewServiceBackendConfig(service *corev1.Service, targetGroupProps *elbv2gw.TargetGroupProps, servicePort *corev1.ServicePort) *ServiceBackendConfig { + return &ServiceBackendConfig{ + service: service, + targetGroupProps: targetGroupProps, + servicePort: servicePort, + } +} + +func (s *ServiceBackendConfig) GetTargetType(defaultTargetType elbv2model.TargetType) elbv2model.TargetType { + if s.targetGroupProps == nil || s.targetGroupProps.TargetType == nil { + return defaultTargetType + } + + return elbv2model.TargetType(*s.targetGroupProps.TargetType) +} + +func (s *ServiceBackendConfig) GetHealthCheckPort(targetType elbv2model.TargetType, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) { + portConfigNotExist := s.targetGroupProps == nil || s.targetGroupProps.HealthCheckConfig == nil || s.targetGroupProps.HealthCheckConfig.HealthCheckPort == nil + + if portConfigNotExist && isServiceExternalTrafficPolicyTypeLocal { + return intstr.FromInt32(s.service.Spec.HealthCheckNodePort), nil + } + + if portConfigNotExist || *s.targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil + } + + healthCheckPort := intstr.Parse(*s.targetGroupProps.HealthCheckConfig.HealthCheckPort) + if healthCheckPort.Type == intstr.Int { + return healthCheckPort, nil + } + hcSvcPort, err := k8s.LookupServicePort(s.service, healthCheckPort) + if err != nil { + return intstr.FromString(""), err + } + + if targetType == elbv2model.TargetTypeInstance { + return intstr.FromInt(int(hcSvcPort.NodePort)), nil + } + + if hcSvcPort.TargetPort.Type == intstr.Int { + return hcSvcPort.TargetPort, nil + } + return intstr.IntOrString{}, errors.New("cannot use named healthCheckPort for IP TargetType when service's targetPort is a named port") +} + +// GetTargetGroupPort constructs the TargetGroup's port. +// Note: TargetGroup's port is not in the data path as we always register targets with port specified. +// so these settings don't really matter to our controller, +// and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing. + +func (s *ServiceBackendConfig) GetTargetGroupPort(targetType elbv2model.TargetType) int32 { + if targetType == elbv2model.TargetTypeInstance { + return s.servicePort.NodePort + } + if s.servicePort.TargetPort.Type == intstr.Int { + return int32(s.servicePort.TargetPort.IntValue()) + } + // If all else fails, return 1 as alluded to above, this setting doesn't really matter. + return 1 +} + +func (s *ServiceBackendConfig) GetIPAddressType() elbv2model.TargetGroupIPAddressType { + var ipv6Configured bool + for _, ipFamily := range s.service.Spec.IPFamilies { + if ipFamily == corev1.IPv6Protocol { + ipv6Configured = true + break + } + } + if ipv6Configured { + return elbv2model.TargetGroupIPAddressTypeIPv6 + } + return elbv2model.TargetGroupIPAddressTypeIPv4 +} + +func (s *ServiceBackendConfig) GetExternalTrafficPolicy() corev1.ServiceExternalTrafficPolicyType { + return s.service.Spec.ExternalTrafficPolicy +} + +func (s *ServiceBackendConfig) GetServicePort() *corev1.ServicePort { + return s.servicePort +} + +func (s *ServiceBackendConfig) GetIdentifierPort() intstr.IntOrString { + return s.servicePort.TargetPort +} + +func (s *ServiceBackendConfig) GetBackendNamespacedName() types.NamespacedName { + return k8s.NamespacedName(s.service) +} + +func (s *ServiceBackendConfig) GetTargetGroupProps() *elbv2gw.TargetGroupProps { + return s.targetGroupProps +} + +func serviceLoader(ctx context.Context, k8sClient client.Client, routeIdentifier types.NamespacedName, routeKind RouteKind, backendRef gwv1.BackendRef) (*ServiceBackendConfig, error, error) { + if backendRef.Port == nil { + initialErrorMessage := "Port is required" + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedValue, &wrappedGatewayErrorMessage, nil), nil + } + + var svcNamespace string + if backendRef.Namespace == nil { + svcNamespace = routeIdentifier.Namespace + } else { + svcNamespace = string(*backendRef.Namespace) + } + + svcIdentifier := types.NamespacedName{ + Namespace: svcNamespace, + Name: string(backendRef.Name), + } + + // Check for reference grant when performing cross namespace gateway -> route attachment + if svcNamespace != routeIdentifier.Namespace { + allowed, err := referenceGrantCheck(ctx, k8sClient, serviceKind, svcIdentifier, routeIdentifier, routeKind) + if err != nil { + // Currently, this API only fails for a k8s related error message, hence no status update + make the error fatal. + return nil, nil, errors.Wrapf(err, "Unable to perform reference grant check") + } + + // We should not give any hints about the existence of this resource, therefore, we return nil. + // That way, users can't infer if the route is missing because of a misconfigured service reference + // or the sentence grant is not allowing the connection. + if !allowed { + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(referenceGrantNotExists, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", referenceGrantNotExists), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonRefNotPermitted, &wrappedGatewayErrorMessage, nil), nil + } + } + + svc := &corev1.Service{} + err := k8sClient.Get(ctx, svcIdentifier, svc) + if err != nil { + + convertToNotFoundError := client.IgnoreNotFound(err) + + if convertToNotFoundError == nil { + // Svc not found, post an updated status. + initialErrorMessage := fmt.Sprintf("Service (%s:%s) not found)", svcIdentifier.Namespace, svcIdentifier.Name) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + // Otherwise, general error. No need for status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) + } + + // TODO -- This should be updated, to handle UDP and TCP on the same service port. + // Currently, it will just arbitrarily take one. + + var servicePort *corev1.ServicePort + + for _, svcPort := range svc.Spec.Ports { + if svcPort.Port == int32(*backendRef.Port) { + servicePort = &svcPort + break + } + } + + if servicePort == nil { + initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + + tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, serviceKind, k8s.NamespacedName(svc)) + + if err != nil { + // As of right now, this error can only be thrown because of a k8s api error hence no status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) + } + + var tgProps *elbv2gw.TargetGroupProps + + if tgConfig != nil { + tgProps = tgConfigConstructor.ConstructTargetGroupConfigForRoute(tgConfig, routeIdentifier.Name, routeIdentifier.Namespace, string(routeKind)) + } + + // validate if protocol version is compatible with appProtocol + if tgProps != nil && servicePort.AppProtocol != nil { + appProtocol := strings.ToLower(*servicePort.AppProtocol) + if tgProps.ProtocolVersion != nil { + isCompatible := true + switch *tgProps.ProtocolVersion { + case elbv2gw.ProtocolVersionGRPC: + if appProtocol == "http" { + isCompatible = false + } + case elbv2gw.ProtocolVersionHTTP1, elbv2gw.ProtocolVersionHTTP2: + if appProtocol == "grpc" { + isCompatible = false + } + } + if !isCompatible { + initialErrorMessage := fmt.Sprintf("Service port appProtocol %s is not compatible with target group protocolVersion %s", *servicePort.AppProtocol, *tgProps.ProtocolVersion) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + + // This potentially could be fatal, but let's make the reconcile cycle as resilient as possible. + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedProtocol, &wrappedGatewayErrorMessage, nil), nil + } + } + } + + return NewServiceBackendConfig(svc, tgProps, servicePort), nil, nil +} diff --git a/pkg/gateway/routeutils/backend_service_test.go b/pkg/gateway/routeutils/backend_service_test.go new file mode 100644 index 0000000000..f3c91d93a8 --- /dev/null +++ b/pkg/gateway/routeutils/backend_service_test.go @@ -0,0 +1,266 @@ +package routeutils + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + "testing" +) + +func Test_buildTargetGroupPort(t *testing.T) { + testCases := []struct { + name string + targetType elbv2model.TargetType + svcPort *corev1.ServicePort + expected int32 + }{ + { + name: "instance", + svcPort: &corev1.ServicePort{ + NodePort: 8080, + }, + targetType: elbv2model.TargetTypeInstance, + expected: 8080, + }, + { + name: "instance - no node port", + svcPort: &corev1.ServicePort{}, + targetType: elbv2model.TargetTypeInstance, + expected: 0, + }, + { + name: "ip", + svcPort: &corev1.ServicePort{ + NodePort: 8080, + TargetPort: intstr.FromInt32(80), + }, + targetType: elbv2model.TargetTypeIP, + expected: 80, + }, + { + name: "ip - str port", + svcPort: &corev1.ServicePort{ + NodePort: 8080, + TargetPort: intstr.FromString("foo"), + }, + targetType: elbv2model.TargetTypeIP, + expected: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + svcBackend := NewServiceBackendConfig(nil, nil, tc.svcPort) + res := svcBackend.GetTargetGroupPort(tc.targetType) + assert.Equal(t, res, tc.expected) + }) + } +} + +func Test_buildTargetGroupHealthCheckPort(t *testing.T) { + testCases := []struct { + name string + isServiceExternalTrafficPolicyTypeLocal bool + targetGroupProps *elbv2gw.TargetGroupProps + targetType elbv2model.TargetType + svc *corev1.Service + expected intstr.IntOrString + expectErr bool + }{ + { + name: "nil props", + isServiceExternalTrafficPolicyTypeLocal: false, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "nil hc props", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{}, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "nil hc port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{}, + }, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "explicit is use traffic port hc port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), + }, + }, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "explicit port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("80"), + }, + }, + expected: intstr.FromInt32(80), + }, + { + name: "resolve str port", + isServiceExternalTrafficPolicyTypeLocal: false, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromInt32(80), + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(80), + }, + { + name: "resolve str port - instance", + isServiceExternalTrafficPolicyTypeLocal: false, + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromInt32(80), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(1000), + }, + { + name: "resolve str port - resolves to other str port (error)", + isServiceExternalTrafficPolicyTypeLocal: false, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expectErr: true, + }, + { + name: "resolve str port - resolves to other str port but instance mode", + isServiceExternalTrafficPolicyTypeLocal: false, + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(1000), + }, + { + name: "resolve str port - cant find configured port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "baz", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expectErr: true, + }, + { + name: "with ExternalTrafficPolicyTypeLocal and HealthCheckNodePort specified", + isServiceExternalTrafficPolicyTypeLocal: true, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + HealthCheckNodePort: 32000, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + }, + }, + expected: intstr.FromInt32(32000), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + svcBackend := NewServiceBackendConfig(tc.svc, tc.targetGroupProps, nil) + res, err := svcBackend.GetHealthCheckPort(tc.targetType, tc.isServiceExternalTrafficPolicyTypeLocal) + if tc.expectErr { + assert.Error(t, err, res) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupTargetType(t *testing.T) { + svcBackend := NewServiceBackendConfig(nil, nil, nil) + + res := svcBackend.GetTargetType(elbv2model.TargetTypeIP) + assert.Equal(t, elbv2model.TargetTypeIP, res) + + svcBackendEmptyProps := NewServiceBackendConfig(nil, &elbv2gw.TargetGroupProps{}, nil) + + res = svcBackend.GetTargetType(elbv2model.TargetTypeIP) + + res = svcBackendEmptyProps.GetTargetType(elbv2model.TargetTypeIP) + assert.Equal(t, elbv2model.TargetTypeIP, res) + + inst := elbv2gw.TargetTypeInstance + svcBackendWithProps := NewServiceBackendConfig(nil, &elbv2gw.TargetGroupProps{ + TargetType: &inst, + }, nil) + res = svcBackendWithProps.GetTargetType(elbv2model.TargetTypeIP) + assert.Equal(t, elbv2model.TargetTypeInstance, res) +} diff --git a/pkg/gateway/routeutils/backend_test.go b/pkg/gateway/routeutils/backend_test.go index 889bdbf6d5..2d2f4fe2ec 100644 --- a/pkg/gateway/routeutils/backend_test.go +++ b/pkg/gateway/routeutils/backend_test.go @@ -390,14 +390,14 @@ func TestCommonBackendLoader_Service(t *testing.T) { return } - assert.Equal(t, tc.storedService, result.ServiceBackend.Service) + assert.Equal(t, tc.storedService, result.ServiceBackend.service) assert.Equal(t, tc.weight, result.Weight) - assert.Equal(t, tc.servicePort, result.ServiceBackend.ServicePort.Port) + assert.Equal(t, tc.servicePort, result.ServiceBackend.servicePort.Port) if tc.expectedTargetGroup == nil { - assert.Nil(t, result.ServiceBackend.ELBV2TargetGroupProps) + assert.Nil(t, result.ServiceBackend.targetGroupProps) } else { - assert.Equal(t, tc.expectedTargetGroup, result.ServiceBackend.ELBV2TargetGroupProps) + assert.Equal(t, tc.expectedTargetGroup, result.ServiceBackend.targetGroupProps) } }) } @@ -425,7 +425,7 @@ func TestCommonBackendLoader_TargetGroupName(t *testing.T) { name: "valid name", backendRef: gwv1.BackendRef{ BackendObjectReference: gwv1.BackendObjectReference{ - Kind: (*gwv1.Kind)(awssdk.String(TargetGroupNameBackend)), + Kind: (*gwv1.Kind)(awssdk.String(targetGroupNameBackend)), Name: "foo", }, }, @@ -462,12 +462,14 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { testCases := []struct { name string allTargetGroupConfigurations []elbv2gw.TargetGroupConfiguration - serviceMetadata types.NamespacedName + objectMetadata types.NamespacedName + kind string expectErr bool expectedTGConfiguration *elbv2gw.TargetGroupConfiguration }{ { - name: "happy path, exactly one tg config", + name: "happy path, exactly one tg config - service", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -482,7 +484,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -499,8 +501,43 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, + { + name: "happy path, exactly one tg config - gateway", + kind: gatewayKind, + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(gatewayKind), + Name: "svc1", + }, + }, + }, + }, + objectMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + expectedTGConfiguration: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(gatewayKind), + Name: "svc1", + }, + }, + }, + }, { name: "happy path, exactly one tg config (kind not specified)", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -514,7 +551,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -532,6 +569,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, { name: "sad path, svc name different", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -546,13 +584,14 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, }, { name: "sad path, kind not supported", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -567,13 +606,14 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, }, { name: "sad path, many tg none match", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -612,7 +652,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -620,7 +660,8 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, { name: "sad path, no tg none match", - serviceMetadata: types.NamespacedName{ + kind: serviceKind, + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -636,7 +677,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { assert.NoError(t, err) } - result, err := LookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.serviceMetadata) + result, err := LookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.kind, tc.objectMetadata) if tc.expectErr { assert.Error(t, err) @@ -655,16 +696,18 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { func Test_referenceGrantCheck(t *testing.T) { kind := HTTPRouteKind testCases := []struct { - name string - referenceGrants []gwbeta1.ReferenceGrant - svcIdentifier types.NamespacedName - routeIdentifier types.NamespacedName - expected bool - expectErr bool + name string + kind string + referenceGrants []gwbeta1.ReferenceGrant + objectIdentifier types.NamespacedName + routeIdentifier types.NamespacedName + expected bool + expectErr bool }{ { - name: "happy path", - svcIdentifier: types.NamespacedName{ + name: "happy path - service", + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -696,9 +739,45 @@ func Test_referenceGrantCheck(t *testing.T) { }, expected: true, }, + { + name: "happy path - gateway", + kind: gatewayKind, + objectIdentifier: types.NamespacedName{ + Namespace: "gw-namespace", + Name: "gw-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "gw-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: gatewayKind, + Name: (*gwbeta1.ObjectName)(awssdk.String("gw-name")), + }, + }, + }, + }, + }, + expected: true, + }, { name: "happy path (no name equals wildcard)", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -731,7 +810,8 @@ func Test_referenceGrantCheck(t *testing.T) { }, { name: "no grants, should not allow", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -743,7 +823,8 @@ func Test_referenceGrantCheck(t *testing.T) { }, { name: "from is allowed, but not to", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -777,7 +858,8 @@ func Test_referenceGrantCheck(t *testing.T) { }, { name: "to is allowed, but not from", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -808,6 +890,41 @@ func Test_referenceGrantCheck(t *testing.T) { }, expected: false, }, + { + name: "reference grant is for wrong type", + kind: gatewayKind, + objectIdentifier: types.NamespacedName{ + Namespace: "gw-namespace", + Name: "gw-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "gw-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + Name: (*gwbeta1.ObjectName)(awssdk.String("gw-name")), + }, + }, + }, + }, + }, + expected: false, + }, } for _, tc := range testCases { @@ -818,7 +935,7 @@ func Test_referenceGrantCheck(t *testing.T) { assert.NoError(t, err) } - result, err := referenceGrantCheck(context.Background(), k8sClient, tc.svcIdentifier, tc.routeIdentifier, kind) + result, err := referenceGrantCheck(context.Background(), k8sClient, tc.kind, tc.objectIdentifier, tc.routeIdentifier, kind) if tc.expectErr { assert.Error(t, err) return diff --git a/pkg/gateway/routeutils/constants.go b/pkg/gateway/routeutils/constants.go index 7a18b048b8..956339188e 100644 --- a/pkg/gateway/routeutils/constants.go +++ b/pkg/gateway/routeutils/constants.go @@ -19,7 +19,7 @@ const ( ) const ( - TargetGroupNameBackend string = "TargetGroupName" + targetGroupNameBackend string = "TargetGroupName" ) // RouteKind to Route Loader. These functions will pull data directly from the kube api or local cache. diff --git a/pkg/gateway/routeutils/utils.go b/pkg/gateway/routeutils/utils.go index 7a148d5584..619d446516 100644 --- a/pkg/gateway/routeutils/utils.go +++ b/pkg/gateway/routeutils/utils.go @@ -79,6 +79,9 @@ func FilterRoutesBySvc(routes []preLoadRouteDescriptor, svc *corev1.Service) []p // Assuming we are only supporting services as backendRefs on Routes func isServiceReferredByRoute(route preLoadRouteDescriptor, svcID types.NamespacedName) bool { for _, backendRef := range route.GetBackendRefs() { + if backendRef.Kind != nil && *backendRef.Kind != "Service" { + continue + } namespace := route.GetRouteNamespacedName().Namespace if backendRef.Namespace != nil { namespace = string(*backendRef.Namespace) diff --git a/pkg/ingress/model_build_frontend_nlb.go b/pkg/ingress/model_build_frontend_nlb.go index 08fc16f7f4..1ef2a9abac 100644 --- a/pkg/ingress/model_build_frontend_nlb.go +++ b/pkg/ingress/model_build_frontend_nlb.go @@ -561,7 +561,13 @@ func (t *defaultModelBuildTask) buildFrontendNlbListenerSpec(ctx context.Context defaultActions := t.buildFrontendNlbListenerDefaultActions(ctx, targetGroup) - t.frontendNlbTargetGroupDesiredState.AddTargetGroup(targetGroup.Spec.Name, targetGroup.TargetGroupARN(), t.loadBalancer.LoadBalancerARN(), *targetGroup.Spec.Port, config.TargetPort) + t.localFrontendNlbData[targetGroup.Spec.Name] = &elbv2model.FrontendNlbTargetGroupState{ + Name: targetGroup.Spec.Name, + ARN: targetGroup.TargetGroupARN(), + Port: port, + TargetARN: t.loadBalancer.LoadBalancerARN(), + TargetPort: config.TargetPort, + } return elbv2model.ListenerSpec{ LoadBalancerARN: t.frontendNlb.LoadBalancerARN(), diff --git a/pkg/ingress/model_build_frontend_nlb_test.go b/pkg/ingress/model_build_frontend_nlb_test.go index 7fce2e7f92..c7bde65ae1 100644 --- a/pkg/ingress/model_build_frontend_nlb_test.go +++ b/pkg/ingress/model_build_frontend_nlb_test.go @@ -1321,20 +1321,17 @@ func Test_defaultModelBuildTask_buildFrontendNlbListeners(t *testing.T) { t.Run(tt.name, func(t *testing.T) { stack := core.NewDefaultStack(core.StackID{Name: "awesome-stack"}) - desiredState := &core.FrontendNlbTargetGroupDesiredState{ - TargetGroups: make(map[string]*core.FrontendNlbTargetGroupState), - } mockLoadBalancer := elbv2model.NewLoadBalancer(stack, "FrontendNlb", elbv2model.LoadBalancerSpec{ IPAddressType: elbv2model.IPAddressTypeIPV4, }) task := &defaultModelBuildTask{ - ingGroup: tt.ingGroup, - annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), - loadBalancer: tt.loadBalancer, - frontendNlb: mockLoadBalancer, - stack: stack, - frontendNlbTargetGroupDesiredState: desiredState, + ingGroup: tt.ingGroup, + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + loadBalancer: tt.loadBalancer, + frontendNlb: mockLoadBalancer, + stack: stack, + localFrontendNlbData: make(map[string]*elbv2model.FrontendNlbTargetGroupState), } err := task.buildFrontendNlbListeners(context.Background(), tt.listenerPortConfigByIngress) diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 265782bb5c..f9ba29e310 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -46,7 +46,7 @@ const ( // ModelBuilder is responsible for build mode stack for a IngressGroup. type ModelBuilder interface { // build mode stack for a IngressGroup. - Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *core.FrontendNlbTargetGroupDesiredState, *elbv2model.LoadBalancer, error) + Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *elbv2model.LoadBalancer, error) } // NewDefaultModelBuilder constructs new defaultModelBuilder. @@ -135,9 +135,8 @@ type defaultModelBuilder struct { } // build mode stack for a IngressGroup. -func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *core.FrontendNlbTargetGroupDesiredState, *elbv2model.LoadBalancer, error) { +func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *elbv2model.LoadBalancer, error) { stack := core.NewDefaultStack(core.StackID(ingGroup.ID)) - frontendNlbTargetGroupDesiredState := core.NewFrontendNlbTargetGroupDesiredState() task := &defaultModelBuildTask{ k8sClient: b.k8sClient, @@ -165,9 +164,8 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics enableIPTargetType: b.enableIPTargetType, metricsCollector: b.metricsCollector, - ingGroup: ingGroup, - stack: stack, - frontendNlbTargetGroupDesiredState: frontendNlbTargetGroupDesiredState, + ingGroup: ingGroup, + stack: stack, defaultTags: b.defaultTags, externalManagedTags: b.externalManagedTags, @@ -192,11 +190,14 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics backendServices: make(map[types.NamespacedName]*corev1.Service), targetGroupNameToArnMapper: b.targetGroupNameToArnMapper, webACLNameToArnMapper: b.webACLNameToArnMapper, + localFrontendNlbData: make(map[string]*elbv2model.FrontendNlbTargetGroupState), } if err := task.run(ctx); err != nil { - return nil, nil, nil, false, nil, nil, err + return nil, nil, nil, false, nil, err } - return task.stack, task.loadBalancer, task.secretKeys, task.backendSGAllocated, frontendNlbTargetGroupDesiredState, task.frontendNlb, nil + + _ = elbv2model.NewFrontendNlbTargetGroupDesiredState(task.stack, task.localFrontendNlbData) + return task.stack, task.loadBalancer, task.secretKeys, task.backendSGAllocated, task.frontendNlb, nil } // the default model build task @@ -248,14 +249,14 @@ type defaultModelBuildTask struct { defaultHealthCheckMatcherHTTPCode string defaultHealthCheckMatcherGRPCCode string - loadBalancer *elbv2model.LoadBalancer - tgByResID map[string]*elbv2model.TargetGroup - backendServices map[types.NamespacedName]*corev1.Service - secretKeys []types.NamespacedName - frontendNlb *elbv2model.LoadBalancer - frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState - targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper - webACLNameToArnMapper *webACLNameToArnMapper + loadBalancer *elbv2model.LoadBalancer + tgByResID map[string]*elbv2model.TargetGroup + backendServices map[types.NamespacedName]*corev1.Service + secretKeys []types.NamespacedName + frontendNlb *elbv2model.LoadBalancer + localFrontendNlbData map[string]*elbv2model.FrontendNlbTargetGroupState + targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper + webACLNameToArnMapper *webACLNameToArnMapper metricsCollector lbcmetrics.MetricCollector } diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index 2f2c34bc4b..a42be793d0 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -299,6 +299,11 @@ const baseStackJSON = ` } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding":{ "ns-1/ing-1-svc-1:http":{ "spec":{ @@ -798,6 +803,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": { "spec": { @@ -1735,6 +1745,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { "ns-1/ing-1-svc-2:http": null, "ns-1/ing-1-svc-3:https": null }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:80": { "spec": { @@ -2603,6 +2618,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding":{ "ns-1/ing-1-svc-1:http":{ "spec":{ @@ -3166,6 +3186,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -3398,6 +3423,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -3668,6 +3698,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -3829,6 +3864,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -4793,7 +4833,7 @@ func Test_defaultModelBuilder_Build(t *testing.T) { b.enableIPTargetType = *tt.enableIPTargetType } - gotStack, _, _, _, _, _, err := b.Build(context.Background(), tt.args.ingGroup, b.metricsCollector) + gotStack, _, _, _, _, err := b.Build(context.Background(), tt.args.ingGroup, b.metricsCollector) if tt.wantErr != "" { assert.EqualError(t, err, tt.wantErr) } else { diff --git a/pkg/model/core/frontend_nlb_target_group.go b/pkg/model/core/frontend_nlb_target_group.go deleted file mode 100644 index f83ef844b0..0000000000 --- a/pkg/model/core/frontend_nlb_target_group.go +++ /dev/null @@ -1,31 +0,0 @@ -package core - -// FrontendNlbTargetGroupState represents the state of a single ALB Target Type target group with its ALB target -type FrontendNlbTargetGroupState struct { - Name string - ARN StringToken - Port int32 - TargetARN StringToken - TargetPort int32 -} - -// FrontendNlbTargetGroupDesiredState maintains a mapping of target groups targeting ALB -type FrontendNlbTargetGroupDesiredState struct { - TargetGroups map[string]*FrontendNlbTargetGroupState -} - -func NewFrontendNlbTargetGroupDesiredState() *FrontendNlbTargetGroupDesiredState { - return &FrontendNlbTargetGroupDesiredState{ - TargetGroups: make(map[string]*FrontendNlbTargetGroupState), - } -} - -func (m *FrontendNlbTargetGroupDesiredState) AddTargetGroup(targetGroupName string, targetGroupARN StringToken, targetARN StringToken, port int32, targetPort int32) { - m.TargetGroups[targetGroupName] = &FrontendNlbTargetGroupState{ - Name: targetGroupName, - ARN: targetGroupARN, - Port: port, - TargetARN: targetARN, - TargetPort: targetPort, - } -} diff --git a/pkg/model/elbv2/frontend_nlb_target_group.go b/pkg/model/elbv2/frontend_nlb_target_group.go new file mode 100644 index 0000000000..cf3cd6f446 --- /dev/null +++ b/pkg/model/elbv2/frontend_nlb_target_group.go @@ -0,0 +1,37 @@ +package elbv2 + +import "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + +var _ core.Resource = &FrontendNlbTargetGroupDesiredState{} + +const ( + FrontNLBResourceId = "FrontendNLBTargetGroup" +) + +// FrontendNlbTargetGroupState represents the state of a single ALB Target Type target group with its ALB target +type FrontendNlbTargetGroupState struct { + Name string + ARN core.StringToken + // Port -> NLB Listener Port + Port int32 + TargetARN core.StringToken + // TargetPort -> ALB Listener Port + TargetPort int32 +} + +// FrontendNlbTargetGroupDesiredState maintains a mapping of target groups targeting ALB +type FrontendNlbTargetGroupDesiredState struct { + core.ResourceMeta `json:"-"` + + // Maps target group name -> The FE NLB configuration. + TargetGroups map[string]*FrontendNlbTargetGroupState +} + +func NewFrontendNlbTargetGroupDesiredState(stack core.Stack, stateConfig map[string]*FrontendNlbTargetGroupState) *FrontendNlbTargetGroupDesiredState { + desiredState := &FrontendNlbTargetGroupDesiredState{ + ResourceMeta: core.NewResourceMeta(stack, FrontNLBResourceId, FrontNLBResourceId), + TargetGroups: stateConfig, + } + stack.AddResource(desiredState) + return desiredState +} diff --git a/test/e2e/gateway/alb_nlb_test.go b/test/e2e/gateway/alb_nlb_test.go new file mode 100644 index 0000000000..4872bcc07c --- /dev/null +++ b/test/e2e/gateway/alb_nlb_test.go @@ -0,0 +1,277 @@ +package gateway + +import ( + "context" + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "strings" + "time" +) + +var _ = Describe("test combined ALB and NLB gateways with HTTPRoute and TCPRoute", func() { + var ( + ctx context.Context + albStack ALBTestStack + nlbStack NLBTestStack + ) + + BeforeEach(func() { + if !tf.Options.EnableGatewayTests { + Skip("Skipping gateway tests") + } + ctx = context.Background() + albStack = ALBTestStack{} + nlbStack = NLBTestStack{} + }) + + AfterEach(func() { + albStack.Cleanup(ctx, tf) + nlbStack.Cleanup(ctx, tf) + }) + + Context("with ALB and NLB gateways using IP targets", func() { + var albDnsName string + var albARN string + var nlbDnsName string + var nlbARN string + var refGrant *gwbeta1.ReferenceGrant + It("should provision both ALB and NLB load balancers with HTTPRoute and TCPRoute", func() { + // ALB Configuration + albInterf := elbv2gw.LoadBalancerSchemeInternal + albLbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ + Scheme: &albInterf, + } + + // NLB Configuration + nlbInterf := elbv2gw.LoadBalancerSchemeInternetFacing + nlbLbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ + Scheme: &nlbInterf, + } + + // Configure TLS for both if certificates are available + var hasTLS bool + if len(tf.Options.CertificateARNs) > 0 { + cert := strings.Split(tf.Options.CertificateARNs, ",")[0] + + // ALB HTTPS listener + albLsConfig := elbv2gw.ListenerConfiguration{ + ProtocolPort: "HTTPS:443", + DefaultCertificate: &cert, + } + albLbcSpec.ListenerConfigurations = &[]elbv2gw.ListenerConfiguration{albLsConfig} + hasTLS = true + } + + // IP target type for both + ipTargetType := elbv2gw.TargetTypeIP + tgSpec := elbv2gw.TargetGroupConfigurationSpec{ + DefaultConfiguration: elbv2gw.TargetGroupProps{ + TargetType: &ipTargetType, + }, + } + lrcSpec := elbv2gw.ListenerRuleConfigurationSpec{} + + // ALB Gateway listeners + albGwListeners := []gwv1.Listener{ + { + Name: "http80", + Port: 80, + Protocol: gwv1.HTTPProtocolType, + }, + } + if hasTLS { + albGwListeners = append(albGwListeners, gwv1.Listener{ + Name: "https443", + Port: 443, + Protocol: gwv1.HTTPSProtocolType, + }) + } + + // HTTPRoute for ALB + httpr := buildHTTPRoute([]string{}, []gwv1.HTTPRouteRule{}, nil) + + By("deploying ALB stack", func() { + err := albStack.DeployHTTP(ctx, nil, tf, albGwListeners, []*gwv1.HTTPRoute{httpr}, albLbcSpec, tgSpec, lrcSpec, nil, true) + Expect(err).NotTo(HaveOccurred()) + }) + + By("deploying NLB stack", func() { + err := nlbStack.DeployFrontendNLB(ctx, albStack, tf, nlbLbcSpec, hasTLS, true) + Expect(err).NotTo(HaveOccurred()) + }) + By("checking alb gateway status for lb dns name", func() { + time.Sleep(2 * time.Minute) + albDnsName = albStack.GetLoadBalancerIngressHostName() + Expect(albDnsName).ToNot(BeEmpty()) + }) + By("querying AWS loadbalancer from the dns name", func() { + var err error + albARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, albDnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(albARN).ToNot(BeEmpty()) + }) + By("checking nlb gateway status for lb dns name", func() { + nlbDnsName = nlbStack.GetLoadBalancerIngressHostName() + Expect(nlbDnsName).ToNot(BeEmpty()) + }) + By("querying AWS loadbalancer from the dns name", func() { + var err error + nlbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, nlbDnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(nlbARN).ToNot(BeEmpty()) + }) + By("verify alb configuration", func() { + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "HTTP", + Port: 80, + NumTargets: int(*albStack.albResourceStack.commonStack.dps[0].Spec.Replicas), + TargetType: "ip", + TargetGroupHC: DEFAULT_ALB_TARGET_GROUP_HC, + }, + } + + listenerPortMap := albStack.albResourceStack.getListenersPortMap() + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, albARN, verifier.LoadBalancerExpectation{ + Type: "application", + Scheme: "internal", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("verify nlb configuration", func() { + // No ref grants, means no tg or listener. + expectedTargetGroups := []verifier.ExpectedTargetGroup{} + + listenerPortMap := map[string]string{} + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, nlbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("deploy reference grant that allows nlb <-> alb attachment", func() { + var err error + refGrant, err = nlbStack.CreateFENLBReferenceGrant(ctx, tf, albStack.albResourceStack.commonStack.ns) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(2 * time.Minute) + }) + By("deploy reference grant that allows nlb <-> alb attachment", func() { + // No ref grants, means no tg or listener. + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "TCP", + Port: 80, + NumTargets: 1, + TargetType: "alb", + TargetGroupHC: &verifier.TargetGroupHC{ + Protocol: "HTTP", + Port: "traffic-port", + Path: "/", + Interval: 15, + Timeout: 5, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }, + } + + if hasTLS { + expectedTargetGroups = append(expectedTargetGroups, verifier.ExpectedTargetGroup{ + Protocol: "TCP", + Port: 443, + NumTargets: 1, + TargetType: "alb", + TargetGroupHC: &verifier.TargetGroupHC{ + Protocol: "HTTPS", + Port: "traffic-port", + Path: "/", + Interval: 15, + Timeout: 5, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }) + } + + fmt.Printf("%+v\n", refGrant) + + listenerPortMap := nlbStack.nlbResourceStack.getListenersPortMap() + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, nlbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("verify port 80 works", func() { + url := fmt.Sprintf("http://%v/any-path", nlbDnsName) + err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + }) + if hasTLS { + By("verify port 443 works", func() { + url := fmt.Sprintf("https://%v/any-path", nlbDnsName) + urlOptions := http.URLOptions{ + InsecureSkipVerify: true, + } + err := tf.HTTPVerifier.VerifyURLWithOptions(url, urlOptions, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + }) + } + By("remove reference grant should remove nlb listener but keep alb listener intact", func() { + err := tf.K8sClient.Delete(ctx, refGrant) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(2 * time.Minute) + }) + By("verify alb configuration", func() { + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "HTTP", + Port: 80, + NumTargets: int(*albStack.albResourceStack.commonStack.dps[0].Spec.Replicas), + TargetType: "ip", + TargetGroupHC: DEFAULT_ALB_TARGET_GROUP_HC, + }, + } + + listenerPortMap := albStack.albResourceStack.getListenersPortMap() + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, albARN, verifier.LoadBalancerExpectation{ + Type: "application", + Scheme: "internal", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("verify nlb configuration", func() { + // No ref grants, means no tg or listener. + expectedTargetGroups := []verifier.ExpectedTargetGroup{} + + listenerPortMap := map[string]string{} + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, nlbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) +}) diff --git a/test/e2e/gateway/nlb_test_helper.go b/test/e2e/gateway/nlb_test_helper.go index 9c79198dfc..5fdca62ae6 100644 --- a/test/e2e/gateway/nlb_test_helper.go +++ b/test/e2e/gateway/nlb_test_helper.go @@ -2,14 +2,18 @@ package gateway import ( "context" + "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/test/framework" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) type NLBTestStack struct { @@ -78,6 +82,118 @@ func (s *NLBTestStack) Deploy(ctx context.Context, f *framework.Framework, auxil return s.nlbResourceStack.Deploy(ctx, f) } +func (s *NLBTestStack) DeployFrontendNLB(ctx context.Context, albStack ALBTestStack, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, hasTLS bool, readinessGateEnabled bool) error { + gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb") + + if f.Options.IPFamily == framework.IPv6 { + v6 := elbv2gw.LoadBalancerIpAddressTypeDualstack + lbConfSpec.IpAddressType = &v6 + } + + listeners := []gwv1.Listener{ + { + Name: "port80", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + } + + tcprs := []*gwalpha2.TCPRoute{buildFENLBTCPRoute(albStack.albResourceStack.commonStack.gw.Name, albStack.albResourceStack.commonStack.gw.Namespace, gwalpha2.PortNumber(80))} + + if hasTLS { + listeners = append(listeners, gwv1.Listener{ + Name: "port443", + Port: 443, + Protocol: gwv1.TCPProtocolType, + }) + tcpForHTTPS := buildFENLBTCPRoute(albStack.albResourceStack.commonStack.gw.Name, albStack.albResourceStack.commonStack.gw.Namespace, gwalpha2.PortNumber(443)) + tcprs = append(tcprs, tcpForHTTPS) + + } + + gw := buildBasicGatewaySpec(gwc, listeners) + + lbc := buildLoadBalancerConfig(lbConfSpec) + + s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{}, []*corev1.Service{}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{}, tcprs, []*gwalpha2.UDPRoute{}, nil, "nlb-gateway-e2e", readinessGateEnabled) + + err := s.nlbResourceStack.Deploy(ctx, f) + if err != nil { + return err + } + + // The special TGC is just to support HTTPS and HTTP health check routes on the same underlying gateway. + if !hasTLS { + return nil + } + + // Hack to get TargetGroupConfiguration working correctly, as it needs the namespace which is allocated in the deploy step. + + http := elbv2gw.TargetGroupHealthCheckProtocolHTTP + https := elbv2gw.TargetGroupHealthCheckProtocolHTTPS + + return createTargetGroupConfigs(ctx, f, []*elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("alb-https-hc-config"), + Namespace: albStack.albResourceStack.commonStack.gw.Namespace, + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Group: nil, + Kind: awssdk.String("Gateway"), + Name: albStack.albResourceStack.commonStack.gw.Name, + }, + DefaultConfiguration: elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{HealthCheckProtocol: &http}, + }, + RouteConfigurations: []elbv2gw.RouteConfiguration{ + { + RouteIdentifier: elbv2gw.RouteIdentifier{ + RouteName: tcprs[1].Name, + RouteNamespace: tcprs[1].Namespace, + RouteKind: "TCPRoute", + }, + TargetGroupProps: elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{HealthCheckProtocol: &https}, + }, + }, + }, + }, + }, + }) +} + +func (s *NLBTestStack) CreateFENLBReferenceGrant(ctx context.Context, f *framework.Framework, albNamespace *corev1.Namespace) (*gwbeta1.ReferenceGrant, error) { + refGrant := &gwbeta1.ReferenceGrant{ + + ObjectMeta: metav1.ObjectMeta{ + Name: "refgrant-fe-nlb", + Namespace: albNamespace.Name, + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Group: gwbeta1.Group(gwbeta1.GroupName), + Kind: gwbeta1.Kind("TCPRoute"), + Namespace: gwbeta1.Namespace(s.nlbResourceStack.commonStack.ns.Name), + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: "Gateway", + }, + }, + }, + } + + if err := createReferenceGrants(ctx, f, []*gwbeta1.ReferenceGrant{refGrant}); err != nil { + return nil, err + } + + return refGrant, nil +} + func (s *NLBTestStack) Cleanup(ctx context.Context, f *framework.Framework) { s.nlbResourceStack.Cleanup(ctx, f) } diff --git a/test/e2e/gateway/shared_resource_definitions.go b/test/e2e/gateway/shared_resource_definitions.go index 1f3b943d54..87fa4b4879 100644 --- a/test/e2e/gateway/shared_resource_definitions.go +++ b/test/e2e/gateway/shared_resource_definitions.go @@ -315,6 +315,39 @@ func buildTCPRoute() *gwalpha2.TCPRoute { return tcpr } +func buildFENLBTCPRoute(albGatewayName, albNamespace string, port gwalpha2.PortNumber) *gwalpha2.TCPRoute { + tcpr := &gwalpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("fenlb-tcp-route-%d", port), + }, + Spec: gwalpha2.TCPRouteSpec{ + CommonRouteSpec: gwalpha2.CommonRouteSpec{ + ParentRefs: []gwv1.ParentReference{ + { + Name: defaultName, + Port: &port, + }, + }, + }, + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: gwv1.ObjectName(albGatewayName), + Kind: (*gwv1.Kind)(awssdk.String("Gateway")), + Namespace: (*gwv1.Namespace)(&albNamespace), + Port: &port, + }, + }, + }, + }, + }, + }, + } + return tcpr +} + func buildUDPRoute() *gwalpha2.UDPRoute { port := gwalpha2.PortNumber(8080) udpr := &gwalpha2.UDPRoute{ diff --git a/test/e2e/ingress/vanilla_ingress_test.go b/test/e2e/ingress/vanilla_ingress_test.go index 3d1291207a..031c83ece2 100644 --- a/test/e2e/ingress/vanilla_ingress_test.go +++ b/test/e2e/ingress/vanilla_ingress_test.go @@ -3,13 +3,14 @@ package ingress import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net/http" "strings" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/gavv/httpexpect/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -17,7 +18,6 @@ import ( corev1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/test/framework" "sigs.k8s.io/aws-load-balancer-controller/test/framework/fixture"